@aztec/world-state 0.0.1-commit.001888fc → 0.0.1-commit.017a351
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/native/ipc_world_state_instance.d.ts +45 -0
- package/dest/native/ipc_world_state_instance.d.ts.map +1 -0
- package/dest/native/ipc_world_state_instance.js +750 -0
- package/dest/native/merkle_trees_facade.d.ts +8 -4
- package/dest/native/merkle_trees_facade.d.ts.map +1 -1
- package/dest/native/merkle_trees_facade.js +34 -15
- package/dest/native/message.d.ts +7 -8
- package/dest/native/message.d.ts.map +1 -1
- package/dest/native/native_world_state.d.ts +19 -7
- package/dest/native/native_world_state.d.ts.map +1 -1
- package/dest/native/native_world_state.js +89 -24
- package/dest/native/native_world_state_instance.d.ts +21 -39
- package/dest/native/native_world_state_instance.d.ts.map +1 -1
- package/dest/native/native_world_state_instance.js +8 -202
- package/dest/native/world_state_ops_queue.js +5 -5
- package/dest/synchronizer/config.d.ts +1 -1
- package/dest/synchronizer/config.d.ts.map +1 -1
- package/dest/synchronizer/config.js +9 -10
- package/dest/synchronizer/factory.d.ts +4 -4
- package/dest/synchronizer/factory.d.ts.map +1 -1
- package/dest/synchronizer/factory.js +7 -6
- package/dest/synchronizer/server_world_state_synchronizer.d.ts +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.d.ts.map +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.js +40 -11
- package/dest/testing.d.ts +4 -3
- package/dest/testing.d.ts.map +1 -1
- package/dest/testing.js +10 -6
- package/package.json +13 -10
- package/src/native/ipc_world_state_instance.ts +863 -0
- package/src/native/merkle_trees_facade.ts +41 -19
- package/src/native/message.ts +6 -7
- package/src/native/native_world_state.ts +98 -35
- package/src/native/native_world_state_instance.ts +28 -276
- package/src/native/world_state_ops_queue.ts +5 -5
- package/src/synchronizer/config.ts +14 -10
- package/src/synchronizer/factory.ts +11 -10
- package/src/synchronizer/server_world_state_synchronizer.ts +35 -13
- package/src/testing.ts +8 -9
|
@@ -1,291 +1,43 @@
|
|
|
1
|
-
import {
|
|
2
|
-
ARCHIVE_HEIGHT,
|
|
3
|
-
DomainSeparator,
|
|
4
|
-
L1_TO_L2_MSG_TREE_HEIGHT,
|
|
5
|
-
MAX_NULLIFIERS_PER_TX,
|
|
6
|
-
MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
|
|
7
|
-
NOTE_HASH_TREE_HEIGHT,
|
|
8
|
-
NULLIFIER_TREE_HEIGHT,
|
|
9
|
-
PUBLIC_DATA_TREE_HEIGHT,
|
|
10
|
-
} from '@aztec/constants';
|
|
11
|
-
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
12
|
-
import { NativeWorldState as BaseNativeWorldState, MsgpackChannel } from '@aztec/native';
|
|
13
|
-
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
14
|
-
import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
15
|
-
|
|
16
|
-
import assert from 'assert';
|
|
17
|
-
import { cpus } from 'os';
|
|
18
|
-
|
|
19
|
-
import type { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
20
|
-
import type { WorldStateTreeMapSizes } from '../synchronizer/factory.js';
|
|
21
|
-
import {
|
|
1
|
+
import type {
|
|
22
2
|
WorldStateMessageType,
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
isWithCanonical,
|
|
27
|
-
isWithForkId,
|
|
28
|
-
isWithRevision,
|
|
3
|
+
WorldStateRequest,
|
|
4
|
+
WorldStateRequestCategories,
|
|
5
|
+
WorldStateResponse,
|
|
29
6
|
} from './message.js';
|
|
30
|
-
import { WorldStateOpsQueue } from './world_state_ops_queue.js';
|
|
31
|
-
|
|
32
|
-
const MAX_WORLD_STATE_THREADS = +(process.env.HARDWARE_CONCURRENCY || '16');
|
|
33
7
|
|
|
8
|
+
/**
|
|
9
|
+
* Backend-agnostic handle to a running aztec-wsdb world state, accessed by the TS layer.
|
|
10
|
+
*
|
|
11
|
+
* Two implementations exist:
|
|
12
|
+
* - {@link IpcWorldState} — talks to a standalone aztec-wsdb process over UDS or shared memory.
|
|
13
|
+
*
|
|
14
|
+
* The legacy in-process NAPI implementation has been removed; the C++ AVM (NAPI) now connects to
|
|
15
|
+
* the same aztec-wsdb process via UDS using the socket path returned by {@link getSocketPath}.
|
|
16
|
+
*/
|
|
34
17
|
export interface NativeWorldStateInstance {
|
|
18
|
+
/**
|
|
19
|
+
* Send a typed msgpack message to the backing world state and await its response.
|
|
20
|
+
*
|
|
21
|
+
* @param responseHandler — optional pre-resolution hook executed on the per-fork queue, useful
|
|
22
|
+
* for caching responses while the queue still holds the fork lock.
|
|
23
|
+
* @param errorHandler — optional pre-rejection hook executed on the per-fork queue.
|
|
24
|
+
*/
|
|
35
25
|
call<T extends WorldStateMessageType>(
|
|
36
26
|
messageType: T,
|
|
37
27
|
body: WorldStateRequest[T] & WorldStateRequestCategories,
|
|
28
|
+
responseHandler?: (response: WorldStateResponse[T]) => WorldStateResponse[T],
|
|
29
|
+
errorHandler?: (error: string) => void,
|
|
38
30
|
): Promise<WorldStateResponse[T]>;
|
|
39
|
-
// TODO(dbanks12): this returns any type, but we should strongly type it
|
|
40
|
-
getHandle(): any;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Strongly-typed interface to access the WorldState class in the native world_state_napi module.
|
|
45
|
-
*/
|
|
46
|
-
export class NativeWorldState implements NativeWorldStateInstance {
|
|
47
|
-
private open = true;
|
|
48
|
-
|
|
49
|
-
// We maintain a map of queue to fork
|
|
50
|
-
private queues = new Map<number, WorldStateOpsQueue>();
|
|
51
|
-
|
|
52
|
-
private instance: MsgpackChannel<WorldStateMessageType, WorldStateRequest, WorldStateResponse>;
|
|
53
|
-
|
|
54
|
-
/** Creates a new native WorldState instance */
|
|
55
|
-
constructor(
|
|
56
|
-
private readonly dataDir: string,
|
|
57
|
-
private readonly wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
58
|
-
private readonly prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
59
|
-
private readonly instrumentation: WorldStateInstrumentation,
|
|
60
|
-
bindings?: LoggerBindings,
|
|
61
|
-
private readonly log: Logger = createLogger('world-state:database', bindings),
|
|
62
|
-
) {
|
|
63
|
-
const threads = Math.min(cpus().length, MAX_WORLD_STATE_THREADS);
|
|
64
|
-
log.info(
|
|
65
|
-
`Creating world state data store at directory ${dataDir} with map sizes ${JSON.stringify(
|
|
66
|
-
wsTreeMapSizes,
|
|
67
|
-
)} and ${threads} threads.`,
|
|
68
|
-
);
|
|
69
|
-
const prefilledPublicDataBufferArray = prefilledPublicData.map(d => [d.slot.toBuffer(), d.value.toBuffer()]);
|
|
70
|
-
const ws = new BaseNativeWorldState(
|
|
71
|
-
dataDir,
|
|
72
|
-
{
|
|
73
|
-
[MerkleTreeId.NULLIFIER_TREE]: NULLIFIER_TREE_HEIGHT,
|
|
74
|
-
[MerkleTreeId.NOTE_HASH_TREE]: NOTE_HASH_TREE_HEIGHT,
|
|
75
|
-
[MerkleTreeId.PUBLIC_DATA_TREE]: PUBLIC_DATA_TREE_HEIGHT,
|
|
76
|
-
[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: L1_TO_L2_MSG_TREE_HEIGHT,
|
|
77
|
-
[MerkleTreeId.ARCHIVE]: ARCHIVE_HEIGHT,
|
|
78
|
-
},
|
|
79
|
-
{
|
|
80
|
-
[MerkleTreeId.NULLIFIER_TREE]: 2 * MAX_NULLIFIERS_PER_TX,
|
|
81
|
-
[MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
|
|
82
|
-
},
|
|
83
|
-
prefilledPublicDataBufferArray,
|
|
84
|
-
DomainSeparator.BLOCK_HEADER_HASH,
|
|
85
|
-
{
|
|
86
|
-
[MerkleTreeId.NULLIFIER_TREE]: wsTreeMapSizes.nullifierTreeMapSizeKb,
|
|
87
|
-
[MerkleTreeId.NOTE_HASH_TREE]: wsTreeMapSizes.noteHashTreeMapSizeKb,
|
|
88
|
-
[MerkleTreeId.PUBLIC_DATA_TREE]: wsTreeMapSizes.publicDataTreeMapSizeKb,
|
|
89
|
-
[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: wsTreeMapSizes.messageTreeMapSizeKb,
|
|
90
|
-
[MerkleTreeId.ARCHIVE]: wsTreeMapSizes.archiveTreeMapSizeKb,
|
|
91
|
-
},
|
|
92
|
-
threads,
|
|
93
|
-
);
|
|
94
|
-
this.instance = new MsgpackChannel(ws);
|
|
95
|
-
// Manually create the queue for the canonical fork
|
|
96
|
-
this.queues.set(0, new WorldStateOpsQueue());
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
public getDataDir() {
|
|
100
|
-
return this.dataDir;
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
public clone() {
|
|
104
|
-
return new NativeWorldState(
|
|
105
|
-
this.dataDir,
|
|
106
|
-
this.wsTreeMapSizes,
|
|
107
|
-
this.prefilledPublicData,
|
|
108
|
-
this.instrumentation,
|
|
109
|
-
this.log.getBindings(),
|
|
110
|
-
this.log,
|
|
111
|
-
);
|
|
112
|
-
}
|
|
113
31
|
|
|
114
32
|
/**
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
* that wraps the underlying C++ WorldState pointer.
|
|
118
|
-
* @returns The NAPI External handle to the native WorldState instance,since
|
|
119
|
-
* the NAPI external type is opaque, we return any (we could also use an opaque symbol type)
|
|
33
|
+
* UDS path the underlying aztec-wsdb process listens on. The C++ AVM uses this to attach to the
|
|
34
|
+
* same world state instance the TS layer is using.
|
|
120
35
|
*/
|
|
121
|
-
|
|
122
|
-
const worldStateWrapper = (this.instance as any).dest;
|
|
123
|
-
|
|
124
|
-
if (!worldStateWrapper) {
|
|
125
|
-
throw new Error('No WorldStateWrapper found');
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
if (typeof worldStateWrapper.getHandle !== 'function') {
|
|
129
|
-
throw new Error('WorldStateWrapper does not have getHandle method');
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
// Call getHandle() to get the NAPI External
|
|
133
|
-
try {
|
|
134
|
-
return worldStateWrapper.getHandle();
|
|
135
|
-
} catch (error) {
|
|
136
|
-
this.log.error('Failed to get native WorldState handle', error);
|
|
137
|
-
}
|
|
138
|
-
}
|
|
36
|
+
getSocketPath(): string;
|
|
139
37
|
|
|
140
38
|
/**
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
* @param body - The message body
|
|
144
|
-
* @param responseHandler - A callback accepting the response, executed on the job queue
|
|
145
|
-
* @param errorHandler - A callback called on request error, executed on the job queue
|
|
146
|
-
* @returns The response to the message
|
|
39
|
+
* Shut down the world state instance. Cancels any in-flight queues, closes the IPC channel, and
|
|
40
|
+
* terminates the underlying aztec-wsdb process. Idempotent.
|
|
147
41
|
*/
|
|
148
|
-
|
|
149
|
-
messageType: T,
|
|
150
|
-
body: WorldStateRequest[T] & WorldStateRequestCategories,
|
|
151
|
-
// allows for the pre-processing of responses on the job queue before being passed back
|
|
152
|
-
responseHandler = (response: WorldStateResponse[T]): WorldStateResponse[T] => response,
|
|
153
|
-
errorHandler = (_: string) => {},
|
|
154
|
-
): Promise<WorldStateResponse[T]> {
|
|
155
|
-
// Here we determine which fork the request is being executed against and whether it requires uncommitted data
|
|
156
|
-
// We use the fork Id to select the appropriate request queue and the uncommitted data flag to pass to the queue
|
|
157
|
-
let forkId = -1;
|
|
158
|
-
// We assume it includes uncommitted unless explicitly told otherwise
|
|
159
|
-
let committedOnly = false;
|
|
160
|
-
|
|
161
|
-
// Canonical requests ALWAYS go against the canonical fork
|
|
162
|
-
// These include things like block syncs/unwinds etc
|
|
163
|
-
// These requests don't contain a fork ID
|
|
164
|
-
if (isWithCanonical(body)) {
|
|
165
|
-
forkId = 0;
|
|
166
|
-
} else if (isWithForkId(body)) {
|
|
167
|
-
forkId = body.forkId;
|
|
168
|
-
} else if (isWithRevision(body)) {
|
|
169
|
-
forkId = body.revision.forkId;
|
|
170
|
-
committedOnly = body.revision.includeUncommitted === false;
|
|
171
|
-
} else {
|
|
172
|
-
const _: never = body;
|
|
173
|
-
throw new Error(`Unable to determine forkId for message=${WorldStateMessageType[messageType]}`);
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
// Get the queue or create a new one
|
|
177
|
-
let requestQueue = this.queues.get(forkId);
|
|
178
|
-
if (requestQueue === undefined) {
|
|
179
|
-
requestQueue = new WorldStateOpsQueue();
|
|
180
|
-
this.queues.set(forkId, requestQueue);
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
// Enqueue the request and wait for the response
|
|
184
|
-
const response = await requestQueue.execute(
|
|
185
|
-
async () => {
|
|
186
|
-
assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the native instance');
|
|
187
|
-
assert.equal(this.open, true, 'Native instance is closed');
|
|
188
|
-
let response: WorldStateResponse[T];
|
|
189
|
-
try {
|
|
190
|
-
response = await this._sendMessage(messageType, body);
|
|
191
|
-
} catch (error: any) {
|
|
192
|
-
errorHandler(error.message);
|
|
193
|
-
throw error;
|
|
194
|
-
}
|
|
195
|
-
return responseHandler(response);
|
|
196
|
-
},
|
|
197
|
-
messageType,
|
|
198
|
-
committedOnly,
|
|
199
|
-
);
|
|
200
|
-
|
|
201
|
-
// If the request was to delete the fork then we clean it up here
|
|
202
|
-
if (messageType === WorldStateMessageType.DELETE_FORK) {
|
|
203
|
-
await requestQueue.stop();
|
|
204
|
-
this.queues.delete(forkId);
|
|
205
|
-
}
|
|
206
|
-
return response;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
/**
|
|
210
|
-
* Stops the native instance.
|
|
211
|
-
*/
|
|
212
|
-
public async close(): Promise<void> {
|
|
213
|
-
if (!this.open) {
|
|
214
|
-
return;
|
|
215
|
-
}
|
|
216
|
-
this.open = false;
|
|
217
|
-
const queue = this.queues.get(0)!;
|
|
218
|
-
|
|
219
|
-
await queue.execute(
|
|
220
|
-
async () => {
|
|
221
|
-
await this._sendMessage(WorldStateMessageType.CLOSE, { canonical: true });
|
|
222
|
-
},
|
|
223
|
-
WorldStateMessageType.CLOSE,
|
|
224
|
-
false,
|
|
225
|
-
);
|
|
226
|
-
await queue.stop();
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
private async _sendMessage<T extends WorldStateMessageType>(
|
|
230
|
-
messageType: T,
|
|
231
|
-
body: WorldStateRequest[T] & WorldStateRequestCategories,
|
|
232
|
-
): Promise<WorldStateResponse[T]> {
|
|
233
|
-
let logMetadata: Record<string, any> = {};
|
|
234
|
-
|
|
235
|
-
if (body) {
|
|
236
|
-
if ('treeId' in body) {
|
|
237
|
-
logMetadata['treeId'] = MerkleTreeId[body.treeId];
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
if ('revision' in body) {
|
|
241
|
-
logMetadata = { ...logMetadata, ...body.revision };
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
if ('forkId' in body) {
|
|
245
|
-
logMetadata['forkId'] = body.forkId;
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
if ('blockNumber' in body) {
|
|
249
|
-
logMetadata['blockNumber'] = body.blockNumber;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
if ('toBlockNumber' in body) {
|
|
253
|
-
logMetadata['toBlockNumber'] = body.toBlockNumber;
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
if ('leafIndex' in body) {
|
|
257
|
-
logMetadata['leafIndex'] = body.leafIndex;
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
if ('blockHeaderHash' in body) {
|
|
261
|
-
logMetadata['blockHeaderHash'] = '0x' + body.blockHeaderHash.toString('hex');
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
if ('leaves' in body) {
|
|
265
|
-
logMetadata['leavesCount'] = body.leaves.length;
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
// sync operation
|
|
269
|
-
if ('paddedNoteHashes' in body) {
|
|
270
|
-
logMetadata['notesCount'] = body.paddedNoteHashes.length;
|
|
271
|
-
logMetadata['nullifiersCount'] = body.paddedNullifiers.length;
|
|
272
|
-
logMetadata['l1ToL2MessagesCount'] = body.paddedL1ToL2Messages.length;
|
|
273
|
-
logMetadata['publicDataWritesCount'] = body.publicDataWrites.length;
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
try {
|
|
278
|
-
const { duration, response } = await this.instance.sendMessage(messageType, body);
|
|
279
|
-
this.log.trace(`Call ${WorldStateMessageType[messageType]} took (ms)`, {
|
|
280
|
-
duration,
|
|
281
|
-
...logMetadata,
|
|
282
|
-
});
|
|
283
|
-
|
|
284
|
-
this.instrumentation.recordRoundTrip(duration.totalUs, messageType);
|
|
285
|
-
return response;
|
|
286
|
-
} catch (error) {
|
|
287
|
-
this.log.error(`Call ${WorldStateMessageType[messageType]} failed: ${error}`, error, logMetadata);
|
|
288
|
-
throw error;
|
|
289
|
-
}
|
|
290
|
-
}
|
|
42
|
+
close(): Promise<void>;
|
|
291
43
|
}
|
|
@@ -96,7 +96,7 @@ export class WorldStateOpsQueue {
|
|
|
96
96
|
// then send the request immediately
|
|
97
97
|
// If a mutating request is in flight then we must wait
|
|
98
98
|
// If a mutating request is not in flight but something is queued then it must be a mutating request
|
|
99
|
-
if (this.inFlightMutatingCount
|
|
99
|
+
if (this.inFlightMutatingCount === 0 && this.requests.length === 0) {
|
|
100
100
|
this.sendEnqueuedRequest(op);
|
|
101
101
|
} else {
|
|
102
102
|
this.requests.push(op);
|
|
@@ -122,7 +122,7 @@ export class WorldStateOpsQueue {
|
|
|
122
122
|
--this.inFlightCount;
|
|
123
123
|
|
|
124
124
|
// If there are still requests in flight then do nothing further
|
|
125
|
-
if (this.inFlightCount
|
|
125
|
+
if (this.inFlightCount !== 0) {
|
|
126
126
|
return;
|
|
127
127
|
}
|
|
128
128
|
|
|
@@ -134,7 +134,7 @@ export class WorldStateOpsQueue {
|
|
|
134
134
|
while (this.requests.length > 0) {
|
|
135
135
|
const next = this.requests[0];
|
|
136
136
|
if (next.mutating) {
|
|
137
|
-
if (this.inFlightCount
|
|
137
|
+
if (this.inFlightCount === 0) {
|
|
138
138
|
// send the mutating request
|
|
139
139
|
this.requests.shift();
|
|
140
140
|
this.sendEnqueuedRequest(next);
|
|
@@ -149,7 +149,7 @@ export class WorldStateOpsQueue {
|
|
|
149
149
|
}
|
|
150
150
|
|
|
151
151
|
// If the queue is empty, there is nothing in flight and we have been told to stop, then resolve the stop promise
|
|
152
|
-
if (this.inFlightCount
|
|
152
|
+
if (this.inFlightCount === 0 && this.stopResolve !== undefined) {
|
|
153
153
|
this.stopResolve();
|
|
154
154
|
}
|
|
155
155
|
}
|
|
@@ -182,7 +182,7 @@ export class WorldStateOpsQueue {
|
|
|
182
182
|
});
|
|
183
183
|
|
|
184
184
|
// If no outstanding requests then immediately resolve the promise
|
|
185
|
-
if (this.requests.length
|
|
185
|
+
if (this.requests.length === 0 && this.inFlightCount === 0 && this.stopResolve !== undefined) {
|
|
186
186
|
this.stopResolve();
|
|
187
187
|
}
|
|
188
188
|
return this.stopPromise;
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
type ConfigMappingsType,
|
|
3
|
+
getConfigFromMappings,
|
|
4
|
+
numberConfigHelper,
|
|
5
|
+
optionalNumberConfigHelper,
|
|
6
|
+
} from '@aztec/foundation/config';
|
|
2
7
|
|
|
3
8
|
/** World State synchronizer configuration values. */
|
|
4
9
|
export interface WorldStateConfig {
|
|
@@ -36,47 +41,46 @@ export interface WorldStateConfig {
|
|
|
36
41
|
export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
|
|
37
42
|
worldStateBlockCheckIntervalMS: {
|
|
38
43
|
env: 'WS_BLOCK_CHECK_INTERVAL_MS',
|
|
39
|
-
|
|
40
|
-
defaultValue: 100,
|
|
44
|
+
...numberConfigHelper(100),
|
|
41
45
|
description: 'The frequency in which to check.',
|
|
42
46
|
},
|
|
43
47
|
worldStateBlockRequestBatchSize: {
|
|
44
48
|
env: 'WS_BLOCK_REQUEST_BATCH_SIZE',
|
|
45
|
-
|
|
49
|
+
...optionalNumberConfigHelper(),
|
|
46
50
|
description: 'Size of the batch for each get-blocks request from the synchronizer to the archiver.',
|
|
47
51
|
},
|
|
48
52
|
worldStateDbMapSizeKb: {
|
|
49
53
|
env: 'WS_DB_MAP_SIZE_KB',
|
|
50
|
-
|
|
54
|
+
...optionalNumberConfigHelper(),
|
|
51
55
|
description: 'The maximum possible size of the world state DB in KB. Overwrites the general dataStoreMapSizeKb.',
|
|
52
56
|
},
|
|
53
57
|
archiveTreeMapSizeKb: {
|
|
54
58
|
env: 'ARCHIVE_TREE_MAP_SIZE_KB',
|
|
55
|
-
|
|
59
|
+
...optionalNumberConfigHelper(),
|
|
56
60
|
description:
|
|
57
61
|
'The maximum possible size of the world state archive tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
58
62
|
},
|
|
59
63
|
nullifierTreeMapSizeKb: {
|
|
60
64
|
env: 'NULLIFIER_TREE_MAP_SIZE_KB',
|
|
61
|
-
|
|
65
|
+
...optionalNumberConfigHelper(),
|
|
62
66
|
description:
|
|
63
67
|
'The maximum possible size of the world state nullifier tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
64
68
|
},
|
|
65
69
|
noteHashTreeMapSizeKb: {
|
|
66
70
|
env: 'NOTE_HASH_TREE_MAP_SIZE_KB',
|
|
67
|
-
|
|
71
|
+
...optionalNumberConfigHelper(),
|
|
68
72
|
description:
|
|
69
73
|
'The maximum possible size of the world state note hash tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
70
74
|
},
|
|
71
75
|
messageTreeMapSizeKb: {
|
|
72
76
|
env: 'MESSAGE_TREE_MAP_SIZE_KB',
|
|
73
|
-
|
|
77
|
+
...optionalNumberConfigHelper(),
|
|
74
78
|
description:
|
|
75
79
|
'The maximum possible size of the world state message tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
76
80
|
},
|
|
77
81
|
publicDataTreeMapSizeKb: {
|
|
78
82
|
env: 'PUBLIC_DATA_TREE_MAP_SIZE_KB',
|
|
79
|
-
|
|
83
|
+
...optionalNumberConfigHelper(),
|
|
80
84
|
description:
|
|
81
85
|
'The maximum possible size of the world state public data tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
82
86
|
},
|
|
@@ -2,7 +2,7 @@ import type { LoggerBindings } from '@aztec/foundation/log';
|
|
|
2
2
|
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
3
3
|
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
4
4
|
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
5
|
-
import type
|
|
5
|
+
import { EMPTY_GENESIS_DATA, type GenesisData, isGenesisData } from '@aztec/stdlib/world-state';
|
|
6
6
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
7
7
|
|
|
8
8
|
import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
@@ -21,12 +21,14 @@ export interface WorldStateTreeMapSizes {
|
|
|
21
21
|
export async function createWorldStateSynchronizer(
|
|
22
22
|
config: WorldStateConfig & DataStoreConfig,
|
|
23
23
|
l2BlockSource: L2BlockSource & L1ToL2MessageSource,
|
|
24
|
-
|
|
24
|
+
genesisOrNativeWorldState: GenesisData | NativeWorldStateService,
|
|
25
25
|
client: TelemetryClient = getTelemetryClient(),
|
|
26
26
|
bindings?: LoggerBindings,
|
|
27
27
|
) {
|
|
28
28
|
const instrumentation = new WorldStateInstrumentation(client);
|
|
29
|
-
const merkleTrees =
|
|
29
|
+
const merkleTrees = isGenesisData(genesisOrNativeWorldState)
|
|
30
|
+
? await createWorldState(config, genesisOrNativeWorldState, instrumentation, bindings)
|
|
31
|
+
: genesisOrNativeWorldState;
|
|
30
32
|
return new ServerWorldStateSynchronizer(merkleTrees, l2BlockSource, config, instrumentation);
|
|
31
33
|
}
|
|
32
34
|
|
|
@@ -41,8 +43,8 @@ export async function createWorldState(
|
|
|
41
43
|
| 'messageTreeMapSizeKb'
|
|
42
44
|
| 'publicDataTreeMapSizeKb'
|
|
43
45
|
> &
|
|
44
|
-
Pick<DataStoreConfig, 'dataDirectory' | 'dataStoreMapSizeKb' | '
|
|
45
|
-
|
|
46
|
+
Pick<DataStoreConfig, 'dataDirectory' | 'dataStoreMapSizeKb' | 'rollupAddress'>,
|
|
47
|
+
genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
46
48
|
instrumentation: WorldStateInstrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
47
49
|
bindings?: LoggerBindings,
|
|
48
50
|
) {
|
|
@@ -56,24 +58,23 @@ export async function createWorldState(
|
|
|
56
58
|
publicDataTreeMapSizeKb: config.publicDataTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
57
59
|
};
|
|
58
60
|
|
|
59
|
-
if (!config.
|
|
61
|
+
if (!config.rollupAddress) {
|
|
60
62
|
throw new Error('Rollup address is required to create a world state synchronizer.');
|
|
61
63
|
}
|
|
62
64
|
|
|
63
65
|
// If a data directory is provided in config, then create a persistent store.
|
|
64
66
|
const merkleTrees = dataDirectory
|
|
65
67
|
? await NativeWorldStateService.new(
|
|
66
|
-
config.
|
|
68
|
+
config.rollupAddress,
|
|
67
69
|
dataDirectory,
|
|
68
70
|
wsTreeMapSizes,
|
|
69
|
-
|
|
71
|
+
genesis,
|
|
70
72
|
instrumentation,
|
|
71
73
|
bindings,
|
|
72
74
|
)
|
|
73
75
|
: await NativeWorldStateService.tmp(
|
|
74
|
-
config.l1Contracts.rollupAddress,
|
|
75
76
|
!['true', '1'].includes(process.env.DEBUG_WORLD_STATE!),
|
|
76
|
-
|
|
77
|
+
genesis,
|
|
77
78
|
instrumentation,
|
|
78
79
|
bindings,
|
|
79
80
|
);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { INITIAL_CHECKPOINT_NUMBER } from '@aztec/constants';
|
|
2
2
|
import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
|
|
3
3
|
import type { Fr } from '@aztec/foundation/curves/bn254';
|
|
4
4
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
@@ -287,10 +287,15 @@ export class ServerWorldStateSynchronizer
|
|
|
287
287
|
// but we use a block stream so we need to provide 'local' L2Tips.
|
|
288
288
|
// We configure the block stream to ignore checkpoints and set checkpoint values to genesis here.
|
|
289
289
|
const genesisCheckpointHeaderHash = GENESIS_CHECKPOINT_HEADER_HASH.toString();
|
|
290
|
+
const initialBlockHash = (await this.merkleTreeCommitted.getInitialHeader().hash()).toString();
|
|
290
291
|
return {
|
|
291
292
|
proposed: latestBlockId,
|
|
292
293
|
checkpointed: {
|
|
293
|
-
block: { number:
|
|
294
|
+
block: { number: BlockNumber.ZERO, hash: initialBlockHash },
|
|
295
|
+
checkpoint: { number: INITIAL_CHECKPOINT_NUMBER, hash: genesisCheckpointHeaderHash },
|
|
296
|
+
},
|
|
297
|
+
proposedCheckpoint: {
|
|
298
|
+
block: { number: BlockNumber.ZERO, hash: initialBlockHash },
|
|
294
299
|
checkpoint: { number: INITIAL_CHECKPOINT_NUMBER, hash: genesisCheckpointHeaderHash },
|
|
295
300
|
},
|
|
296
301
|
finalized: {
|
|
@@ -388,39 +393,54 @@ export class ServerWorldStateSynchronizer
|
|
|
388
393
|
|
|
389
394
|
private async handleChainFinalized(blockNumber: BlockNumber) {
|
|
390
395
|
this.log.verbose(`Finalized chain is now at block ${blockNumber}`);
|
|
396
|
+
// If the finalized block number is older than the oldest available block in world state,
|
|
397
|
+
// skip entirely. The finalized block number can jump backwards (e.g. when the finalization
|
|
398
|
+
// heuristic changes) and try to read block data that has already been pruned. When this
|
|
399
|
+
// happens, there is nothing useful to do — the native world state is already finalized
|
|
400
|
+
// past this point and pruning has already happened.
|
|
401
|
+
const currentSummary = await this.merkleTreeDb.getStatusSummary();
|
|
402
|
+
if (blockNumber < currentSummary.oldestHistoricalBlock || blockNumber < 1) {
|
|
403
|
+
this.log.trace(
|
|
404
|
+
`Finalized block ${blockNumber} is older than the oldest available block ${currentSummary.oldestHistoricalBlock}. Skipping.`,
|
|
405
|
+
);
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
391
408
|
const summary = await this.merkleTreeDb.setFinalized(blockNumber);
|
|
392
409
|
if (this.historyToKeep === undefined) {
|
|
393
410
|
return;
|
|
394
411
|
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
if (finalisedCheckpoint === undefined) {
|
|
412
|
+
const finalisedBlockData = await this.l2BlockSource.getBlockData({ number: summary.finalizedBlockNumber });
|
|
413
|
+
if (finalisedBlockData === undefined) {
|
|
398
414
|
this.log.warn(
|
|
399
415
|
`Failed to retrieve checkpointed block for finalized block number: ${summary.finalizedBlockNumber}`,
|
|
400
416
|
);
|
|
401
417
|
return;
|
|
402
418
|
}
|
|
403
419
|
// Compute the required historic checkpoint number
|
|
404
|
-
const newHistoricCheckpointNumber =
|
|
420
|
+
const newHistoricCheckpointNumber = finalisedBlockData.checkpointNumber - this.historyToKeep + 1;
|
|
405
421
|
if (newHistoricCheckpointNumber <= 1) {
|
|
406
422
|
return;
|
|
407
423
|
}
|
|
408
424
|
// Retrieve the historic checkpoint
|
|
409
|
-
const
|
|
410
|
-
CheckpointNumber(newHistoricCheckpointNumber),
|
|
411
|
-
|
|
412
|
-
)
|
|
413
|
-
if (historicCheckpoints.length === 0 || historicCheckpoints[0] === undefined) {
|
|
425
|
+
const historicCheckpoint = await this.l2BlockSource.getCheckpoint({
|
|
426
|
+
number: CheckpointNumber(newHistoricCheckpointNumber),
|
|
427
|
+
});
|
|
428
|
+
if (!historicCheckpoint) {
|
|
414
429
|
this.log.warn(`Failed to retrieve checkpoint number ${newHistoricCheckpointNumber} from Archiver`);
|
|
415
430
|
return;
|
|
416
431
|
}
|
|
417
|
-
const historicCheckpoint = historicCheckpoints[0];
|
|
418
432
|
if (historicCheckpoint.checkpoint.blocks.length === 0 || historicCheckpoint.checkpoint.blocks[0] === undefined) {
|
|
419
433
|
this.log.warn(`Retrieved checkpoint number ${newHistoricCheckpointNumber} has no blocks!`);
|
|
420
434
|
return;
|
|
421
435
|
}
|
|
422
436
|
// Find the block at the start of the checkpoint and remove blocks up to this one
|
|
423
437
|
const newHistoricBlock = historicCheckpoint.checkpoint.blocks[0];
|
|
438
|
+
if (newHistoricBlock.number <= currentSummary.oldestHistoricalBlock) {
|
|
439
|
+
this.log.debug(
|
|
440
|
+
`Historic block ${newHistoricBlock.number} is not newer than oldest available ${currentSummary.oldestHistoricalBlock}. Skipping prune.`,
|
|
441
|
+
);
|
|
442
|
+
return;
|
|
443
|
+
}
|
|
424
444
|
this.log.verbose(`Pruning historic blocks to ${newHistoricBlock.number}`);
|
|
425
445
|
const status = await this.merkleTreeDb.removeHistoricalBlocks(BlockNumber(newHistoricBlock.number));
|
|
426
446
|
this.log.debug(`World state summary `, status.summary);
|
|
@@ -435,7 +455,9 @@ export class ServerWorldStateSynchronizer
|
|
|
435
455
|
private async handleChainPruned(blockNumber: BlockNumber) {
|
|
436
456
|
this.log.info(`Chain pruned to block ${blockNumber}`);
|
|
437
457
|
const status = await this.merkleTreeDb.unwindBlocks(blockNumber);
|
|
438
|
-
this.provenBlockNumber
|
|
458
|
+
if (this.provenBlockNumber !== undefined && this.provenBlockNumber > blockNumber) {
|
|
459
|
+
this.provenBlockNumber = undefined;
|
|
460
|
+
}
|
|
439
461
|
this.instrumentation.updateWorldStateMetrics(status);
|
|
440
462
|
}
|
|
441
463
|
|
package/src/testing.ts
CHANGED
|
@@ -3,22 +3,19 @@ import { Fr } from '@aztec/foundation/curves/bn254';
|
|
|
3
3
|
import { computeFeePayerBalanceLeafSlot } from '@aztec/protocol-contracts/fee-juice';
|
|
4
4
|
import type { AztecAddress } from '@aztec/stdlib/aztec-address';
|
|
5
5
|
import { MerkleTreeId, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
6
|
+
import type { GenesisData } from '@aztec/stdlib/world-state';
|
|
6
7
|
|
|
7
8
|
import { NativeWorldStateService } from './native/index.js';
|
|
8
9
|
|
|
9
|
-
async function generateGenesisValues(
|
|
10
|
-
if (!prefilledPublicData.length) {
|
|
10
|
+
async function generateGenesisValues(genesis: GenesisData) {
|
|
11
|
+
if (!genesis.prefilledPublicData.length && genesis.genesisTimestamp === 0n) {
|
|
11
12
|
return {
|
|
12
13
|
genesisArchiveRoot: new Fr(GENESIS_ARCHIVE_ROOT),
|
|
13
14
|
};
|
|
14
15
|
}
|
|
15
16
|
|
|
16
17
|
// Create a temporary world state to compute the genesis values.
|
|
17
|
-
const ws = await NativeWorldStateService.tmp(
|
|
18
|
-
undefined /* rollupAddress */,
|
|
19
|
-
true /* cleanupTmpDir */,
|
|
20
|
-
prefilledPublicData,
|
|
21
|
-
);
|
|
18
|
+
const ws = await NativeWorldStateService.tmp(/*cleanupTmpDir=*/ true, genesis);
|
|
22
19
|
const genesisArchiveRoot = new Fr((await ws.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
|
|
23
20
|
await ws.close();
|
|
24
21
|
|
|
@@ -33,6 +30,7 @@ export async function getGenesisValues(
|
|
|
33
30
|
initialAccounts: AztecAddress[],
|
|
34
31
|
initialAccountFeeJuice = defaultInitialAccountFeeJuice,
|
|
35
32
|
genesisPublicData: PublicDataTreeLeaf[] = [],
|
|
33
|
+
genesisTimestamp: bigint = 0n,
|
|
36
34
|
) {
|
|
37
35
|
// Top up the accounts with fee juice.
|
|
38
36
|
let prefilledPublicData = await Promise.all(
|
|
@@ -46,11 +44,12 @@ export async function getGenesisValues(
|
|
|
46
44
|
|
|
47
45
|
prefilledPublicData.sort((a, b) => (b.slot.lt(a.slot) ? 1 : -1));
|
|
48
46
|
|
|
49
|
-
const
|
|
47
|
+
const genesis: GenesisData = { prefilledPublicData, genesisTimestamp };
|
|
48
|
+
const { genesisArchiveRoot } = await generateGenesisValues(genesis);
|
|
50
49
|
|
|
51
50
|
return {
|
|
52
51
|
genesisArchiveRoot,
|
|
53
|
-
|
|
52
|
+
genesis,
|
|
54
53
|
fundingNeeded: BigInt(initialAccounts.length) * initialAccountFeeJuice.toBigInt(),
|
|
55
54
|
};
|
|
56
55
|
}
|