@aztec/world-state 0.0.1-commit.1142ef1 → 0.0.1-commit.125b3452
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/instrumentation/instrumentation.d.ts +1 -1
- package/dest/instrumentation/instrumentation.d.ts.map +1 -1
- package/dest/instrumentation/instrumentation.js +9 -2
- package/dest/native/fork_checkpoint.d.ts +7 -1
- package/dest/native/fork_checkpoint.d.ts.map +1 -1
- package/dest/native/fork_checkpoint.js +15 -3
- package/dest/native/merkle_trees_facade.d.ts +5 -5
- package/dest/native/merkle_trees_facade.d.ts.map +1 -1
- package/dest/native/merkle_trees_facade.js +21 -9
- package/dest/native/message.d.ts +14 -5
- package/dest/native/message.d.ts.map +1 -1
- package/dest/native/native_world_state.d.ts +7 -7
- package/dest/native/native_world_state.d.ts.map +1 -1
- package/dest/native/native_world_state.js +9 -8
- package/dest/native/native_world_state_instance.d.ts +3 -3
- package/dest/native/native_world_state_instance.d.ts.map +1 -1
- package/dest/native/native_world_state_instance.js +4 -4
- package/dest/synchronizer/config.d.ts +3 -5
- package/dest/synchronizer/config.d.ts.map +1 -1
- package/dest/synchronizer/config.js +7 -9
- package/dest/synchronizer/factory.d.ts +5 -4
- package/dest/synchronizer/factory.d.ts.map +1 -1
- package/dest/synchronizer/factory.js +5 -5
- package/dest/synchronizer/server_world_state_synchronizer.d.ts +4 -5
- package/dest/synchronizer/server_world_state_synchronizer.d.ts.map +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.js +77 -34
- package/dest/test/utils.d.ts +7 -7
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +5 -5
- package/dest/world-state-db/merkle_tree_db.d.ts +3 -12
- package/dest/world-state-db/merkle_tree_db.d.ts.map +1 -1
- package/package.json +10 -11
- package/src/instrumentation/instrumentation.ts +9 -1
- package/src/native/fork_checkpoint.ts +19 -3
- package/src/native/merkle_trees_facade.ts +24 -8
- package/src/native/message.ts +15 -4
- package/src/native/native_world_state.ts +22 -10
- package/src/native/native_world_state_instance.ts +6 -4
- package/src/synchronizer/config.ts +8 -19
- package/src/synchronizer/factory.ts +8 -2
- package/src/synchronizer/server_world_state_synchronizer.ts +95 -44
- package/src/test/utils.ts +6 -6
- package/src/world-state-db/merkle_tree_db.ts +2 -12
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
Metrics,
|
|
8
8
|
type TelemetryClient,
|
|
9
9
|
type UpDownCounter,
|
|
10
|
+
createUpDownCounterWithDefault,
|
|
10
11
|
} from '@aztec/telemetry-client';
|
|
11
12
|
|
|
12
13
|
import {
|
|
@@ -63,7 +64,14 @@ export class WorldStateInstrumentation {
|
|
|
63
64
|
|
|
64
65
|
this.requestHistogram = meter.createHistogram(Metrics.WORLD_STATE_REQUEST_TIME);
|
|
65
66
|
|
|
66
|
-
this.criticalErrors = meter
|
|
67
|
+
this.criticalErrors = createUpDownCounterWithDefault(meter, Metrics.WORLD_STATE_CRITICAL_ERROR_COUNT, {
|
|
68
|
+
[Attributes.ERROR_TYPE]: [
|
|
69
|
+
'synch_pending_block',
|
|
70
|
+
'finalize_block',
|
|
71
|
+
'prune_pending_block',
|
|
72
|
+
'prune_historical_block',
|
|
73
|
+
],
|
|
74
|
+
});
|
|
67
75
|
}
|
|
68
76
|
|
|
69
77
|
private updateTreeStats(treeDbStats: TreeDBStats, treeMeta: TreeMeta, tree: MerkleTreeId) {
|
|
@@ -3,11 +3,14 @@ import type { MerkleTreeCheckpointOperations } from '@aztec/stdlib/interfaces/se
|
|
|
3
3
|
export class ForkCheckpoint {
|
|
4
4
|
private completed = false;
|
|
5
5
|
|
|
6
|
-
private constructor(
|
|
6
|
+
private constructor(
|
|
7
|
+
private readonly fork: MerkleTreeCheckpointOperations,
|
|
8
|
+
public readonly depth: number,
|
|
9
|
+
) {}
|
|
7
10
|
|
|
8
11
|
static async new(fork: MerkleTreeCheckpointOperations): Promise<ForkCheckpoint> {
|
|
9
|
-
await fork.createCheckpoint();
|
|
10
|
-
return new ForkCheckpoint(fork);
|
|
12
|
+
const depth = await fork.createCheckpoint();
|
|
13
|
+
return new ForkCheckpoint(fork, depth);
|
|
11
14
|
}
|
|
12
15
|
|
|
13
16
|
async commit(): Promise<void> {
|
|
@@ -27,4 +30,17 @@ export class ForkCheckpoint {
|
|
|
27
30
|
await this.fork.revertCheckpoint();
|
|
28
31
|
this.completed = true;
|
|
29
32
|
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Reverts this checkpoint and any nested checkpoints created on top of it,
|
|
36
|
+
* leaving the checkpoint depth at the level it was before this checkpoint was created.
|
|
37
|
+
*/
|
|
38
|
+
async revertToCheckpoint(): Promise<void> {
|
|
39
|
+
if (this.completed) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
await this.fork.revertAllCheckpointsTo(this.depth - 1);
|
|
44
|
+
this.completed = true;
|
|
45
|
+
}
|
|
30
46
|
}
|
|
@@ -292,10 +292,19 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
|
|
|
292
292
|
|
|
293
293
|
public async close(): Promise<void> {
|
|
294
294
|
assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set');
|
|
295
|
-
|
|
295
|
+
try {
|
|
296
|
+
await this.instance.call(WorldStateMessageType.DELETE_FORK, { forkId: this.revision.forkId });
|
|
297
|
+
} catch (err: any) {
|
|
298
|
+
// Ignore errors due to native instance being closed during shutdown.
|
|
299
|
+
// This can happen when validators are still processing block proposals while the node is stopping.
|
|
300
|
+
if (err?.message === 'Native instance is closed') {
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
throw err;
|
|
304
|
+
}
|
|
296
305
|
}
|
|
297
306
|
|
|
298
|
-
async [Symbol.
|
|
307
|
+
async [Symbol.asyncDispose](): Promise<void> {
|
|
299
308
|
if (this.opts.closeDelayMs) {
|
|
300
309
|
void sleep(this.opts.closeDelayMs)
|
|
301
310
|
.then(() => this.close())
|
|
@@ -310,9 +319,10 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
|
|
|
310
319
|
}
|
|
311
320
|
}
|
|
312
321
|
|
|
313
|
-
public async createCheckpoint(): Promise<
|
|
322
|
+
public async createCheckpoint(): Promise<number> {
|
|
314
323
|
assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set');
|
|
315
|
-
await this.instance.call(WorldStateMessageType.CREATE_CHECKPOINT, { forkId: this.revision.forkId });
|
|
324
|
+
const resp = await this.instance.call(WorldStateMessageType.CREATE_CHECKPOINT, { forkId: this.revision.forkId });
|
|
325
|
+
return resp.depth;
|
|
316
326
|
}
|
|
317
327
|
|
|
318
328
|
public async commitCheckpoint(): Promise<void> {
|
|
@@ -325,14 +335,20 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
|
|
|
325
335
|
await this.instance.call(WorldStateMessageType.REVERT_CHECKPOINT, { forkId: this.revision.forkId });
|
|
326
336
|
}
|
|
327
337
|
|
|
328
|
-
public async
|
|
338
|
+
public async commitAllCheckpointsTo(depth: number): Promise<void> {
|
|
329
339
|
assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set');
|
|
330
|
-
await this.instance.call(WorldStateMessageType.COMMIT_ALL_CHECKPOINTS, {
|
|
340
|
+
await this.instance.call(WorldStateMessageType.COMMIT_ALL_CHECKPOINTS, {
|
|
341
|
+
forkId: this.revision.forkId,
|
|
342
|
+
depth,
|
|
343
|
+
});
|
|
331
344
|
}
|
|
332
345
|
|
|
333
|
-
public async
|
|
346
|
+
public async revertAllCheckpointsTo(depth: number): Promise<void> {
|
|
334
347
|
assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set');
|
|
335
|
-
await this.instance.call(WorldStateMessageType.REVERT_ALL_CHECKPOINTS, {
|
|
348
|
+
await this.instance.call(WorldStateMessageType.REVERT_ALL_CHECKPOINTS, {
|
|
349
|
+
forkId: this.revision.forkId,
|
|
350
|
+
depth,
|
|
351
|
+
});
|
|
336
352
|
}
|
|
337
353
|
}
|
|
338
354
|
|
package/src/native/message.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
3
|
import type { Tuple } from '@aztec/foundation/serialize';
|
|
4
|
+
import type { BlockHash } from '@aztec/stdlib/block';
|
|
4
5
|
import { AppendOnlyTreeSnapshot, MerkleTreeId } from '@aztec/stdlib/trees';
|
|
5
6
|
import type { StateReference } from '@aztec/stdlib/tx';
|
|
6
7
|
import type { UInt32 } from '@aztec/stdlib/types';
|
|
@@ -283,6 +284,16 @@ interface WithForkId {
|
|
|
283
284
|
forkId: number;
|
|
284
285
|
}
|
|
285
286
|
|
|
287
|
+
interface CreateCheckpointResponse {
|
|
288
|
+
depth: number;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** Request to commit/revert all checkpoints down to a target depth. The resulting depth after the operation equals the given depth. */
|
|
292
|
+
interface CheckpointDepthRequest extends WithForkId {
|
|
293
|
+
/** The target depth after the operation. All checkpoints above this depth are committed/reverted. */
|
|
294
|
+
depth: number;
|
|
295
|
+
}
|
|
296
|
+
|
|
286
297
|
interface WithWorldStateRevision {
|
|
287
298
|
revision: WorldStateRevision;
|
|
288
299
|
}
|
|
@@ -412,7 +423,7 @@ interface UpdateArchiveRequest extends WithForkId {
|
|
|
412
423
|
interface SyncBlockRequest extends WithCanonicalForkId {
|
|
413
424
|
blockNumber: BlockNumber;
|
|
414
425
|
blockStateRef: BlockStateReference;
|
|
415
|
-
blockHeaderHash:
|
|
426
|
+
blockHeaderHash: BlockHash;
|
|
416
427
|
paddedNoteHashes: readonly SerializedLeafValue[];
|
|
417
428
|
paddedL1ToL2Messages: readonly SerializedLeafValue[];
|
|
418
429
|
paddedNullifiers: readonly SerializedLeafValue[];
|
|
@@ -486,8 +497,8 @@ export type WorldStateRequest = {
|
|
|
486
497
|
[WorldStateMessageType.CREATE_CHECKPOINT]: WithForkId;
|
|
487
498
|
[WorldStateMessageType.COMMIT_CHECKPOINT]: WithForkId;
|
|
488
499
|
[WorldStateMessageType.REVERT_CHECKPOINT]: WithForkId;
|
|
489
|
-
[WorldStateMessageType.COMMIT_ALL_CHECKPOINTS]:
|
|
490
|
-
[WorldStateMessageType.REVERT_ALL_CHECKPOINTS]:
|
|
500
|
+
[WorldStateMessageType.COMMIT_ALL_CHECKPOINTS]: CheckpointDepthRequest;
|
|
501
|
+
[WorldStateMessageType.REVERT_ALL_CHECKPOINTS]: CheckpointDepthRequest;
|
|
491
502
|
|
|
492
503
|
[WorldStateMessageType.COPY_STORES]: CopyStoresRequest;
|
|
493
504
|
|
|
@@ -528,7 +539,7 @@ export type WorldStateResponse = {
|
|
|
528
539
|
|
|
529
540
|
[WorldStateMessageType.GET_STATUS]: WorldStateStatusSummary;
|
|
530
541
|
|
|
531
|
-
[WorldStateMessageType.CREATE_CHECKPOINT]:
|
|
542
|
+
[WorldStateMessageType.CREATE_CHECKPOINT]: CreateCheckpointResponse;
|
|
532
543
|
[WorldStateMessageType.COMMIT_CHECKPOINT]: void;
|
|
533
544
|
[WorldStateMessageType.REVERT_CHECKPOINT]: void;
|
|
534
545
|
[WorldStateMessageType.COMMIT_ALL_CHECKPOINTS]: void;
|
|
@@ -4,9 +4,9 @@ import { fromEntries, padArrayEnd } from '@aztec/foundation/collection';
|
|
|
4
4
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
5
5
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
6
6
|
import { tryRmDir } from '@aztec/foundation/fs';
|
|
7
|
-
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
8
|
-
import type {
|
|
9
|
-
import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
|
|
7
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
8
|
+
import type { L2Block } from '@aztec/stdlib/block';
|
|
9
|
+
import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
|
|
10
10
|
import type {
|
|
11
11
|
IndexedTreeId,
|
|
12
12
|
MerkleTreeReadOperations,
|
|
@@ -52,7 +52,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
52
52
|
protected constructor(
|
|
53
53
|
protected instance: NativeWorldState,
|
|
54
54
|
protected readonly worldStateInstrumentation: WorldStateInstrumentation,
|
|
55
|
-
protected readonly log: Logger
|
|
55
|
+
protected readonly log: Logger,
|
|
56
56
|
private readonly cleanup = () => Promise.resolve(),
|
|
57
57
|
) {}
|
|
58
58
|
|
|
@@ -62,9 +62,10 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
62
62
|
wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
63
63
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
64
64
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
65
|
-
|
|
65
|
+
bindings?: LoggerBindings,
|
|
66
66
|
cleanup = () => Promise.resolve(),
|
|
67
67
|
): Promise<NativeWorldStateService> {
|
|
68
|
+
const log = createLogger('world-state:database', bindings);
|
|
68
69
|
const worldStateDirectory = join(dataDir, WORLD_STATE_DIR);
|
|
69
70
|
// Create a version manager to handle versioning
|
|
70
71
|
const versionManager = new DatabaseVersionManager({
|
|
@@ -72,7 +73,9 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
72
73
|
rollupAddress,
|
|
73
74
|
dataDirectory: worldStateDirectory,
|
|
74
75
|
onOpen: (dir: string) => {
|
|
75
|
-
return Promise.resolve(
|
|
76
|
+
return Promise.resolve(
|
|
77
|
+
new NativeWorldState(dir, wsTreeMapSizes, prefilledPublicData, instrumentation, bindings),
|
|
78
|
+
);
|
|
76
79
|
},
|
|
77
80
|
});
|
|
78
81
|
|
|
@@ -93,8 +96,9 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
93
96
|
cleanupTmpDir = true,
|
|
94
97
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
95
98
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
99
|
+
bindings?: LoggerBindings,
|
|
96
100
|
): Promise<NativeWorldStateService> {
|
|
97
|
-
const log = createLogger('world-state:database');
|
|
101
|
+
const log = createLogger('world-state:database', bindings);
|
|
98
102
|
const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
|
|
99
103
|
const dbMapSizeKb = 10 * 1024 * 1024;
|
|
100
104
|
const worldStateTreeMapSizes: WorldStateTreeMapSizes = {
|
|
@@ -116,7 +120,15 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
116
120
|
}
|
|
117
121
|
};
|
|
118
122
|
|
|
119
|
-
return this.new(
|
|
123
|
+
return this.new(
|
|
124
|
+
rollupAddress,
|
|
125
|
+
dataDir,
|
|
126
|
+
worldStateTreeMapSizes,
|
|
127
|
+
prefilledPublicData,
|
|
128
|
+
instrumentation,
|
|
129
|
+
bindings,
|
|
130
|
+
cleanup,
|
|
131
|
+
);
|
|
120
132
|
}
|
|
121
133
|
|
|
122
134
|
protected async init() {
|
|
@@ -135,7 +147,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
135
147
|
|
|
136
148
|
// the initial header _must_ be the first element in the archive tree
|
|
137
149
|
// if this assertion fails, check that the hashing done in Header in yarn-project matches the initial header hash done in world_state.cpp
|
|
138
|
-
const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [await this.initialHeader.hash()]);
|
|
150
|
+
const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [(await this.initialHeader.hash()).toFr()]);
|
|
139
151
|
const initialHeaderIndex = indices[0];
|
|
140
152
|
assert.strictEqual(initialHeaderIndex, 0n, 'Invalid initial archive state');
|
|
141
153
|
}
|
|
@@ -184,7 +196,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
184
196
|
return this.initialHeader!;
|
|
185
197
|
}
|
|
186
198
|
|
|
187
|
-
public async handleL2BlockAndMessages(l2Block:
|
|
199
|
+
public async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise<WorldStateStatusFull> {
|
|
188
200
|
const isFirstBlock = l2Block.indexWithinCheckpoint === 0;
|
|
189
201
|
if (!isFirstBlock && l1ToL2Messages.length > 0) {
|
|
190
202
|
throw new Error(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ARCHIVE_HEIGHT,
|
|
3
|
-
|
|
3
|
+
DomainSeparator,
|
|
4
4
|
L1_TO_L2_MSG_TREE_HEIGHT,
|
|
5
5
|
MAX_NULLIFIERS_PER_TX,
|
|
6
6
|
MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
NULLIFIER_TREE_HEIGHT,
|
|
9
9
|
PUBLIC_DATA_TREE_HEIGHT,
|
|
10
10
|
} from '@aztec/constants';
|
|
11
|
-
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
11
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
12
12
|
import { NativeWorldState as BaseNativeWorldState, MsgpackChannel } from '@aztec/native';
|
|
13
13
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
14
14
|
import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
@@ -57,7 +57,8 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
57
57
|
private readonly wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
58
58
|
private readonly prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
59
59
|
private readonly instrumentation: WorldStateInstrumentation,
|
|
60
|
-
|
|
60
|
+
bindings?: LoggerBindings,
|
|
61
|
+
private readonly log: Logger = createLogger('world-state:database', bindings),
|
|
61
62
|
) {
|
|
62
63
|
const threads = Math.min(cpus().length, MAX_WORLD_STATE_THREADS);
|
|
63
64
|
log.info(
|
|
@@ -80,7 +81,7 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
80
81
|
[MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
|
|
81
82
|
},
|
|
82
83
|
prefilledPublicDataBufferArray,
|
|
83
|
-
|
|
84
|
+
DomainSeparator.BLOCK_HEADER_HASH,
|
|
84
85
|
{
|
|
85
86
|
[MerkleTreeId.NULLIFIER_TREE]: wsTreeMapSizes.nullifierTreeMapSizeKb,
|
|
86
87
|
[MerkleTreeId.NOTE_HASH_TREE]: wsTreeMapSizes.noteHashTreeMapSizeKb,
|
|
@@ -105,6 +106,7 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
105
106
|
this.wsTreeMapSizes,
|
|
106
107
|
this.prefilledPublicData,
|
|
107
108
|
this.instrumentation,
|
|
109
|
+
this.log.getBindings(),
|
|
108
110
|
this.log,
|
|
109
111
|
);
|
|
110
112
|
}
|
|
@@ -1,18 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
type ConfigMappingsType,
|
|
3
|
-
booleanConfigHelper,
|
|
4
|
-
getConfigFromMappings,
|
|
5
|
-
numberConfigHelper,
|
|
6
|
-
} from '@aztec/foundation/config';
|
|
1
|
+
import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
|
|
7
2
|
|
|
8
3
|
/** World State synchronizer configuration values. */
|
|
9
4
|
export interface WorldStateConfig {
|
|
10
5
|
/** The frequency in which to check. */
|
|
11
6
|
worldStateBlockCheckIntervalMS: number;
|
|
12
7
|
|
|
13
|
-
/** Whether to follow only the proven chain. */
|
|
14
|
-
worldStateProvenBlocksOnly: boolean;
|
|
15
|
-
|
|
16
8
|
/** Size of the batch for each get-blocks request from the synchronizer to the archiver. */
|
|
17
9
|
worldStateBlockRequestBatchSize?: number;
|
|
18
10
|
|
|
@@ -37,8 +29,8 @@ export interface WorldStateConfig {
|
|
|
37
29
|
/** Optional directory for the world state DB, if unspecified will default to the general data directory */
|
|
38
30
|
worldStateDataDirectory?: string;
|
|
39
31
|
|
|
40
|
-
/** The number of historic blocks to maintain */
|
|
41
|
-
|
|
32
|
+
/** The number of historic checkpoints worth of blocks to maintain */
|
|
33
|
+
worldStateCheckpointHistory: number;
|
|
42
34
|
}
|
|
43
35
|
|
|
44
36
|
export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
|
|
@@ -48,11 +40,6 @@ export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
|
|
|
48
40
|
defaultValue: 100,
|
|
49
41
|
description: 'The frequency in which to check.',
|
|
50
42
|
},
|
|
51
|
-
worldStateProvenBlocksOnly: {
|
|
52
|
-
env: 'WS_PROVEN_BLOCKS_ONLY',
|
|
53
|
-
description: 'Whether to follow only the proven chain.',
|
|
54
|
-
...booleanConfigHelper(),
|
|
55
|
-
},
|
|
56
43
|
worldStateBlockRequestBatchSize: {
|
|
57
44
|
env: 'WS_BLOCK_REQUEST_BATCH_SIZE',
|
|
58
45
|
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
@@ -97,9 +84,11 @@ export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
|
|
|
97
84
|
env: 'WS_DATA_DIRECTORY',
|
|
98
85
|
description: 'Optional directory for the world state database',
|
|
99
86
|
},
|
|
100
|
-
|
|
101
|
-
env: '
|
|
102
|
-
description:
|
|
87
|
+
worldStateCheckpointHistory: {
|
|
88
|
+
env: 'WS_NUM_HISTORIC_CHECKPOINTS',
|
|
89
|
+
description:
|
|
90
|
+
'The number of historic checkpoints worth of blocks to maintain. Values less than 1 mean all history is maintained',
|
|
91
|
+
fallback: ['WS_NUM_HISTORIC_BLOCKS'],
|
|
103
92
|
...numberConfigHelper(64),
|
|
104
93
|
},
|
|
105
94
|
};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { LoggerBindings } from '@aztec/foundation/log';
|
|
2
2
|
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
3
|
+
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
3
4
|
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
4
5
|
import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
5
6
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
@@ -22,9 +23,10 @@ export async function createWorldStateSynchronizer(
|
|
|
22
23
|
l2BlockSource: L2BlockSource & L1ToL2MessageSource,
|
|
23
24
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
24
25
|
client: TelemetryClient = getTelemetryClient(),
|
|
26
|
+
bindings?: LoggerBindings,
|
|
25
27
|
) {
|
|
26
28
|
const instrumentation = new WorldStateInstrumentation(client);
|
|
27
|
-
const merkleTrees = await createWorldState(config, prefilledPublicData, instrumentation);
|
|
29
|
+
const merkleTrees = await createWorldState(config, prefilledPublicData, instrumentation, bindings);
|
|
28
30
|
return new ServerWorldStateSynchronizer(merkleTrees, l2BlockSource, config, instrumentation);
|
|
29
31
|
}
|
|
30
32
|
|
|
@@ -42,6 +44,7 @@ export async function createWorldState(
|
|
|
42
44
|
Pick<DataStoreConfig, 'dataDirectory' | 'dataStoreMapSizeKb' | 'l1Contracts'>,
|
|
43
45
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
44
46
|
instrumentation: WorldStateInstrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
47
|
+
bindings?: LoggerBindings,
|
|
45
48
|
) {
|
|
46
49
|
const dataDirectory = config.worldStateDataDirectory ?? config.dataDirectory;
|
|
47
50
|
const dataStoreMapSizeKb = config.worldStateDbMapSizeKb ?? config.dataStoreMapSizeKb;
|
|
@@ -65,11 +68,14 @@ export async function createWorldState(
|
|
|
65
68
|
wsTreeMapSizes,
|
|
66
69
|
prefilledPublicData,
|
|
67
70
|
instrumentation,
|
|
71
|
+
bindings,
|
|
68
72
|
)
|
|
69
73
|
: await NativeWorldStateService.tmp(
|
|
70
74
|
config.l1Contracts.rollupAddress,
|
|
71
75
|
!['true', '1'].includes(process.env.DEBUG_WORLD_STATE!),
|
|
72
76
|
prefilledPublicData,
|
|
77
|
+
instrumentation,
|
|
78
|
+
bindings,
|
|
73
79
|
);
|
|
74
80
|
|
|
75
81
|
return merkleTrees;
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { GENESIS_BLOCK_HEADER_HASH,
|
|
2
|
-
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
1
|
+
import { GENESIS_BLOCK_HEADER_HASH, INITIAL_CHECKPOINT_NUMBER, INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
|
|
2
|
+
import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
|
|
3
3
|
import type { Fr } from '@aztec/foundation/curves/bn254';
|
|
4
4
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
5
5
|
import { promiseWithResolvers } from '@aztec/foundation/promise';
|
|
6
6
|
import { elapsed } from '@aztec/foundation/timer';
|
|
7
7
|
import {
|
|
8
|
+
type BlockHash,
|
|
8
9
|
GENESIS_CHECKPOINT_HEADER_HASH,
|
|
10
|
+
type L2Block,
|
|
9
11
|
type L2BlockId,
|
|
10
|
-
type L2BlockNew,
|
|
11
12
|
type L2BlockSource,
|
|
12
13
|
L2BlockStream,
|
|
13
14
|
type L2BlockStreamEvent,
|
|
@@ -48,7 +49,6 @@ export class ServerWorldStateSynchronizer
|
|
|
48
49
|
private latestBlockNumberAtStart = BlockNumber.ZERO;
|
|
49
50
|
private historyToKeep: number | undefined;
|
|
50
51
|
private currentState: WorldStateRunningState = WorldStateRunningState.IDLE;
|
|
51
|
-
private latestBlockHashQuery: { blockNumber: BlockNumber; hash: string | undefined } | undefined = undefined;
|
|
52
52
|
|
|
53
53
|
private syncPromise = promiseWithResolvers<void>();
|
|
54
54
|
protected blockStream: L2BlockStream | undefined;
|
|
@@ -65,7 +65,7 @@ export class ServerWorldStateSynchronizer
|
|
|
65
65
|
private readonly log: Logger = createLogger('world_state'),
|
|
66
66
|
) {
|
|
67
67
|
this.merkleTreeCommitted = this.merkleTreeDb.getCommitted();
|
|
68
|
-
this.historyToKeep = config.
|
|
68
|
+
this.historyToKeep = config.worldStateCheckpointHistory < 1 ? undefined : config.worldStateCheckpointHistory;
|
|
69
69
|
this.log.info(
|
|
70
70
|
`Created world state synchroniser with block history of ${
|
|
71
71
|
this.historyToKeep === undefined ? 'infinity' : this.historyToKeep
|
|
@@ -102,11 +102,7 @@ export class ServerWorldStateSynchronizer
|
|
|
102
102
|
}
|
|
103
103
|
|
|
104
104
|
// Get the current latest block number
|
|
105
|
-
this.latestBlockNumberAtStart = BlockNumber(
|
|
106
|
-
await (this.config.worldStateProvenBlocksOnly
|
|
107
|
-
? this.l2BlockSource.getProvenBlockNumber()
|
|
108
|
-
: this.l2BlockSource.getBlockNumber()),
|
|
109
|
-
);
|
|
105
|
+
this.latestBlockNumberAtStart = BlockNumber(await this.l2BlockSource.getBlockNumber());
|
|
110
106
|
|
|
111
107
|
const blockToDownloadFrom = (await this.getLatestBlockNumber()) + 1;
|
|
112
108
|
|
|
@@ -130,7 +126,6 @@ export class ServerWorldStateSynchronizer
|
|
|
130
126
|
protected createBlockStream(): L2BlockStream {
|
|
131
127
|
const logger = createLogger('world-state:block_stream');
|
|
132
128
|
return new L2BlockStream(this.l2BlockSource, this, this, logger, {
|
|
133
|
-
proven: this.config.worldStateProvenBlocksOnly,
|
|
134
129
|
pollIntervalMS: this.config.worldStateBlockCheckIntervalMS,
|
|
135
130
|
batchSize: this.config.worldStateBlockRequestBatchSize,
|
|
136
131
|
ignoreCheckpoints: true,
|
|
@@ -183,13 +178,10 @@ export class ServerWorldStateSynchronizer
|
|
|
183
178
|
/**
|
|
184
179
|
* Forces an immediate sync.
|
|
185
180
|
* @param targetBlockNumber - The target block number that we must sync to. Will download unproven blocks if needed to reach it.
|
|
186
|
-
* @param
|
|
181
|
+
* @param blockHash - If provided, verifies the block at targetBlockNumber matches this hash. On mismatch, triggers a resync (reorg detection).
|
|
187
182
|
* @returns A promise that resolves with the block number the world state was synced to
|
|
188
183
|
*/
|
|
189
|
-
public async syncImmediate(
|
|
190
|
-
targetBlockNumber?: BlockNumber,
|
|
191
|
-
skipThrowIfTargetNotReached?: boolean,
|
|
192
|
-
): Promise<BlockNumber> {
|
|
184
|
+
public async syncImmediate(targetBlockNumber?: BlockNumber, blockHash?: BlockHash): Promise<BlockNumber> {
|
|
193
185
|
if (this.currentState !== WorldStateRunningState.RUNNING) {
|
|
194
186
|
throw new Error(`World State is not running. Unable to perform sync.`);
|
|
195
187
|
}
|
|
@@ -201,7 +193,19 @@ export class ServerWorldStateSynchronizer
|
|
|
201
193
|
// If we have been given a block number to sync to and we have reached that number then return
|
|
202
194
|
const currentBlockNumber = await this.getLatestBlockNumber();
|
|
203
195
|
if (targetBlockNumber !== undefined && targetBlockNumber <= currentBlockNumber) {
|
|
204
|
-
|
|
196
|
+
if (blockHash === undefined) {
|
|
197
|
+
return currentBlockNumber;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// If a block hash was provided, verify we're on the expected fork
|
|
201
|
+
const currentHash = await this.getL2BlockHash(targetBlockNumber);
|
|
202
|
+
if (currentHash === blockHash.toString()) {
|
|
203
|
+
return currentBlockNumber;
|
|
204
|
+
}
|
|
205
|
+
// Hash mismatch: a reorg may have occurred, fall through to trigger sync
|
|
206
|
+
this.log.debug(
|
|
207
|
+
`World state block hash mismatch at ${targetBlockNumber} (expected ${blockHash}, got ${currentHash}). Triggering resync.`,
|
|
208
|
+
);
|
|
205
209
|
}
|
|
206
210
|
this.log.debug(`World State at ${currentBlockNumber} told to sync to ${targetBlockNumber ?? 'latest'}`);
|
|
207
211
|
|
|
@@ -219,7 +223,7 @@ export class ServerWorldStateSynchronizer
|
|
|
219
223
|
|
|
220
224
|
// If we have been given a block number to sync to and we have not reached that number then fail
|
|
221
225
|
const updatedBlockNumber = await this.getLatestBlockNumber();
|
|
222
|
-
if (
|
|
226
|
+
if (targetBlockNumber !== undefined && targetBlockNumber > updatedBlockNumber) {
|
|
223
227
|
throw new WorldStateSynchronizerError(
|
|
224
228
|
`Unable to sync to block number ${targetBlockNumber} (last synced is ${updatedBlockNumber})`,
|
|
225
229
|
{
|
|
@@ -233,6 +237,24 @@ export class ServerWorldStateSynchronizer
|
|
|
233
237
|
);
|
|
234
238
|
}
|
|
235
239
|
|
|
240
|
+
// If a block hash was provided, verify we're on the expected fork after syncing, throw otherwise
|
|
241
|
+
if (blockHash !== undefined && targetBlockNumber !== undefined) {
|
|
242
|
+
const updatedHash = await this.getL2BlockHash(targetBlockNumber);
|
|
243
|
+
if (updatedHash !== blockHash.toString()) {
|
|
244
|
+
throw new WorldStateSynchronizerError(
|
|
245
|
+
`Block hash mismatch at block ${targetBlockNumber} (expected ${blockHash} but got ${updatedHash})`,
|
|
246
|
+
{
|
|
247
|
+
cause: {
|
|
248
|
+
reason: 'block_hash_mismatch',
|
|
249
|
+
targetBlockNumber,
|
|
250
|
+
expectedHash: blockHash.toString(),
|
|
251
|
+
actualHash: updatedHash,
|
|
252
|
+
},
|
|
253
|
+
},
|
|
254
|
+
);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
236
258
|
return updatedBlockNumber;
|
|
237
259
|
}
|
|
238
260
|
|
|
@@ -241,21 +263,24 @@ export class ServerWorldStateSynchronizer
|
|
|
241
263
|
if (number === BlockNumber.ZERO) {
|
|
242
264
|
return (await this.merkleTreeCommitted.getInitialHeader().hash()).toString();
|
|
243
265
|
}
|
|
244
|
-
|
|
245
|
-
this.latestBlockHashQuery = {
|
|
246
|
-
hash: await this.merkleTreeCommitted
|
|
247
|
-
.getLeafValue(MerkleTreeId.ARCHIVE, BigInt(number))
|
|
248
|
-
.then(leaf => leaf?.toString()),
|
|
249
|
-
blockNumber: number,
|
|
250
|
-
};
|
|
251
|
-
}
|
|
252
|
-
return this.latestBlockHashQuery.hash;
|
|
266
|
+
return this.merkleTreeCommitted.getLeafValue(MerkleTreeId.ARCHIVE, BigInt(number)).then(leaf => leaf?.toString());
|
|
253
267
|
}
|
|
254
268
|
|
|
255
269
|
/** Returns the latest L2 block number for each tip of the chain (latest, proven, finalized). */
|
|
256
270
|
public async getL2Tips(): Promise<L2Tips> {
|
|
257
271
|
const status = await this.merkleTreeDb.getStatusSummary();
|
|
258
|
-
const
|
|
272
|
+
const unfinalizedBlockHashPromise = this.getL2BlockHash(status.unfinalizedBlockNumber);
|
|
273
|
+
const finalizedBlockHashPromise = this.getL2BlockHash(status.finalizedBlockNumber);
|
|
274
|
+
|
|
275
|
+
const provenBlockNumber = this.provenBlockNumber ?? status.finalizedBlockNumber;
|
|
276
|
+
const provenBlockHashPromise =
|
|
277
|
+
this.provenBlockNumber === undefined ? finalizedBlockHashPromise : this.getL2BlockHash(this.provenBlockNumber);
|
|
278
|
+
|
|
279
|
+
const [unfinalizedBlockHash, finalizedBlockHash, provenBlockHash] = await Promise.all([
|
|
280
|
+
unfinalizedBlockHashPromise,
|
|
281
|
+
finalizedBlockHashPromise,
|
|
282
|
+
provenBlockHashPromise,
|
|
283
|
+
]);
|
|
259
284
|
const latestBlockId: L2BlockId = { number: status.unfinalizedBlockNumber, hash: unfinalizedBlockHash! };
|
|
260
285
|
|
|
261
286
|
// World state doesn't track checkpointed blocks or checkpoints themselves.
|
|
@@ -266,16 +291,16 @@ export class ServerWorldStateSynchronizer
|
|
|
266
291
|
proposed: latestBlockId,
|
|
267
292
|
checkpointed: {
|
|
268
293
|
block: { number: INITIAL_L2_BLOCK_NUM, hash: GENESIS_BLOCK_HEADER_HASH.toString() },
|
|
269
|
-
checkpoint: { number:
|
|
294
|
+
checkpoint: { number: INITIAL_CHECKPOINT_NUMBER, hash: genesisCheckpointHeaderHash },
|
|
270
295
|
},
|
|
271
296
|
finalized: {
|
|
272
|
-
block: { number: status.finalizedBlockNumber, hash: '' },
|
|
273
|
-
checkpoint: { number:
|
|
297
|
+
block: { number: status.finalizedBlockNumber, hash: finalizedBlockHash ?? '' },
|
|
298
|
+
checkpoint: { number: INITIAL_CHECKPOINT_NUMBER, hash: genesisCheckpointHeaderHash },
|
|
274
299
|
},
|
|
275
300
|
proven: {
|
|
276
|
-
block: { number:
|
|
277
|
-
checkpoint: { number:
|
|
278
|
-
},
|
|
301
|
+
block: { number: provenBlockNumber, hash: provenBlockHash ?? '' },
|
|
302
|
+
checkpoint: { number: INITIAL_CHECKPOINT_NUMBER, hash: genesisCheckpointHeaderHash },
|
|
303
|
+
},
|
|
279
304
|
};
|
|
280
305
|
}
|
|
281
306
|
|
|
@@ -302,8 +327,8 @@ export class ServerWorldStateSynchronizer
|
|
|
302
327
|
* @param l2Blocks - The L2 blocks to handle.
|
|
303
328
|
* @returns Whether the block handled was produced by this same node.
|
|
304
329
|
*/
|
|
305
|
-
private async handleL2Blocks(l2Blocks:
|
|
306
|
-
this.log.
|
|
330
|
+
private async handleL2Blocks(l2Blocks: L2Block[]) {
|
|
331
|
+
this.log.debug(`Handling L2 blocks ${l2Blocks[0].number} to ${l2Blocks.at(-1)!.number}`);
|
|
307
332
|
|
|
308
333
|
// Fetch the L1->L2 messages for the first block in a checkpoint.
|
|
309
334
|
const messagesForBlocks = new Map<BlockNumber, Fr[]>();
|
|
@@ -343,11 +368,13 @@ export class ServerWorldStateSynchronizer
|
|
|
343
368
|
* @param l1ToL2Messages - The L1 to L2 messages for the block.
|
|
344
369
|
* @returns Whether the block handled was produced by this same node.
|
|
345
370
|
*/
|
|
346
|
-
private async handleL2Block(l2Block:
|
|
347
|
-
this.log.
|
|
371
|
+
private async handleL2Block(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise<WorldStateStatusFull> {
|
|
372
|
+
this.log.debug(`Pushing L2 block ${l2Block.number} to merkle tree db `, {
|
|
348
373
|
blockNumber: l2Block.number,
|
|
349
374
|
blockHash: await l2Block.hash().then(h => h.toString()),
|
|
350
375
|
l1ToL2Messages: l1ToL2Messages.map(msg => msg.toString()),
|
|
376
|
+
blockHeader: l2Block.header.toInspect(),
|
|
377
|
+
blockStats: l2Block.getStats(),
|
|
351
378
|
});
|
|
352
379
|
const result = await this.merkleTreeDb.handleL2BlockAndMessages(l2Block, l1ToL2Messages);
|
|
353
380
|
|
|
@@ -365,12 +392,37 @@ export class ServerWorldStateSynchronizer
|
|
|
365
392
|
if (this.historyToKeep === undefined) {
|
|
366
393
|
return;
|
|
367
394
|
}
|
|
368
|
-
|
|
369
|
-
|
|
395
|
+
// Get the checkpointed block for the finalized block number
|
|
396
|
+
const finalisedCheckpoint = await this.l2BlockSource.getCheckpointedBlock(summary.finalizedBlockNumber);
|
|
397
|
+
if (finalisedCheckpoint === undefined) {
|
|
398
|
+
this.log.warn(
|
|
399
|
+
`Failed to retrieve checkpointed block for finalized block number: ${summary.finalizedBlockNumber}`,
|
|
400
|
+
);
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
// Compute the required historic checkpoint number
|
|
404
|
+
const newHistoricCheckpointNumber = finalisedCheckpoint.checkpointNumber - this.historyToKeep + 1;
|
|
405
|
+
if (newHistoricCheckpointNumber <= 1) {
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
408
|
+
// Retrieve the historic checkpoint
|
|
409
|
+
const historicCheckpoints = await this.l2BlockSource.getCheckpoints(
|
|
410
|
+
CheckpointNumber(newHistoricCheckpointNumber),
|
|
411
|
+
1,
|
|
412
|
+
);
|
|
413
|
+
if (historicCheckpoints.length === 0 || historicCheckpoints[0] === undefined) {
|
|
414
|
+
this.log.warn(`Failed to retrieve checkpoint number ${newHistoricCheckpointNumber} from Archiver`);
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
const historicCheckpoint = historicCheckpoints[0];
|
|
418
|
+
if (historicCheckpoint.checkpoint.blocks.length === 0 || historicCheckpoint.checkpoint.blocks[0] === undefined) {
|
|
419
|
+
this.log.warn(`Retrieved checkpoint number ${newHistoricCheckpointNumber} has no blocks!`);
|
|
370
420
|
return;
|
|
371
421
|
}
|
|
372
|
-
|
|
373
|
-
const
|
|
422
|
+
// Find the block at the start of the checkpoint and remove blocks up to this one
|
|
423
|
+
const newHistoricBlock = historicCheckpoint.checkpoint.blocks[0];
|
|
424
|
+
this.log.verbose(`Pruning historic blocks to ${newHistoricBlock.number}`);
|
|
425
|
+
const status = await this.merkleTreeDb.removeHistoricalBlocks(BlockNumber(newHistoricBlock.number));
|
|
374
426
|
this.log.debug(`World state summary `, status.summary);
|
|
375
427
|
}
|
|
376
428
|
|
|
@@ -381,9 +433,8 @@ export class ServerWorldStateSynchronizer
|
|
|
381
433
|
}
|
|
382
434
|
|
|
383
435
|
private async handleChainPruned(blockNumber: BlockNumber) {
|
|
384
|
-
this.log.
|
|
436
|
+
this.log.info(`Chain pruned to block ${blockNumber}`);
|
|
385
437
|
const status = await this.merkleTreeDb.unwindBlocks(blockNumber);
|
|
386
|
-
this.latestBlockHashQuery = undefined;
|
|
387
438
|
this.provenBlockNumber = undefined;
|
|
388
439
|
this.instrumentation.updateWorldStateMetrics(status);
|
|
389
440
|
}
|