@aztec/world-state 0.76.4 → 0.77.0-testnet-ignition.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/index.js +0 -1
- package/dest/instrumentation/instrumentation.d.ts +2 -0
- package/dest/instrumentation/instrumentation.d.ts.map +1 -1
- package/dest/instrumentation/instrumentation.js +50 -22
- package/dest/native/fork_checkpoint.d.ts +10 -0
- package/dest/native/fork_checkpoint.d.ts.map +1 -0
- package/dest/native/fork_checkpoint.js +26 -0
- package/dest/native/index.d.ts +1 -0
- package/dest/native/index.d.ts.map +1 -1
- package/dest/native/index.js +1 -1
- package/dest/native/merkle_trees_facade.d.ts +6 -4
- package/dest/native/merkle_trees_facade.d.ts.map +1 -1
- package/dest/native/merkle_trees_facade.js +80 -75
- package/dest/native/message.d.ts +5 -3
- package/dest/native/message.d.ts.map +1 -1
- package/dest/native/message.js +33 -19
- package/dest/native/native_world_state.d.ts +9 -5
- package/dest/native/native_world_state.d.ts.map +1 -1
- package/dest/native/native_world_state.js +126 -74
- package/dest/native/native_world_state_instance.d.ts +3 -2
- package/dest/native/native_world_state_instance.d.ts.map +1 -1
- package/dest/native/native_world_state_instance.js +43 -40
- package/dest/native/world_state_ops_queue.js +18 -29
- package/dest/native/world_state_version.d.ts +1 -1
- package/dest/native/world_state_version.d.ts.map +1 -1
- package/dest/native/world_state_version.js +8 -9
- package/dest/synchronizer/config.js +12 -14
- package/dest/synchronizer/factory.d.ts +7 -5
- package/dest/synchronizer/factory.d.ts.map +1 -1
- package/dest/synchronizer/factory.js +8 -8
- package/dest/synchronizer/index.js +0 -1
- package/dest/synchronizer/server_world_state_synchronizer.d.ts +7 -4
- package/dest/synchronizer/server_world_state_synchronizer.d.ts.map +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.js +86 -65
- package/dest/test/index.js +0 -1
- package/dest/test/utils.d.ts +4 -3
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +21 -17
- package/dest/testing.d.ts +10 -0
- package/dest/testing.d.ts.map +1 -0
- package/dest/testing.js +37 -0
- package/dest/world-state-db/index.d.ts +1 -1
- package/dest/world-state-db/index.d.ts.map +1 -1
- package/dest/world-state-db/index.js +0 -1
- package/dest/world-state-db/merkle_tree_db.d.ts +6 -5
- package/dest/world-state-db/merkle_tree_db.d.ts.map +1 -1
- package/dest/world-state-db/merkle_tree_db.js +2 -4
- package/package.json +14 -12
- package/src/instrumentation/instrumentation.ts +29 -3
- package/src/native/fork_checkpoint.ts +30 -0
- package/src/native/index.ts +1 -0
- package/src/native/merkle_trees_facade.ts +15 -18
- package/src/native/message.ts +5 -3
- package/src/native/native_world_state.ts +82 -69
- package/src/native/native_world_state_instance.ts +7 -3
- package/src/native/world_state_version.ts +1 -1
- package/src/synchronizer/factory.ts +11 -5
- package/src/synchronizer/server_world_state_synchronizer.ts +43 -26
- package/src/test/utils.ts +6 -10
- package/src/testing.ts +60 -0
- package/src/world-state-db/index.ts +1 -1
- package/src/world-state-db/merkle_tree_db.ts +7 -5
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import { MerkleTreeId } from '@aztec/circuit-types';
|
|
2
1
|
import { createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
3
3
|
import {
|
|
4
4
|
Attributes,
|
|
5
5
|
type Gauge,
|
|
6
6
|
type Histogram,
|
|
7
7
|
Metrics,
|
|
8
8
|
type TelemetryClient,
|
|
9
|
+
type UpDownCounter,
|
|
9
10
|
ValueType,
|
|
10
11
|
} from '@aztec/telemetry-client';
|
|
11
12
|
|
|
@@ -19,6 +20,15 @@ import {
|
|
|
19
20
|
|
|
20
21
|
type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks' | 'block_indices';
|
|
21
22
|
|
|
23
|
+
const durationTrackDenylist = new Set<WorldStateMessageType>([
|
|
24
|
+
WorldStateMessageType.GET_INITIAL_STATE_REFERENCE,
|
|
25
|
+
WorldStateMessageType.CLOSE,
|
|
26
|
+
|
|
27
|
+
// these aren't used anymore, should be removed from the API
|
|
28
|
+
WorldStateMessageType.COMMIT,
|
|
29
|
+
WorldStateMessageType.ROLLBACK,
|
|
30
|
+
]);
|
|
31
|
+
|
|
22
32
|
export class WorldStateInstrumentation {
|
|
23
33
|
private dbMapSize: Gauge;
|
|
24
34
|
private treeSize: Gauge;
|
|
@@ -28,6 +38,7 @@ export class WorldStateInstrumentation {
|
|
|
28
38
|
private dbNumItems: Gauge;
|
|
29
39
|
private dbUsedSize: Gauge;
|
|
30
40
|
private requestHistogram: Histogram;
|
|
41
|
+
private criticalErrors: UpDownCounter;
|
|
31
42
|
|
|
32
43
|
constructor(public readonly telemetry: TelemetryClient, private log = createLogger('world-state:instrumentation')) {
|
|
33
44
|
const meter = telemetry.getMeter('World State');
|
|
@@ -71,6 +82,11 @@ export class WorldStateInstrumentation {
|
|
|
71
82
|
unit: 'us',
|
|
72
83
|
valueType: ValueType.INT,
|
|
73
84
|
});
|
|
85
|
+
|
|
86
|
+
this.criticalErrors = meter.createUpDownCounter(Metrics.WORLD_STATE_CRITICAL_ERROR_COUNT, {
|
|
87
|
+
description: 'The number of critical errors in the world state',
|
|
88
|
+
valueType: ValueType.INT,
|
|
89
|
+
});
|
|
74
90
|
}
|
|
75
91
|
|
|
76
92
|
private updateTreeStats(treeDbStats: TreeDBStats, treeMeta: TreeMeta, tree: MerkleTreeId) {
|
|
@@ -141,8 +157,18 @@ export class WorldStateInstrumentation {
|
|
|
141
157
|
}
|
|
142
158
|
|
|
143
159
|
public recordRoundTrip(timeUs: number, request: WorldStateMessageType) {
|
|
144
|
-
|
|
145
|
-
|
|
160
|
+
if (!durationTrackDenylist.has(request)) {
|
|
161
|
+
this.requestHistogram.record(Math.ceil(timeUs), {
|
|
162
|
+
[Attributes.WORLD_STATE_REQUEST_TYPE]: WorldStateMessageType[request],
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
public incCriticalErrors(
|
|
168
|
+
errorType: 'synch_pending_block' | 'finalize_block' | 'prune_pending_block' | 'prune_historical_block',
|
|
169
|
+
) {
|
|
170
|
+
this.criticalErrors.add(1, {
|
|
171
|
+
[Attributes.ERROR_TYPE]: errorType,
|
|
146
172
|
});
|
|
147
173
|
}
|
|
148
174
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { MerkleTreeCheckpointOperations } from '@aztec/stdlib/interfaces/server';
|
|
2
|
+
|
|
3
|
+
export class ForkCheckpoint {
|
|
4
|
+
private completed = false;
|
|
5
|
+
|
|
6
|
+
private constructor(private readonly fork: MerkleTreeCheckpointOperations) {}
|
|
7
|
+
|
|
8
|
+
static async new(fork: MerkleTreeCheckpointOperations): Promise<ForkCheckpoint> {
|
|
9
|
+
await fork.createCheckpoint();
|
|
10
|
+
return new ForkCheckpoint(fork);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
async commit(): Promise<void> {
|
|
14
|
+
if (this.completed) {
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
await this.fork.commitCheckpoint();
|
|
19
|
+
this.completed = true;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async revert(): Promise<void> {
|
|
23
|
+
if (this.completed) {
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
await this.fork.revertCheckpoint();
|
|
28
|
+
this.completed = true;
|
|
29
|
+
}
|
|
30
|
+
}
|
package/src/native/index.ts
CHANGED
|
@@ -1,26 +1,23 @@
|
|
|
1
|
+
import { Fr } from '@aztec/foundation/fields';
|
|
2
|
+
import { serializeToBuffer } from '@aztec/foundation/serialize';
|
|
3
|
+
import { type IndexedTreeLeafPreimage, SiblingPath } from '@aztec/foundation/trees';
|
|
4
|
+
import type {
|
|
5
|
+
BatchInsertionResult,
|
|
6
|
+
IndexedTreeId,
|
|
7
|
+
MerkleTreeLeafType,
|
|
8
|
+
MerkleTreeReadOperations,
|
|
9
|
+
MerkleTreeWriteOperations,
|
|
10
|
+
SequentialInsertionResult,
|
|
11
|
+
TreeInfo,
|
|
12
|
+
} from '@aztec/stdlib/interfaces/server';
|
|
1
13
|
import {
|
|
2
|
-
type BatchInsertionResult,
|
|
3
|
-
type IndexedTreeId,
|
|
4
14
|
MerkleTreeId,
|
|
5
|
-
type MerkleTreeLeafType,
|
|
6
|
-
type MerkleTreeReadOperations,
|
|
7
|
-
type MerkleTreeWriteOperations,
|
|
8
|
-
type SequentialInsertionResult,
|
|
9
|
-
SiblingPath,
|
|
10
|
-
type TreeInfo,
|
|
11
|
-
} from '@aztec/circuit-types';
|
|
12
|
-
import {
|
|
13
|
-
type BlockHeader,
|
|
14
|
-
Fr,
|
|
15
15
|
NullifierLeaf,
|
|
16
16
|
NullifierLeafPreimage,
|
|
17
|
-
PartialStateReference,
|
|
18
17
|
PublicDataTreeLeaf,
|
|
19
18
|
PublicDataTreeLeafPreimage,
|
|
20
|
-
|
|
21
|
-
} from '@aztec/
|
|
22
|
-
import { serializeToBuffer } from '@aztec/foundation/serialize';
|
|
23
|
-
import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees';
|
|
19
|
+
} from '@aztec/stdlib/trees';
|
|
20
|
+
import { type BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
|
|
24
21
|
|
|
25
22
|
import assert from 'assert';
|
|
26
23
|
|
|
@@ -32,7 +29,7 @@ import {
|
|
|
32
29
|
blockStateReference,
|
|
33
30
|
treeStateReferenceToSnapshot,
|
|
34
31
|
} from './message.js';
|
|
35
|
-
import {
|
|
32
|
+
import type { NativeWorldStateInstance } from './native_world_state_instance.js';
|
|
36
33
|
|
|
37
34
|
export class MerkleTreesFacade implements MerkleTreeReadOperations {
|
|
38
35
|
constructor(
|
package/src/native/message.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
1
|
+
import { Fr } from '@aztec/foundation/fields';
|
|
2
|
+
import type { Tuple } from '@aztec/foundation/serialize';
|
|
3
|
+
import { AppendOnlyTreeSnapshot, MerkleTreeId } from '@aztec/stdlib/trees';
|
|
4
|
+
import type { StateReference } from '@aztec/stdlib/tx';
|
|
5
|
+
import type { UInt32 } from '@aztec/stdlib/types';
|
|
4
6
|
|
|
5
7
|
export enum WorldStateMessageType {
|
|
6
8
|
GET_TREE_INFO = 100,
|
|
@@ -1,25 +1,16 @@
|
|
|
1
|
-
import {
|
|
2
|
-
type IndexedTreeId,
|
|
3
|
-
type L2Block,
|
|
4
|
-
MerkleTreeId,
|
|
5
|
-
type MerkleTreeReadOperations,
|
|
6
|
-
type MerkleTreeWriteOperations,
|
|
7
|
-
} from '@aztec/circuit-types';
|
|
8
|
-
import {
|
|
9
|
-
BlockHeader,
|
|
10
|
-
EthAddress,
|
|
11
|
-
Fr,
|
|
12
|
-
MAX_NOTE_HASHES_PER_TX,
|
|
13
|
-
MAX_NULLIFIERS_PER_TX,
|
|
14
|
-
NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
|
|
15
|
-
NullifierLeaf,
|
|
16
|
-
type NullifierLeafPreimage,
|
|
17
|
-
PartialStateReference,
|
|
18
|
-
PublicDataTreeLeaf,
|
|
19
|
-
StateReference,
|
|
20
|
-
} from '@aztec/circuits.js';
|
|
1
|
+
import { MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
|
|
21
2
|
import { padArrayEnd } from '@aztec/foundation/collection';
|
|
3
|
+
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
4
|
+
import { Fr } from '@aztec/foundation/fields';
|
|
22
5
|
import { createLogger } from '@aztec/foundation/log';
|
|
6
|
+
import type { L2Block } from '@aztec/stdlib/block';
|
|
7
|
+
import type {
|
|
8
|
+
IndexedTreeId,
|
|
9
|
+
MerkleTreeReadOperations,
|
|
10
|
+
MerkleTreeWriteOperations,
|
|
11
|
+
} from '@aztec/stdlib/interfaces/server';
|
|
12
|
+
import { MerkleTreeId, NullifierLeaf, type NullifierLeafPreimage, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
13
|
+
import { BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
|
|
23
14
|
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
24
15
|
|
|
25
16
|
import assert from 'assert/strict';
|
|
@@ -28,7 +19,7 @@ import { tmpdir } from 'os';
|
|
|
28
19
|
import { join } from 'path';
|
|
29
20
|
|
|
30
21
|
import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
31
|
-
import {
|
|
22
|
+
import type { MerkleTreeAdminDatabase as MerkleTreeDatabase } from '../world-state-db/merkle_tree_db.js';
|
|
32
23
|
import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkle_trees_facade.js';
|
|
33
24
|
import {
|
|
34
25
|
WorldStateMessageType,
|
|
@@ -68,6 +59,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
68
59
|
rollupAddress: EthAddress,
|
|
69
60
|
dataDir: string,
|
|
70
61
|
dbMapSizeKb: number,
|
|
62
|
+
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
71
63
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
72
64
|
log = createLogger('world-state:database'),
|
|
73
65
|
cleanup = () => Promise.resolve(),
|
|
@@ -78,13 +70,13 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
78
70
|
|
|
79
71
|
if (!storedWorldStateVersion) {
|
|
80
72
|
log.warn('No world state version found, deleting world state directory');
|
|
81
|
-
await rm(worldStateDirectory, { recursive: true, force: true });
|
|
73
|
+
await rm(worldStateDirectory, { recursive: true, force: true, maxRetries: 3 });
|
|
82
74
|
} else if (!rollupAddress.equals(storedWorldStateVersion.rollupAddress)) {
|
|
83
75
|
log.warn('Rollup address changed, deleting world state directory');
|
|
84
|
-
await rm(worldStateDirectory, { recursive: true, force: true });
|
|
76
|
+
await rm(worldStateDirectory, { recursive: true, force: true, maxRetries: 3 });
|
|
85
77
|
} else if (storedWorldStateVersion.version != WORLD_STATE_DB_VERSION) {
|
|
86
78
|
log.warn('World state version change detected, deleting world state directory');
|
|
87
|
-
await rm(worldStateDirectory, { recursive: true, force: true });
|
|
79
|
+
await rm(worldStateDirectory, { recursive: true, force: true, maxRetries: 3 });
|
|
88
80
|
}
|
|
89
81
|
|
|
90
82
|
const newWorldStateVersion = new WorldStateVersion(WORLD_STATE_DB_VERSION, rollupAddress);
|
|
@@ -92,7 +84,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
92
84
|
await mkdir(worldStateDirectory, { recursive: true });
|
|
93
85
|
await newWorldStateVersion.writeVersionFile(versionFile);
|
|
94
86
|
|
|
95
|
-
const instance = new NativeWorldState(worldStateDirectory, dbMapSizeKb, instrumentation);
|
|
87
|
+
const instance = new NativeWorldState(worldStateDirectory, dbMapSizeKb, prefilledPublicData, instrumentation);
|
|
96
88
|
const worldState = new this(instance, instrumentation, log, cleanup);
|
|
97
89
|
try {
|
|
98
90
|
await worldState.init();
|
|
@@ -107,6 +99,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
107
99
|
static async tmp(
|
|
108
100
|
rollupAddress = EthAddress.ZERO,
|
|
109
101
|
cleanupTmpDir = true,
|
|
102
|
+
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
110
103
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
111
104
|
): Promise<NativeWorldStateService> {
|
|
112
105
|
const log = createLogger('world-state:database');
|
|
@@ -117,14 +110,14 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
117
110
|
// pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
|
|
118
111
|
const cleanup = async () => {
|
|
119
112
|
if (cleanupTmpDir) {
|
|
120
|
-
await rm(dataDir, { recursive: true, force: true });
|
|
113
|
+
await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
|
|
121
114
|
log.debug(`Deleted temporary world state database: ${dataDir}`);
|
|
122
115
|
} else {
|
|
123
116
|
log.debug(`Leaving temporary world state database: ${dataDir}`);
|
|
124
117
|
}
|
|
125
118
|
};
|
|
126
119
|
|
|
127
|
-
return this.new(rollupAddress, dataDir, dbMapSizeKb, instrumentation, log, cleanup);
|
|
120
|
+
return this.new(rollupAddress, dataDir, dbMapSizeKb, prefilledPublicData, instrumentation, log, cleanup);
|
|
128
121
|
}
|
|
129
122
|
|
|
130
123
|
protected async init() {
|
|
@@ -189,21 +182,26 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
189
182
|
});
|
|
190
183
|
});
|
|
191
184
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
185
|
+
try {
|
|
186
|
+
return await this.instance.call(
|
|
187
|
+
WorldStateMessageType.SYNC_BLOCK,
|
|
188
|
+
{
|
|
189
|
+
blockNumber: l2Block.number,
|
|
190
|
+
blockHeaderHash: await l2Block.header.hash(),
|
|
191
|
+
paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
|
|
192
|
+
paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
|
|
193
|
+
paddedNullifiers: paddedNullifiers.map(serializeLeaf),
|
|
194
|
+
publicDataWrites: publicDataWrites.map(serializeLeaf),
|
|
195
|
+
blockStateRef: blockStateReference(l2Block.header.state),
|
|
196
|
+
canonical: true,
|
|
197
|
+
},
|
|
198
|
+
this.sanitiseAndCacheSummaryFromFull.bind(this),
|
|
199
|
+
this.deleteCachedSummary.bind(this),
|
|
200
|
+
);
|
|
201
|
+
} catch (err) {
|
|
202
|
+
this.worldStateInstrumentation.incCriticalErrors('synch_pending_block');
|
|
203
|
+
throw err;
|
|
204
|
+
}
|
|
207
205
|
}
|
|
208
206
|
|
|
209
207
|
public async close(): Promise<void> {
|
|
@@ -238,15 +236,20 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
238
236
|
* @returns The new WorldStateStatus
|
|
239
237
|
*/
|
|
240
238
|
public async setFinalised(toBlockNumber: bigint) {
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
239
|
+
try {
|
|
240
|
+
await this.instance.call(
|
|
241
|
+
WorldStateMessageType.FINALISE_BLOCKS,
|
|
242
|
+
{
|
|
243
|
+
toBlockNumber,
|
|
244
|
+
canonical: true,
|
|
245
|
+
},
|
|
246
|
+
this.sanitiseAndCacheSummary.bind(this),
|
|
247
|
+
this.deleteCachedSummary.bind(this),
|
|
248
|
+
);
|
|
249
|
+
} catch (err) {
|
|
250
|
+
this.worldStateInstrumentation.incCriticalErrors('finalize_block');
|
|
251
|
+
throw err;
|
|
252
|
+
}
|
|
250
253
|
return this.getStatusSummary();
|
|
251
254
|
}
|
|
252
255
|
|
|
@@ -256,15 +259,20 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
256
259
|
* @returns The new WorldStateStatus
|
|
257
260
|
*/
|
|
258
261
|
public async removeHistoricalBlocks(toBlockNumber: bigint) {
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
262
|
+
try {
|
|
263
|
+
return await this.instance.call(
|
|
264
|
+
WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS,
|
|
265
|
+
{
|
|
266
|
+
toBlockNumber,
|
|
267
|
+
canonical: true,
|
|
268
|
+
},
|
|
269
|
+
this.sanitiseAndCacheSummaryFromFull.bind(this),
|
|
270
|
+
this.deleteCachedSummary.bind(this),
|
|
271
|
+
);
|
|
272
|
+
} catch (err) {
|
|
273
|
+
this.worldStateInstrumentation.incCriticalErrors('prune_historical_block');
|
|
274
|
+
throw err;
|
|
275
|
+
}
|
|
268
276
|
}
|
|
269
277
|
|
|
270
278
|
/**
|
|
@@ -273,15 +281,20 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
273
281
|
* @returns The new WorldStateStatus
|
|
274
282
|
*/
|
|
275
283
|
public async unwindBlocks(toBlockNumber: bigint) {
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
284
|
+
try {
|
|
285
|
+
return await this.instance.call(
|
|
286
|
+
WorldStateMessageType.UNWIND_BLOCKS,
|
|
287
|
+
{
|
|
288
|
+
toBlockNumber,
|
|
289
|
+
canonical: true,
|
|
290
|
+
},
|
|
291
|
+
this.sanitiseAndCacheSummaryFromFull.bind(this),
|
|
292
|
+
this.deleteCachedSummary.bind(this),
|
|
293
|
+
);
|
|
294
|
+
} catch (err) {
|
|
295
|
+
this.worldStateInstrumentation.incCriticalErrors('prune_pending_block');
|
|
296
|
+
throw err;
|
|
297
|
+
}
|
|
285
298
|
}
|
|
286
299
|
|
|
287
300
|
public async getStatusSummary() {
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { MerkleTreeId } from '@aztec/circuit-types';
|
|
2
1
|
import {
|
|
3
2
|
ARCHIVE_HEIGHT,
|
|
4
3
|
GeneratorIndex,
|
|
@@ -8,14 +7,16 @@ import {
|
|
|
8
7
|
NOTE_HASH_TREE_HEIGHT,
|
|
9
8
|
NULLIFIER_TREE_HEIGHT,
|
|
10
9
|
PUBLIC_DATA_TREE_HEIGHT,
|
|
11
|
-
} from '@aztec/
|
|
10
|
+
} from '@aztec/constants';
|
|
12
11
|
import { createLogger } from '@aztec/foundation/log';
|
|
13
12
|
import { NativeWorldState as BaseNativeWorldState, MsgpackChannel } from '@aztec/native';
|
|
13
|
+
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
14
|
+
import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
14
15
|
|
|
15
16
|
import assert from 'assert';
|
|
16
17
|
import { cpus } from 'os';
|
|
17
18
|
|
|
18
|
-
import {
|
|
19
|
+
import type { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
19
20
|
import {
|
|
20
21
|
WorldStateMessageType,
|
|
21
22
|
type WorldStateRequest,
|
|
@@ -51,6 +52,7 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
51
52
|
constructor(
|
|
52
53
|
dataDir: string,
|
|
53
54
|
dbMapSizeKb: number,
|
|
55
|
+
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
54
56
|
private instrumentation: WorldStateInstrumentation,
|
|
55
57
|
private log = createLogger('world-state:database'),
|
|
56
58
|
) {
|
|
@@ -58,6 +60,7 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
58
60
|
log.info(
|
|
59
61
|
`Creating world state data store at directory ${dataDir} with map size ${dbMapSizeKb} KB and ${threads} threads.`,
|
|
60
62
|
);
|
|
63
|
+
const prefilledPublicDataBufferArray = prefilledPublicData.map(d => [d.slot.toBuffer(), d.value.toBuffer()]);
|
|
61
64
|
const ws = new BaseNativeWorldState(
|
|
62
65
|
dataDir,
|
|
63
66
|
{
|
|
@@ -71,6 +74,7 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
71
74
|
[MerkleTreeId.NULLIFIER_TREE]: 2 * MAX_NULLIFIERS_PER_TX,
|
|
72
75
|
[MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
|
|
73
76
|
},
|
|
77
|
+
prefilledPublicDataBufferArray,
|
|
74
78
|
GeneratorIndex.BLOCK_HASH,
|
|
75
79
|
dbMapSizeKb,
|
|
76
80
|
threads,
|
|
@@ -1,25 +1,29 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
1
|
+
import type { DataStoreConfig } from '@aztec/kv-store/config';
|
|
2
|
+
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
3
|
+
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
4
|
+
import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
3
5
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
4
6
|
|
|
5
7
|
import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
6
8
|
import { NativeWorldStateService } from '../native/native_world_state.js';
|
|
7
|
-
import {
|
|
9
|
+
import type { WorldStateConfig } from './config.js';
|
|
8
10
|
import { ServerWorldStateSynchronizer } from './server_world_state_synchronizer.js';
|
|
9
11
|
|
|
10
12
|
export async function createWorldStateSynchronizer(
|
|
11
13
|
config: WorldStateConfig & DataStoreConfig,
|
|
12
14
|
l2BlockSource: L2BlockSource & L1ToL2MessageSource,
|
|
15
|
+
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
13
16
|
client: TelemetryClient = getTelemetryClient(),
|
|
14
17
|
) {
|
|
15
18
|
const instrumentation = new WorldStateInstrumentation(client);
|
|
16
|
-
const merkleTrees = await createWorldState(config, instrumentation);
|
|
19
|
+
const merkleTrees = await createWorldState(config, prefilledPublicData, instrumentation);
|
|
17
20
|
return new ServerWorldStateSynchronizer(merkleTrees, l2BlockSource, config, instrumentation);
|
|
18
21
|
}
|
|
19
22
|
|
|
20
23
|
export async function createWorldState(
|
|
21
24
|
config: WorldStateConfig & DataStoreConfig,
|
|
22
|
-
|
|
25
|
+
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
26
|
+
instrumentation: WorldStateInstrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
23
27
|
) {
|
|
24
28
|
const newConfig = {
|
|
25
29
|
dataDirectory: config.worldStateDataDirectory ?? config.dataDirectory,
|
|
@@ -36,11 +40,13 @@ export async function createWorldState(
|
|
|
36
40
|
config.l1Contracts.rollupAddress,
|
|
37
41
|
newConfig.dataDirectory,
|
|
38
42
|
newConfig.dataStoreMapSizeKB,
|
|
43
|
+
prefilledPublicData,
|
|
39
44
|
instrumentation,
|
|
40
45
|
)
|
|
41
46
|
: await NativeWorldStateService.tmp(
|
|
42
47
|
config.l1Contracts.rollupAddress,
|
|
43
48
|
!['true', '1'].includes(process.env.DEBUG_WORLD_STATE!),
|
|
49
|
+
prefilledPublicData,
|
|
44
50
|
);
|
|
45
51
|
|
|
46
52
|
return merkleTrees;
|
|
@@ -1,34 +1,35 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
type L2Block,
|
|
4
|
-
type L2BlockId,
|
|
5
|
-
type L2BlockSource,
|
|
6
|
-
type L2BlockStream,
|
|
7
|
-
type L2BlockStreamEvent,
|
|
8
|
-
type L2BlockStreamEventHandler,
|
|
9
|
-
type L2BlockStreamLocalDataProvider,
|
|
10
|
-
type L2Tips,
|
|
11
|
-
MerkleTreeId,
|
|
12
|
-
type MerkleTreeReadOperations,
|
|
13
|
-
type MerkleTreeWriteOperations,
|
|
14
|
-
WorldStateRunningState,
|
|
15
|
-
type WorldStateSynchronizer,
|
|
16
|
-
type WorldStateSynchronizerStatus,
|
|
17
|
-
} from '@aztec/circuit-types';
|
|
18
|
-
import { type L2BlockHandledStats } from '@aztec/circuit-types/stats';
|
|
19
|
-
import { MerkleTreeCalculator } from '@aztec/circuits.js';
|
|
20
|
-
import { L1_TO_L2_MSG_SUBTREE_HEIGHT } from '@aztec/circuits.js/constants';
|
|
21
|
-
import { type Fr } from '@aztec/foundation/fields';
|
|
1
|
+
import { L1_TO_L2_MSG_SUBTREE_HEIGHT } from '@aztec/constants';
|
|
2
|
+
import type { Fr } from '@aztec/foundation/fields';
|
|
22
3
|
import { createLogger } from '@aztec/foundation/log';
|
|
23
4
|
import { promiseWithResolvers } from '@aztec/foundation/promise';
|
|
24
5
|
import { elapsed } from '@aztec/foundation/timer';
|
|
6
|
+
import { MerkleTreeCalculator } from '@aztec/foundation/trees';
|
|
25
7
|
import { SHA256Trunc } from '@aztec/merkle-tree';
|
|
8
|
+
import type {
|
|
9
|
+
L2Block,
|
|
10
|
+
L2BlockId,
|
|
11
|
+
L2BlockSource,
|
|
12
|
+
L2BlockStream,
|
|
13
|
+
L2BlockStreamEvent,
|
|
14
|
+
L2BlockStreamEventHandler,
|
|
15
|
+
L2BlockStreamLocalDataProvider,
|
|
16
|
+
L2Tips,
|
|
17
|
+
} from '@aztec/stdlib/block';
|
|
18
|
+
import {
|
|
19
|
+
WorldStateRunningState,
|
|
20
|
+
type WorldStateSyncStatus,
|
|
21
|
+
type WorldStateSynchronizer,
|
|
22
|
+
type WorldStateSynchronizerStatus,
|
|
23
|
+
} from '@aztec/stdlib/interfaces/server';
|
|
24
|
+
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
25
|
+
import type { L2BlockHandledStats } from '@aztec/stdlib/stats';
|
|
26
|
+
import { MerkleTreeId, type MerkleTreeReadOperations, type MerkleTreeWriteOperations } from '@aztec/stdlib/trees';
|
|
26
27
|
import { TraceableL2BlockStream, getTelemetryClient } from '@aztec/telemetry-client';
|
|
27
28
|
|
|
28
29
|
import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
29
|
-
import {
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
30
|
+
import type { WorldStateStatusFull } from '../native/message.js';
|
|
31
|
+
import type { MerkleTreeAdminDatabase } from '../world-state-db/merkle_tree_db.js';
|
|
32
|
+
import type { WorldStateConfig } from './config.js';
|
|
32
33
|
|
|
33
34
|
/**
|
|
34
35
|
* Synchronizes the world state with the L2 blocks from a L2BlockSource via a block stream.
|
|
@@ -128,8 +129,16 @@ export class ServerWorldStateSynchronizer
|
|
|
128
129
|
}
|
|
129
130
|
|
|
130
131
|
public async status(): Promise<WorldStateSynchronizerStatus> {
|
|
132
|
+
const summary = await this.merkleTreeDb.getStatusSummary();
|
|
133
|
+
const status: WorldStateSyncStatus = {
|
|
134
|
+
latestBlockNumber: Number(summary.unfinalisedBlockNumber),
|
|
135
|
+
latestBlockHash: (await this.getL2BlockHash(Number(summary.unfinalisedBlockNumber))) ?? '',
|
|
136
|
+
finalisedBlockNumber: Number(summary.finalisedBlockNumber),
|
|
137
|
+
oldestHistoricBlockNumber: Number(summary.oldestHistoricalBlock),
|
|
138
|
+
treesAreSynched: summary.treesAreSynched,
|
|
139
|
+
};
|
|
131
140
|
return {
|
|
132
|
-
|
|
141
|
+
syncSummary: status,
|
|
133
142
|
state: this.currentState,
|
|
134
143
|
};
|
|
135
144
|
}
|
|
@@ -224,6 +233,8 @@ export class ServerWorldStateSynchronizer
|
|
|
224
233
|
* @returns Whether the block handled was produced by this same node.
|
|
225
234
|
*/
|
|
226
235
|
private async handleL2Blocks(l2Blocks: L2Block[]) {
|
|
236
|
+
this.log.trace(`Handling L2 blocks ${l2Blocks[0].number} to ${l2Blocks.at(-1)!.number}`);
|
|
237
|
+
|
|
227
238
|
const messagePromises = l2Blocks.map(block => this.l2BlockSource.getL1ToL2Messages(BigInt(block.number)));
|
|
228
239
|
const l1ToL2Messages: Fr[][] = await Promise.all(messagePromises);
|
|
229
240
|
let updateStatus: WorldStateStatusFull | undefined = undefined;
|
|
@@ -260,6 +271,11 @@ export class ServerWorldStateSynchronizer
|
|
|
260
271
|
await this.verifyMessagesHashToInHash(l1ToL2Messages, l2Block.header.contentCommitment.inHash);
|
|
261
272
|
|
|
262
273
|
// If the above check succeeds, we can proceed to handle the block.
|
|
274
|
+
this.log.trace(`Pushing L2 block ${l2Block.number} to merkle tree db `, {
|
|
275
|
+
blockNumber: l2Block.number,
|
|
276
|
+
blockHash: await l2Block.hash().then(h => h.toString()),
|
|
277
|
+
l1ToL2Messages: l1ToL2Messages.map(msg => msg.toString()),
|
|
278
|
+
});
|
|
263
279
|
const result = await this.merkleTreeDb.handleL2BlockAndMessages(l2Block, l1ToL2Messages);
|
|
264
280
|
|
|
265
281
|
if (this.currentState === WorldStateRunningState.SYNCHING && l2Block.number >= this.latestBlockNumberAtStart) {
|
|
@@ -281,7 +297,8 @@ export class ServerWorldStateSynchronizer
|
|
|
281
297
|
return;
|
|
282
298
|
}
|
|
283
299
|
this.log.verbose(`Pruning historic blocks to ${newHistoricBlock}`);
|
|
284
|
-
await this.merkleTreeDb.removeHistoricalBlocks(newHistoricBlock);
|
|
300
|
+
const status = await this.merkleTreeDb.removeHistoricalBlocks(newHistoricBlock);
|
|
301
|
+
this.log.debug(`World state summary `, status.summary);
|
|
285
302
|
}
|
|
286
303
|
|
|
287
304
|
private handleChainProven(blockNumber: number) {
|
package/src/test/utils.ts
CHANGED
|
@@ -1,20 +1,16 @@
|
|
|
1
1
|
import {
|
|
2
|
-
L2Block,
|
|
3
|
-
MerkleTreeId,
|
|
4
|
-
type MerkleTreeReadOperations,
|
|
5
|
-
type MerkleTreeWriteOperations,
|
|
6
|
-
} from '@aztec/circuit-types';
|
|
7
|
-
import {
|
|
8
|
-
AppendOnlyTreeSnapshot,
|
|
9
|
-
Fr,
|
|
10
2
|
MAX_NOTE_HASHES_PER_TX,
|
|
11
3
|
MAX_NULLIFIERS_PER_TX,
|
|
12
4
|
NULLIFIER_SUBTREE_HEIGHT,
|
|
13
5
|
NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
|
|
14
|
-
} from '@aztec/
|
|
6
|
+
} from '@aztec/constants';
|
|
15
7
|
import { padArrayEnd } from '@aztec/foundation/collection';
|
|
8
|
+
import { Fr } from '@aztec/foundation/fields';
|
|
9
|
+
import { L2Block } from '@aztec/stdlib/block';
|
|
10
|
+
import type { MerkleTreeReadOperations, MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
|
|
11
|
+
import { AppendOnlyTreeSnapshot, MerkleTreeId } from '@aztec/stdlib/trees';
|
|
16
12
|
|
|
17
|
-
import {
|
|
13
|
+
import type { NativeWorldStateService } from '../native/native_world_state.js';
|
|
18
14
|
|
|
19
15
|
export async function mockBlock(blockNum: number, size: number, fork: MerkleTreeWriteOperations) {
|
|
20
16
|
const l2Block = await L2Block.random(blockNum, size);
|