@aztec/archiver 0.0.1-commit.3469e52 → 0.0.1-commit.3895657bc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dest/archiver.d.ts +12 -6
- package/dest/archiver.d.ts.map +1 -1
- package/dest/archiver.js +52 -111
- package/dest/errors.d.ts +6 -1
- package/dest/errors.d.ts.map +1 -1
- package/dest/errors.js +8 -0
- package/dest/factory.d.ts +6 -3
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +19 -14
- package/dest/index.d.ts +2 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/l1/bin/retrieve-calldata.js +35 -32
- package/dest/l1/calldata_retriever.d.ts +73 -50
- package/dest/l1/calldata_retriever.d.ts.map +1 -1
- package/dest/l1/calldata_retriever.js +190 -259
- package/dest/l1/data_retrieval.d.ts +9 -9
- package/dest/l1/data_retrieval.d.ts.map +1 -1
- package/dest/l1/data_retrieval.js +24 -22
- package/dest/l1/spire_proposer.d.ts +5 -5
- package/dest/l1/spire_proposer.d.ts.map +1 -1
- package/dest/l1/spire_proposer.js +9 -17
- package/dest/l1/validate_trace.d.ts +6 -3
- package/dest/l1/validate_trace.d.ts.map +1 -1
- package/dest/l1/validate_trace.js +13 -9
- package/dest/modules/data_source_base.d.ts +23 -19
- package/dest/modules/data_source_base.d.ts.map +1 -1
- package/dest/modules/data_source_base.js +44 -119
- package/dest/modules/data_store_updater.d.ts +35 -21
- package/dest/modules/data_store_updater.d.ts.map +1 -1
- package/dest/modules/data_store_updater.js +87 -60
- package/dest/modules/instrumentation.d.ts +17 -4
- package/dest/modules/instrumentation.d.ts.map +1 -1
- package/dest/modules/instrumentation.js +36 -12
- package/dest/modules/l1_synchronizer.d.ts +5 -8
- package/dest/modules/l1_synchronizer.d.ts.map +1 -1
- package/dest/modules/l1_synchronizer.js +25 -19
- package/dest/store/block_store.d.ts +50 -32
- package/dest/store/block_store.d.ts.map +1 -1
- package/dest/store/block_store.js +147 -54
- package/dest/store/contract_class_store.d.ts +1 -1
- package/dest/store/contract_class_store.d.ts.map +1 -1
- package/dest/store/contract_class_store.js +11 -7
- package/dest/store/kv_archiver_store.d.ts +43 -25
- package/dest/store/kv_archiver_store.d.ts.map +1 -1
- package/dest/store/kv_archiver_store.js +38 -17
- package/dest/store/l2_tips_cache.d.ts +19 -0
- package/dest/store/l2_tips_cache.d.ts.map +1 -0
- package/dest/store/l2_tips_cache.js +89 -0
- package/dest/store/log_store.d.ts +4 -4
- package/dest/store/log_store.d.ts.map +1 -1
- package/dest/store/log_store.js +57 -37
- package/dest/store/message_store.js +1 -1
- package/dest/test/fake_l1_state.d.ts +9 -4
- package/dest/test/fake_l1_state.d.ts.map +1 -1
- package/dest/test/fake_l1_state.js +56 -18
- package/dest/test/index.js +3 -1
- package/dest/test/mock_archiver.d.ts +1 -1
- package/dest/test/mock_archiver.d.ts.map +1 -1
- package/dest/test/mock_archiver.js +3 -2
- package/dest/test/mock_l2_block_source.d.ts +37 -21
- package/dest/test/mock_l2_block_source.d.ts.map +1 -1
- package/dest/test/mock_l2_block_source.js +154 -109
- package/dest/test/mock_structs.d.ts +6 -2
- package/dest/test/mock_structs.d.ts.map +1 -1
- package/dest/test/mock_structs.js +24 -10
- package/dest/test/noop_l1_archiver.d.ts +23 -0
- package/dest/test/noop_l1_archiver.d.ts.map +1 -0
- package/dest/test/noop_l1_archiver.js +68 -0
- package/package.json +14 -13
- package/src/archiver.ts +78 -137
- package/src/errors.ts +12 -0
- package/src/factory.ts +34 -15
- package/src/index.ts +1 -0
- package/src/l1/README.md +25 -68
- package/src/l1/bin/retrieve-calldata.ts +45 -33
- package/src/l1/calldata_retriever.ts +249 -379
- package/src/l1/data_retrieval.ts +27 -29
- package/src/l1/spire_proposer.ts +7 -15
- package/src/l1/validate_trace.ts +24 -6
- package/src/modules/data_source_base.ts +73 -163
- package/src/modules/data_store_updater.ts +98 -64
- package/src/modules/instrumentation.ts +46 -14
- package/src/modules/l1_synchronizer.ts +33 -25
- package/src/store/block_store.ts +188 -92
- package/src/store/contract_class_store.ts +11 -7
- package/src/store/kv_archiver_store.ts +69 -29
- package/src/store/l2_tips_cache.ts +89 -0
- package/src/store/log_store.ts +105 -43
- package/src/store/message_store.ts +1 -1
- package/src/test/fake_l1_state.ts +77 -19
- package/src/test/index.ts +3 -0
- package/src/test/mock_archiver.ts +3 -2
- package/src/test/mock_l2_block_source.ts +202 -126
- package/src/test/mock_structs.ts +45 -15
- package/src/test/noop_l1_archiver.ts +109 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BlockNumber,
|
|
1
|
+
import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
3
|
import { createLogger } from '@aztec/foundation/log';
|
|
4
4
|
import {
|
|
@@ -10,8 +10,8 @@ import {
|
|
|
10
10
|
ContractInstancePublishedEvent,
|
|
11
11
|
ContractInstanceUpdatedEvent,
|
|
12
12
|
} from '@aztec/protocol-contracts/instance-registry';
|
|
13
|
-
import type {
|
|
14
|
-
import type
|
|
13
|
+
import type { L2Block, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
14
|
+
import { type PublishedCheckpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
15
15
|
import {
|
|
16
16
|
type ExecutablePrivateFunctionWithMembershipProof,
|
|
17
17
|
type UtilityFunctionWithMembershipProof,
|
|
@@ -25,6 +25,7 @@ import type { UInt64 } from '@aztec/stdlib/types';
|
|
|
25
25
|
import groupBy from 'lodash.groupby';
|
|
26
26
|
|
|
27
27
|
import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
|
|
28
|
+
import type { L2TipsCache } from '../store/l2_tips_cache.js';
|
|
28
29
|
|
|
29
30
|
/** Operation type for contract data updates. */
|
|
30
31
|
enum Operation {
|
|
@@ -35,7 +36,7 @@ enum Operation {
|
|
|
35
36
|
/** Result of adding checkpoints with information about any pruned blocks. */
|
|
36
37
|
type ReconcileCheckpointsResult = {
|
|
37
38
|
/** Blocks that were pruned due to conflict with L1 checkpoints. */
|
|
38
|
-
prunedBlocks:
|
|
39
|
+
prunedBlocks: L2Block[] | undefined;
|
|
39
40
|
/** Last block number that was already inserted locally, or undefined if none. */
|
|
40
41
|
lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
|
|
41
42
|
};
|
|
@@ -44,20 +45,28 @@ type ReconcileCheckpointsResult = {
|
|
|
44
45
|
export class ArchiverDataStoreUpdater {
|
|
45
46
|
private readonly log = createLogger('archiver:store_updater');
|
|
46
47
|
|
|
47
|
-
constructor(
|
|
48
|
+
constructor(
|
|
49
|
+
private store: KVArchiverDataStore,
|
|
50
|
+
private l2TipsCache?: L2TipsCache,
|
|
51
|
+
private opts: { rollupManaLimit?: number } = {},
|
|
52
|
+
) {}
|
|
48
53
|
|
|
49
54
|
/**
|
|
50
|
-
* Adds blocks to the store with contract class/instance extraction from logs.
|
|
55
|
+
* Adds proposed blocks to the store with contract class/instance extraction from logs.
|
|
56
|
+
* These are uncheckpointed blocks that have been proposed by the sequencer but not yet included in a checkpoint on L1.
|
|
51
57
|
* Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
|
|
52
58
|
* and individually broadcasted functions from the block logs.
|
|
53
59
|
*
|
|
54
|
-
* @param blocks - The L2 blocks to add.
|
|
60
|
+
* @param blocks - The proposed L2 blocks to add.
|
|
55
61
|
* @param pendingChainValidationStatus - Optional validation status to set.
|
|
56
62
|
* @returns True if the operation is successful.
|
|
57
63
|
*/
|
|
58
|
-
public
|
|
59
|
-
|
|
60
|
-
|
|
64
|
+
public async addProposedBlocks(
|
|
65
|
+
blocks: L2Block[],
|
|
66
|
+
pendingChainValidationStatus?: ValidateCheckpointResult,
|
|
67
|
+
): Promise<boolean> {
|
|
68
|
+
const result = await this.store.transactionAsync(async () => {
|
|
69
|
+
await this.store.addProposedBlocks(blocks);
|
|
61
70
|
|
|
62
71
|
const opResults = await Promise.all([
|
|
63
72
|
// Update the pending chain validation status if provided
|
|
@@ -65,16 +74,18 @@ export class ArchiverDataStoreUpdater {
|
|
|
65
74
|
// Add any logs emitted during the retrieved blocks
|
|
66
75
|
this.store.addLogs(blocks),
|
|
67
76
|
// Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
|
|
68
|
-
...blocks.map(block => this.
|
|
77
|
+
...blocks.map(block => this.addContractDataToDb(block)),
|
|
69
78
|
]);
|
|
70
79
|
|
|
80
|
+
await this.l2TipsCache?.refresh();
|
|
71
81
|
return opResults.every(Boolean);
|
|
72
82
|
});
|
|
83
|
+
return result;
|
|
73
84
|
}
|
|
74
85
|
|
|
75
86
|
/**
|
|
76
87
|
* Reconciles local blocks with incoming checkpoints from L1.
|
|
77
|
-
* Adds checkpoints to the store with contract class/instance extraction from logs.
|
|
88
|
+
* Adds new checkpoints to the store with contract class/instance extraction from logs.
|
|
78
89
|
* Prunes any local blocks that conflict with checkpoint data (by comparing archive roots).
|
|
79
90
|
* Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
|
|
80
91
|
* and individually broadcasted functions from the checkpoint block logs.
|
|
@@ -83,17 +94,21 @@ export class ArchiverDataStoreUpdater {
|
|
|
83
94
|
* @param pendingChainValidationStatus - Optional validation status to set.
|
|
84
95
|
* @returns Result with information about any pruned blocks.
|
|
85
96
|
*/
|
|
86
|
-
public
|
|
97
|
+
public async addCheckpoints(
|
|
87
98
|
checkpoints: PublishedCheckpoint[],
|
|
88
99
|
pendingChainValidationStatus?: ValidateCheckpointResult,
|
|
89
100
|
): Promise<ReconcileCheckpointsResult> {
|
|
90
|
-
|
|
101
|
+
for (const checkpoint of checkpoints) {
|
|
102
|
+
validateCheckpoint(checkpoint.checkpoint, { rollupManaLimit: this.opts?.rollupManaLimit });
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const result = await this.store.transactionAsync(async () => {
|
|
91
106
|
// Before adding checkpoints, check for conflicts with local blocks if any
|
|
92
107
|
const { prunedBlocks, lastAlreadyInsertedBlockNumber } = await this.pruneMismatchingLocalBlocks(checkpoints);
|
|
93
108
|
|
|
94
109
|
await this.store.addCheckpoints(checkpoints);
|
|
95
110
|
|
|
96
|
-
// Filter out blocks that were already inserted via
|
|
111
|
+
// Filter out blocks that were already inserted via addProposedBlocks() to avoid duplicating logs/contract data
|
|
97
112
|
const newBlocks = checkpoints
|
|
98
113
|
.flatMap((ch: PublishedCheckpoint) => ch.checkpoint.blocks)
|
|
99
114
|
.filter(b => lastAlreadyInsertedBlockNumber === undefined || b.number > lastAlreadyInsertedBlockNumber);
|
|
@@ -104,11 +119,13 @@ export class ArchiverDataStoreUpdater {
|
|
|
104
119
|
// Add any logs emitted during the retrieved blocks
|
|
105
120
|
this.store.addLogs(newBlocks),
|
|
106
121
|
// Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
|
|
107
|
-
...newBlocks.map(block => this.
|
|
122
|
+
...newBlocks.map(block => this.addContractDataToDb(block)),
|
|
108
123
|
]);
|
|
109
124
|
|
|
125
|
+
await this.l2TipsCache?.refresh();
|
|
110
126
|
return { prunedBlocks, lastAlreadyInsertedBlockNumber };
|
|
111
127
|
});
|
|
128
|
+
return result;
|
|
112
129
|
}
|
|
113
130
|
|
|
114
131
|
/**
|
|
@@ -185,80 +202,97 @@ export class ArchiverDataStoreUpdater {
|
|
|
185
202
|
}
|
|
186
203
|
|
|
187
204
|
/**
|
|
188
|
-
* Removes all blocks strictly after the specified block number and cleans up associated contract data.
|
|
205
|
+
* Removes all uncheckpointed blocks strictly after the specified block number and cleans up associated contract data.
|
|
189
206
|
* This handles removal of provisionally added blocks along with their contract classes/instances.
|
|
207
|
+
* Verifies that each block being removed is not part of a stored checkpoint.
|
|
190
208
|
*
|
|
191
209
|
* @param blockNumber - Remove all blocks with number greater than this.
|
|
192
210
|
* @returns The removed blocks.
|
|
211
|
+
* @throws Error if any block to be removed is checkpointed.
|
|
193
212
|
*/
|
|
194
|
-
public
|
|
195
|
-
|
|
196
|
-
//
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
]);
|
|
213
|
+
public async removeUncheckpointedBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
|
|
214
|
+
const result = await this.store.transactionAsync(async () => {
|
|
215
|
+
// Verify we're only removing uncheckpointed blocks
|
|
216
|
+
const lastCheckpointedBlockNumber = await this.store.getCheckpointedL2BlockNumber();
|
|
217
|
+
if (blockNumber < lastCheckpointedBlockNumber) {
|
|
218
|
+
throw new Error(
|
|
219
|
+
`Cannot remove blocks after ${blockNumber} because checkpointed blocks exist up to ${lastCheckpointedBlockNumber}`,
|
|
220
|
+
);
|
|
221
|
+
}
|
|
204
222
|
|
|
205
|
-
|
|
223
|
+
const result = await this.removeBlocksAfter(blockNumber);
|
|
224
|
+
await this.l2TipsCache?.refresh();
|
|
225
|
+
return result;
|
|
206
226
|
});
|
|
227
|
+
return result;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Removes all blocks strictly after the given block number along with any logs and contract data.
|
|
232
|
+
* Does not remove their checkpoints.
|
|
233
|
+
*/
|
|
234
|
+
private async removeBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
|
|
235
|
+
// First get the blocks to be removed so we can clean up contract data
|
|
236
|
+
const removedBlocks = await this.store.removeBlocksAfter(blockNumber);
|
|
237
|
+
|
|
238
|
+
// Clean up contract data and logs for the removed blocks
|
|
239
|
+
await Promise.all([
|
|
240
|
+
this.store.deleteLogs(removedBlocks),
|
|
241
|
+
...removedBlocks.map(block => this.removeContractDataFromDb(block)),
|
|
242
|
+
]);
|
|
243
|
+
|
|
244
|
+
return removedBlocks;
|
|
207
245
|
}
|
|
208
246
|
|
|
209
247
|
/**
|
|
210
|
-
*
|
|
248
|
+
* Removes all checkpoints after the given checkpoint number.
|
|
211
249
|
* Deletes ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated data
|
|
212
|
-
* that was stored for the
|
|
250
|
+
* that was stored for the removed checkpoints. Also removes ALL blocks (both checkpointed
|
|
251
|
+
* and uncheckpointed) after the last block of the given checkpoint.
|
|
213
252
|
*
|
|
214
|
-
* @param
|
|
215
|
-
* @param checkpointsToUnwind - The number of checkpoints to unwind.
|
|
253
|
+
* @param checkpointNumber - Remove all checkpoints strictly after this one.
|
|
216
254
|
* @returns True if the operation is successful.
|
|
217
255
|
*/
|
|
218
|
-
public async
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
const last = await this.store.getSynchedCheckpointNumber();
|
|
224
|
-
if (from != last) {
|
|
225
|
-
throw new Error(`Cannot unwind checkpoints from checkpoint ${from} when the last checkpoint is ${last}`);
|
|
226
|
-
}
|
|
256
|
+
public async removeCheckpointsAfter(checkpointNumber: CheckpointNumber): Promise<boolean> {
|
|
257
|
+
return await this.store.transactionAsync(async () => {
|
|
258
|
+
const { blocksRemoved = [] } = await this.store.removeCheckpointsAfter(checkpointNumber);
|
|
227
259
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
blocks.push(...blocksForCheckpoint);
|
|
236
|
-
}
|
|
260
|
+
const opResults = await Promise.all([
|
|
261
|
+
// Prune rolls back to the last proven block, which is by definition valid
|
|
262
|
+
this.store.setPendingChainValidationStatus({ valid: true }),
|
|
263
|
+
// Remove contract data for all blocks being removed
|
|
264
|
+
...blocksRemoved.map(block => this.removeContractDataFromDb(block)),
|
|
265
|
+
this.store.deleteLogs(blocksRemoved),
|
|
266
|
+
]);
|
|
237
267
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
...blocks.map(block => this.removeBlockDataFromDB(block)),
|
|
243
|
-
this.store.deleteLogs(blocks),
|
|
244
|
-
this.store.unwindCheckpoints(from, checkpointsToUnwind),
|
|
245
|
-
]);
|
|
268
|
+
await this.l2TipsCache?.refresh();
|
|
269
|
+
return opResults.every(Boolean);
|
|
270
|
+
});
|
|
271
|
+
}
|
|
246
272
|
|
|
247
|
-
|
|
273
|
+
/**
|
|
274
|
+
* Updates the proven checkpoint number and refreshes the L2 tips cache.
|
|
275
|
+
* @param checkpointNumber - The checkpoint number to set as proven.
|
|
276
|
+
*/
|
|
277
|
+
public async setProvenCheckpointNumber(checkpointNumber: CheckpointNumber): Promise<void> {
|
|
278
|
+
await this.store.transactionAsync(async () => {
|
|
279
|
+
await this.store.setProvenCheckpointNumber(checkpointNumber);
|
|
280
|
+
await this.l2TipsCache?.refresh();
|
|
281
|
+
});
|
|
248
282
|
}
|
|
249
283
|
|
|
250
284
|
/** Extracts and stores contract data from a single block. */
|
|
251
|
-
private
|
|
252
|
-
return this.
|
|
285
|
+
private addContractDataToDb(block: L2Block): Promise<boolean> {
|
|
286
|
+
return this.updateContractDataOnDb(block, Operation.Store);
|
|
253
287
|
}
|
|
254
288
|
|
|
255
289
|
/** Removes contract data associated with a block. */
|
|
256
|
-
private
|
|
257
|
-
return this.
|
|
290
|
+
private removeContractDataFromDb(block: L2Block): Promise<boolean> {
|
|
291
|
+
return this.updateContractDataOnDb(block, Operation.Delete);
|
|
258
292
|
}
|
|
259
293
|
|
|
260
294
|
/** Adds or remove contract data associated with a block. */
|
|
261
|
-
private async
|
|
295
|
+
private async updateContractDataOnDb(block: L2Block, operation: Operation): Promise<boolean> {
|
|
262
296
|
const contractClassLogs = block.body.txEffects.flatMap(txEffect => txEffect.contractClassLogs);
|
|
263
297
|
const privateLogs = block.body.txEffects.flatMap(txEffect => txEffect.privateLogs);
|
|
264
298
|
const publicLogs = block.body.txEffects.flatMap(txEffect => txEffect.publicLogs);
|
|
@@ -1,5 +1,9 @@
|
|
|
1
|
+
import type { SlotNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import { createLogger } from '@aztec/foundation/log';
|
|
2
|
-
import type {
|
|
3
|
+
import type { L2Block } from '@aztec/stdlib/block';
|
|
4
|
+
import type { CheckpointData } from '@aztec/stdlib/checkpoint';
|
|
5
|
+
import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
|
|
6
|
+
import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
|
|
3
7
|
import {
|
|
4
8
|
Attributes,
|
|
5
9
|
type Gauge,
|
|
@@ -10,12 +14,14 @@ import {
|
|
|
10
14
|
type TelemetryClient,
|
|
11
15
|
type Tracer,
|
|
12
16
|
type UpDownCounter,
|
|
17
|
+
createUpDownCounterWithDefault,
|
|
13
18
|
} from '@aztec/telemetry-client';
|
|
14
19
|
|
|
15
20
|
export class ArchiverInstrumentation {
|
|
16
21
|
public readonly tracer: Tracer;
|
|
17
22
|
|
|
18
23
|
private blockHeight: Gauge;
|
|
24
|
+
private checkpointHeight: Gauge;
|
|
19
25
|
private txCount: UpDownCounter;
|
|
20
26
|
private l1BlockHeight: Gauge;
|
|
21
27
|
private proofsSubmittedDelay: Histogram;
|
|
@@ -35,6 +41,8 @@ export class ArchiverInstrumentation {
|
|
|
35
41
|
|
|
36
42
|
private blockProposalTxTargetCount: UpDownCounter;
|
|
37
43
|
|
|
44
|
+
private checkpointL1InclusionDelay: Histogram;
|
|
45
|
+
|
|
38
46
|
private log = createLogger('archiver:instrumentation');
|
|
39
47
|
|
|
40
48
|
private constructor(
|
|
@@ -46,17 +54,21 @@ export class ArchiverInstrumentation {
|
|
|
46
54
|
|
|
47
55
|
this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT);
|
|
48
56
|
|
|
57
|
+
this.checkpointHeight = meter.createGauge(Metrics.ARCHIVER_CHECKPOINT_HEIGHT);
|
|
58
|
+
|
|
49
59
|
this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT);
|
|
50
60
|
|
|
51
|
-
this.txCount = meter
|
|
61
|
+
this.txCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_TOTAL_TXS);
|
|
52
62
|
|
|
53
|
-
this.proofsSubmittedCount = meter
|
|
63
|
+
this.proofsSubmittedCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_ROLLUP_PROOF_COUNT, {
|
|
64
|
+
[Attributes.PROOF_TIMED_OUT]: [true, false],
|
|
65
|
+
});
|
|
54
66
|
|
|
55
67
|
this.proofsSubmittedDelay = meter.createHistogram(Metrics.ARCHIVER_ROLLUP_PROOF_DELAY);
|
|
56
68
|
|
|
57
69
|
this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK);
|
|
58
70
|
|
|
59
|
-
this.syncBlockCount = meter
|
|
71
|
+
this.syncBlockCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
|
|
60
72
|
|
|
61
73
|
this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK);
|
|
62
74
|
|
|
@@ -64,13 +76,21 @@ export class ArchiverInstrumentation {
|
|
|
64
76
|
|
|
65
77
|
this.syncDurationPerMessage = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_MESSAGE);
|
|
66
78
|
|
|
67
|
-
this.syncMessageCount = meter
|
|
79
|
+
this.syncMessageCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_MESSAGE_COUNT);
|
|
68
80
|
|
|
69
81
|
this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION);
|
|
70
82
|
|
|
71
|
-
this.pruneCount = meter
|
|
83
|
+
this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT);
|
|
72
84
|
|
|
73
|
-
this.blockProposalTxTargetCount =
|
|
85
|
+
this.blockProposalTxTargetCount = createUpDownCounterWithDefault(
|
|
86
|
+
meter,
|
|
87
|
+
Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT,
|
|
88
|
+
{
|
|
89
|
+
[Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: [true, false],
|
|
90
|
+
},
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
this.checkpointL1InclusionDelay = meter.createHistogram(Metrics.ARCHIVER_CHECKPOINT_L1_INCLUSION_DELAY);
|
|
74
94
|
|
|
75
95
|
this.dbMetrics = new LmdbMetrics(
|
|
76
96
|
meter,
|
|
@@ -84,10 +104,6 @@ export class ArchiverInstrumentation {
|
|
|
84
104
|
public static async new(telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
|
|
85
105
|
const instance = new ArchiverInstrumentation(telemetry, lmdbStats);
|
|
86
106
|
|
|
87
|
-
instance.syncBlockCount.add(0);
|
|
88
|
-
instance.syncMessageCount.add(0);
|
|
89
|
-
instance.pruneCount.add(0);
|
|
90
|
-
|
|
91
107
|
await instance.telemetry.flush();
|
|
92
108
|
|
|
93
109
|
return instance;
|
|
@@ -97,9 +113,10 @@ export class ArchiverInstrumentation {
|
|
|
97
113
|
return this.telemetry.isEnabled();
|
|
98
114
|
}
|
|
99
115
|
|
|
100
|
-
public processNewBlocks(syncTimePerBlock: number, blocks:
|
|
116
|
+
public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) {
|
|
101
117
|
this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock));
|
|
102
118
|
this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
|
|
119
|
+
this.checkpointHeight.record(Math.max(...blocks.map(b => b.checkpointNumber)));
|
|
103
120
|
this.syncBlockCount.add(blocks.length);
|
|
104
121
|
|
|
105
122
|
for (const block of blocks) {
|
|
@@ -122,8 +139,10 @@ export class ArchiverInstrumentation {
|
|
|
122
139
|
this.pruneDuration.record(Math.ceil(duration));
|
|
123
140
|
}
|
|
124
141
|
|
|
125
|
-
public
|
|
126
|
-
|
|
142
|
+
public updateLastProvenCheckpoint(checkpoint: CheckpointData) {
|
|
143
|
+
const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.blockCount - 1;
|
|
144
|
+
this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' });
|
|
145
|
+
this.checkpointHeight.record(checkpoint.checkpointNumber, { [Attributes.STATUS]: 'proven' });
|
|
127
146
|
}
|
|
128
147
|
|
|
129
148
|
public processProofsVerified(logs: { proverId: string; l2BlockNumber: bigint; delay: bigint }[]) {
|
|
@@ -149,4 +168,17 @@ export class ArchiverInstrumentation {
|
|
|
149
168
|
[Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: usedTrace,
|
|
150
169
|
});
|
|
151
170
|
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Records L1 inclusion timing for a checkpoint observed on L1 (seconds into the L2 slot).
|
|
174
|
+
*/
|
|
175
|
+
public processCheckpointL1Timing(data: {
|
|
176
|
+
slotNumber: SlotNumber;
|
|
177
|
+
l1Timestamp: bigint;
|
|
178
|
+
l1Constants: Pick<L1RollupConstants, 'l1GenesisTime' | 'slotDuration'>;
|
|
179
|
+
}): void {
|
|
180
|
+
const slotStartTs = getTimestampForSlot(data.slotNumber, data.l1Constants);
|
|
181
|
+
const inclusionDelaySeconds = Number(data.l1Timestamp - slotStartTs);
|
|
182
|
+
this.checkpointL1InclusionDelay.record(inclusionDelaySeconds);
|
|
183
|
+
}
|
|
152
184
|
}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { BlobClientInterface } from '@aztec/blob-client/client';
|
|
2
2
|
import { EpochCache } from '@aztec/epoch-cache';
|
|
3
3
|
import { InboxContract, RollupContract } from '@aztec/ethereum/contracts';
|
|
4
|
-
import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
|
|
5
4
|
import type { L1BlockId } from '@aztec/ethereum/l1-types';
|
|
6
5
|
import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
|
|
7
6
|
import { maxBigint } from '@aztec/foundation/bigint';
|
|
@@ -9,14 +8,13 @@ import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/br
|
|
|
9
8
|
import { Buffer32 } from '@aztec/foundation/buffer';
|
|
10
9
|
import { pick } from '@aztec/foundation/collection';
|
|
11
10
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
12
|
-
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
13
11
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
14
12
|
import { count } from '@aztec/foundation/string';
|
|
15
13
|
import { DateProvider, Timer, elapsed } from '@aztec/foundation/timer';
|
|
16
14
|
import { isDefined } from '@aztec/foundation/types';
|
|
17
15
|
import { type ArchiverEmitter, L2BlockSourceEvents, type ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
18
16
|
import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
19
|
-
import { type L1RollupConstants, getEpochAtSlot,
|
|
17
|
+
import { type L1RollupConstants, getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers';
|
|
20
18
|
import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
|
|
21
19
|
import { type Traceable, type Tracer, execInSpan, trackSpan } from '@aztec/telemetry-client';
|
|
22
20
|
|
|
@@ -28,6 +26,7 @@ import {
|
|
|
28
26
|
retrievedToPublishedCheckpoint,
|
|
29
27
|
} from '../l1/data_retrieval.js';
|
|
30
28
|
import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
|
|
29
|
+
import type { L2TipsCache } from '../store/l2_tips_cache.js';
|
|
31
30
|
import type { InboxMessage } from '../structs/inbox_message.js';
|
|
32
31
|
import { ArchiverDataStoreUpdater } from './data_store_updater.js';
|
|
33
32
|
import type { ArchiverInstrumentation } from './instrumentation.js';
|
|
@@ -60,10 +59,6 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
60
59
|
private readonly debugClient: ViemPublicDebugClient,
|
|
61
60
|
private readonly rollup: RollupContract,
|
|
62
61
|
private readonly inbox: InboxContract,
|
|
63
|
-
private readonly l1Addresses: Pick<
|
|
64
|
-
L1ContractAddresses,
|
|
65
|
-
'registryAddress' | 'governanceProposerAddress' | 'slashFactoryAddress'
|
|
66
|
-
> & { slashingProposerAddress: EthAddress },
|
|
67
62
|
private readonly store: KVArchiverDataStore,
|
|
68
63
|
private config: {
|
|
69
64
|
batchSize: number;
|
|
@@ -74,12 +69,19 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
74
69
|
private readonly epochCache: EpochCache,
|
|
75
70
|
private readonly dateProvider: DateProvider,
|
|
76
71
|
private readonly instrumentation: ArchiverInstrumentation,
|
|
77
|
-
private readonly l1Constants: L1RollupConstants & {
|
|
72
|
+
private readonly l1Constants: L1RollupConstants & {
|
|
73
|
+
l1StartBlockHash: Buffer32;
|
|
74
|
+
genesisArchiveRoot: Fr;
|
|
75
|
+
rollupManaLimit?: number;
|
|
76
|
+
},
|
|
78
77
|
private readonly events: ArchiverEmitter,
|
|
79
78
|
tracer: Tracer,
|
|
79
|
+
l2TipsCache?: L2TipsCache,
|
|
80
80
|
private readonly log: Logger = createLogger('archiver:l1-sync'),
|
|
81
81
|
) {
|
|
82
|
-
this.updater = new ArchiverDataStoreUpdater(this.store
|
|
82
|
+
this.updater = new ArchiverDataStoreUpdater(this.store, l2TipsCache, {
|
|
83
|
+
rollupManaLimit: l1Constants.rollupManaLimit,
|
|
84
|
+
});
|
|
83
85
|
this.tracer = tracer;
|
|
84
86
|
}
|
|
85
87
|
|
|
@@ -249,8 +251,7 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
249
251
|
const firstUncheckpointedBlockSlot = firstUncheckpointedBlockHeader?.getSlot();
|
|
250
252
|
|
|
251
253
|
// What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
|
|
252
|
-
const
|
|
253
|
-
const slotAtNextL1Block = getSlotAtTimestamp(nextL1BlockTimestamp, this.l1Constants);
|
|
254
|
+
const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
|
|
254
255
|
|
|
255
256
|
// Prune provisional blocks from slots that have ended without being checkpointed
|
|
256
257
|
if (firstUncheckpointedBlockSlot !== undefined && firstUncheckpointedBlockSlot < slotAtNextL1Block) {
|
|
@@ -258,7 +259,7 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
258
259
|
`Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`,
|
|
259
260
|
{ firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(), slotAtNextL1Block },
|
|
260
261
|
);
|
|
261
|
-
const prunedBlocks = await this.updater.
|
|
262
|
+
const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
|
|
262
263
|
|
|
263
264
|
if (prunedBlocks.length > 0) {
|
|
264
265
|
this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
|
|
@@ -331,10 +332,10 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
331
332
|
this.log.debug(
|
|
332
333
|
`L2 prune from ${provenCheckpointNumber + 1} to ${localPendingCheckpointNumber} will occur on next checkpoint submission.`,
|
|
333
334
|
);
|
|
334
|
-
await this.updater.
|
|
335
|
+
await this.updater.removeCheckpointsAfter(provenCheckpointNumber);
|
|
335
336
|
this.log.warn(
|
|
336
|
-
`
|
|
337
|
-
`
|
|
337
|
+
`Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` +
|
|
338
|
+
`due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
|
|
338
339
|
`Updated latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
|
|
339
340
|
);
|
|
340
341
|
this.instrumentation.processPrune(timer.ms());
|
|
@@ -551,7 +552,7 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
551
552
|
if (provenCheckpointNumber === 0) {
|
|
552
553
|
const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
|
|
553
554
|
if (localProvenCheckpointNumber !== provenCheckpointNumber) {
|
|
554
|
-
await this.
|
|
555
|
+
await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
|
|
555
556
|
this.log.info(`Rolled back proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
|
|
556
557
|
}
|
|
557
558
|
}
|
|
@@ -583,13 +584,13 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
583
584
|
) {
|
|
584
585
|
const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
|
|
585
586
|
if (localProvenCheckpointNumber !== provenCheckpointNumber) {
|
|
586
|
-
await this.
|
|
587
|
+
await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
|
|
587
588
|
this.log.info(`Updated proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
|
|
588
589
|
const provenSlotNumber = localCheckpointForDestinationProvenCheckpointNumber.header.slotNumber;
|
|
589
590
|
const provenEpochNumber: EpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants);
|
|
590
591
|
const lastBlockNumberInCheckpoint =
|
|
591
592
|
localCheckpointForDestinationProvenCheckpointNumber.startBlock +
|
|
592
|
-
localCheckpointForDestinationProvenCheckpointNumber.
|
|
593
|
+
localCheckpointForDestinationProvenCheckpointNumber.blockCount -
|
|
593
594
|
1;
|
|
594
595
|
|
|
595
596
|
this.events.emit(L2BlockSourceEvents.L2BlockProven, {
|
|
@@ -598,7 +599,7 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
598
599
|
slotNumber: provenSlotNumber,
|
|
599
600
|
epochNumber: provenEpochNumber,
|
|
600
601
|
});
|
|
601
|
-
this.instrumentation.
|
|
602
|
+
this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber);
|
|
602
603
|
} else {
|
|
603
604
|
this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`);
|
|
604
605
|
}
|
|
@@ -675,11 +676,11 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
675
676
|
tipAfterUnwind--;
|
|
676
677
|
}
|
|
677
678
|
|
|
678
|
-
const
|
|
679
|
-
await this.updater.
|
|
679
|
+
const checkpointsToRemove = localPendingCheckpointNumber - tipAfterUnwind;
|
|
680
|
+
await this.updater.removeCheckpointsAfter(CheckpointNumber(tipAfterUnwind));
|
|
680
681
|
|
|
681
682
|
this.log.warn(
|
|
682
|
-
`
|
|
683
|
+
`Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` +
|
|
683
684
|
`due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` +
|
|
684
685
|
`Updated L2 latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
|
|
685
686
|
);
|
|
@@ -707,7 +708,6 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
707
708
|
this.blobClient,
|
|
708
709
|
searchStartBlock, // TODO(palla/reorg): If the L2 reorg was due to an L1 reorg, we need to start search earlier
|
|
709
710
|
searchEndBlock,
|
|
710
|
-
this.l1Addresses,
|
|
711
711
|
this.instrumentation,
|
|
712
712
|
this.log,
|
|
713
713
|
!initialSyncComplete, // isHistoricalSync
|
|
@@ -802,12 +802,20 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
802
802
|
);
|
|
803
803
|
}
|
|
804
804
|
|
|
805
|
+
for (const published of validCheckpoints) {
|
|
806
|
+
this.instrumentation.processCheckpointL1Timing({
|
|
807
|
+
slotNumber: published.checkpoint.header.slotNumber,
|
|
808
|
+
l1Timestamp: published.l1.timestamp,
|
|
809
|
+
l1Constants: this.l1Constants,
|
|
810
|
+
});
|
|
811
|
+
}
|
|
812
|
+
|
|
805
813
|
try {
|
|
806
814
|
const updatedValidationResult =
|
|
807
815
|
rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
|
|
808
816
|
const [processDuration, result] = await elapsed(() =>
|
|
809
|
-
execInSpan(this.tracer, 'Archiver.
|
|
810
|
-
this.updater.
|
|
817
|
+
execInSpan(this.tracer, 'Archiver.addCheckpoints', () =>
|
|
818
|
+
this.updater.addCheckpoints(validCheckpoints, updatedValidationResult),
|
|
811
819
|
),
|
|
812
820
|
);
|
|
813
821
|
this.instrumentation.processNewBlocks(
|