@aztec/archiver 0.0.1-commit.d431d1c → 0.0.1-commit.dbf9cec
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dest/archiver.d.ts +10 -6
- package/dest/archiver.d.ts.map +1 -1
- package/dest/archiver.js +37 -107
- package/dest/errors.d.ts +6 -1
- package/dest/errors.d.ts.map +1 -1
- package/dest/errors.js +8 -0
- package/dest/factory.d.ts +5 -2
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +16 -13
- package/dest/index.d.ts +2 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/l1/bin/retrieve-calldata.js +35 -32
- package/dest/l1/calldata_retriever.d.ts +73 -50
- package/dest/l1/calldata_retriever.d.ts.map +1 -1
- package/dest/l1/calldata_retriever.js +190 -259
- package/dest/l1/data_retrieval.d.ts +9 -9
- package/dest/l1/data_retrieval.d.ts.map +1 -1
- package/dest/l1/data_retrieval.js +24 -22
- package/dest/l1/spire_proposer.d.ts +5 -5
- package/dest/l1/spire_proposer.d.ts.map +1 -1
- package/dest/l1/spire_proposer.js +9 -17
- package/dest/l1/validate_trace.d.ts +6 -3
- package/dest/l1/validate_trace.d.ts.map +1 -1
- package/dest/l1/validate_trace.js +13 -9
- package/dest/modules/data_source_base.d.ts +23 -19
- package/dest/modules/data_source_base.d.ts.map +1 -1
- package/dest/modules/data_source_base.js +44 -119
- package/dest/modules/data_store_updater.d.ts +31 -20
- package/dest/modules/data_store_updater.d.ts.map +1 -1
- package/dest/modules/data_store_updater.js +79 -60
- package/dest/modules/instrumentation.d.ts +17 -4
- package/dest/modules/instrumentation.d.ts.map +1 -1
- package/dest/modules/instrumentation.js +36 -12
- package/dest/modules/l1_synchronizer.d.ts +4 -8
- package/dest/modules/l1_synchronizer.d.ts.map +1 -1
- package/dest/modules/l1_synchronizer.js +23 -19
- package/dest/store/block_store.d.ts +50 -32
- package/dest/store/block_store.d.ts.map +1 -1
- package/dest/store/block_store.js +147 -54
- package/dest/store/contract_class_store.d.ts +1 -1
- package/dest/store/contract_class_store.d.ts.map +1 -1
- package/dest/store/contract_class_store.js +11 -7
- package/dest/store/kv_archiver_store.d.ts +43 -25
- package/dest/store/kv_archiver_store.d.ts.map +1 -1
- package/dest/store/kv_archiver_store.js +38 -17
- package/dest/store/l2_tips_cache.d.ts +19 -0
- package/dest/store/l2_tips_cache.d.ts.map +1 -0
- package/dest/store/l2_tips_cache.js +89 -0
- package/dest/store/log_store.d.ts +4 -4
- package/dest/store/log_store.d.ts.map +1 -1
- package/dest/store/log_store.js +57 -37
- package/dest/test/fake_l1_state.d.ts +9 -4
- package/dest/test/fake_l1_state.d.ts.map +1 -1
- package/dest/test/fake_l1_state.js +56 -18
- package/dest/test/index.js +3 -1
- package/dest/test/mock_archiver.d.ts +1 -1
- package/dest/test/mock_archiver.d.ts.map +1 -1
- package/dest/test/mock_archiver.js +3 -2
- package/dest/test/mock_l2_block_source.d.ts +36 -21
- package/dest/test/mock_l2_block_source.d.ts.map +1 -1
- package/dest/test/mock_l2_block_source.js +151 -109
- package/dest/test/mock_structs.d.ts +3 -2
- package/dest/test/mock_structs.d.ts.map +1 -1
- package/dest/test/mock_structs.js +11 -9
- package/dest/test/noop_l1_archiver.d.ts +23 -0
- package/dest/test/noop_l1_archiver.d.ts.map +1 -0
- package/dest/test/noop_l1_archiver.js +68 -0
- package/package.json +14 -13
- package/src/archiver.ts +50 -132
- package/src/errors.ts +12 -0
- package/src/factory.ts +30 -14
- package/src/index.ts +1 -0
- package/src/l1/README.md +25 -68
- package/src/l1/bin/retrieve-calldata.ts +45 -33
- package/src/l1/calldata_retriever.ts +249 -379
- package/src/l1/data_retrieval.ts +27 -29
- package/src/l1/spire_proposer.ts +7 -15
- package/src/l1/validate_trace.ts +24 -6
- package/src/modules/data_source_base.ts +73 -163
- package/src/modules/data_store_updater.ts +92 -63
- package/src/modules/instrumentation.ts +46 -14
- package/src/modules/l1_synchronizer.ts +26 -24
- package/src/store/block_store.ts +188 -92
- package/src/store/contract_class_store.ts +11 -7
- package/src/store/kv_archiver_store.ts +69 -29
- package/src/store/l2_tips_cache.ts +89 -0
- package/src/store/log_store.ts +105 -43
- package/src/test/fake_l1_state.ts +77 -19
- package/src/test/index.ts +3 -0
- package/src/test/mock_archiver.ts +3 -2
- package/src/test/mock_l2_block_source.ts +196 -126
- package/src/test/mock_structs.ts +26 -10
- package/src/test/noop_l1_archiver.ts +109 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BlockNumber,
|
|
1
|
+
import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
3
|
import { createLogger } from '@aztec/foundation/log';
|
|
4
4
|
import {
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
ContractInstancePublishedEvent,
|
|
11
11
|
ContractInstanceUpdatedEvent,
|
|
12
12
|
} from '@aztec/protocol-contracts/instance-registry';
|
|
13
|
-
import type {
|
|
13
|
+
import type { L2Block, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
14
14
|
import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
15
15
|
import {
|
|
16
16
|
type ExecutablePrivateFunctionWithMembershipProof,
|
|
@@ -25,6 +25,7 @@ import type { UInt64 } from '@aztec/stdlib/types';
|
|
|
25
25
|
import groupBy from 'lodash.groupby';
|
|
26
26
|
|
|
27
27
|
import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
|
|
28
|
+
import type { L2TipsCache } from '../store/l2_tips_cache.js';
|
|
28
29
|
|
|
29
30
|
/** Operation type for contract data updates. */
|
|
30
31
|
enum Operation {
|
|
@@ -35,7 +36,7 @@ enum Operation {
|
|
|
35
36
|
/** Result of adding checkpoints with information about any pruned blocks. */
|
|
36
37
|
type ReconcileCheckpointsResult = {
|
|
37
38
|
/** Blocks that were pruned due to conflict with L1 checkpoints. */
|
|
38
|
-
prunedBlocks:
|
|
39
|
+
prunedBlocks: L2Block[] | undefined;
|
|
39
40
|
/** Last block number that was already inserted locally, or undefined if none. */
|
|
40
41
|
lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
|
|
41
42
|
};
|
|
@@ -44,20 +45,27 @@ type ReconcileCheckpointsResult = {
|
|
|
44
45
|
export class ArchiverDataStoreUpdater {
|
|
45
46
|
private readonly log = createLogger('archiver:store_updater');
|
|
46
47
|
|
|
47
|
-
constructor(
|
|
48
|
+
constructor(
|
|
49
|
+
private store: KVArchiverDataStore,
|
|
50
|
+
private l2TipsCache?: L2TipsCache,
|
|
51
|
+
) {}
|
|
48
52
|
|
|
49
53
|
/**
|
|
50
|
-
* Adds blocks to the store with contract class/instance extraction from logs.
|
|
54
|
+
* Adds proposed blocks to the store with contract class/instance extraction from logs.
|
|
55
|
+
* These are uncheckpointed blocks that have been proposed by the sequencer but not yet included in a checkpoint on L1.
|
|
51
56
|
* Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
|
|
52
57
|
* and individually broadcasted functions from the block logs.
|
|
53
58
|
*
|
|
54
|
-
* @param blocks - The L2 blocks to add.
|
|
59
|
+
* @param blocks - The proposed L2 blocks to add.
|
|
55
60
|
* @param pendingChainValidationStatus - Optional validation status to set.
|
|
56
61
|
* @returns True if the operation is successful.
|
|
57
62
|
*/
|
|
58
|
-
public
|
|
59
|
-
|
|
60
|
-
|
|
63
|
+
public async addProposedBlocks(
|
|
64
|
+
blocks: L2Block[],
|
|
65
|
+
pendingChainValidationStatus?: ValidateCheckpointResult,
|
|
66
|
+
): Promise<boolean> {
|
|
67
|
+
const result = await this.store.transactionAsync(async () => {
|
|
68
|
+
await this.store.addProposedBlocks(blocks);
|
|
61
69
|
|
|
62
70
|
const opResults = await Promise.all([
|
|
63
71
|
// Update the pending chain validation status if provided
|
|
@@ -65,16 +73,18 @@ export class ArchiverDataStoreUpdater {
|
|
|
65
73
|
// Add any logs emitted during the retrieved blocks
|
|
66
74
|
this.store.addLogs(blocks),
|
|
67
75
|
// Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
|
|
68
|
-
...blocks.map(block => this.
|
|
76
|
+
...blocks.map(block => this.addContractDataToDb(block)),
|
|
69
77
|
]);
|
|
70
78
|
|
|
79
|
+
await this.l2TipsCache?.refresh();
|
|
71
80
|
return opResults.every(Boolean);
|
|
72
81
|
});
|
|
82
|
+
return result;
|
|
73
83
|
}
|
|
74
84
|
|
|
75
85
|
/**
|
|
76
86
|
* Reconciles local blocks with incoming checkpoints from L1.
|
|
77
|
-
* Adds checkpoints to the store with contract class/instance extraction from logs.
|
|
87
|
+
* Adds new checkpoints to the store with contract class/instance extraction from logs.
|
|
78
88
|
* Prunes any local blocks that conflict with checkpoint data (by comparing archive roots).
|
|
79
89
|
* Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
|
|
80
90
|
* and individually broadcasted functions from the checkpoint block logs.
|
|
@@ -83,17 +93,17 @@ export class ArchiverDataStoreUpdater {
|
|
|
83
93
|
* @param pendingChainValidationStatus - Optional validation status to set.
|
|
84
94
|
* @returns Result with information about any pruned blocks.
|
|
85
95
|
*/
|
|
86
|
-
public
|
|
96
|
+
public async addCheckpoints(
|
|
87
97
|
checkpoints: PublishedCheckpoint[],
|
|
88
98
|
pendingChainValidationStatus?: ValidateCheckpointResult,
|
|
89
99
|
): Promise<ReconcileCheckpointsResult> {
|
|
90
|
-
|
|
100
|
+
const result = await this.store.transactionAsync(async () => {
|
|
91
101
|
// Before adding checkpoints, check for conflicts with local blocks if any
|
|
92
102
|
const { prunedBlocks, lastAlreadyInsertedBlockNumber } = await this.pruneMismatchingLocalBlocks(checkpoints);
|
|
93
103
|
|
|
94
104
|
await this.store.addCheckpoints(checkpoints);
|
|
95
105
|
|
|
96
|
-
// Filter out blocks that were already inserted via
|
|
106
|
+
// Filter out blocks that were already inserted via addProposedBlocks() to avoid duplicating logs/contract data
|
|
97
107
|
const newBlocks = checkpoints
|
|
98
108
|
.flatMap((ch: PublishedCheckpoint) => ch.checkpoint.blocks)
|
|
99
109
|
.filter(b => lastAlreadyInsertedBlockNumber === undefined || b.number > lastAlreadyInsertedBlockNumber);
|
|
@@ -104,11 +114,13 @@ export class ArchiverDataStoreUpdater {
|
|
|
104
114
|
// Add any logs emitted during the retrieved blocks
|
|
105
115
|
this.store.addLogs(newBlocks),
|
|
106
116
|
// Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
|
|
107
|
-
...newBlocks.map(block => this.
|
|
117
|
+
...newBlocks.map(block => this.addContractDataToDb(block)),
|
|
108
118
|
]);
|
|
109
119
|
|
|
120
|
+
await this.l2TipsCache?.refresh();
|
|
110
121
|
return { prunedBlocks, lastAlreadyInsertedBlockNumber };
|
|
111
122
|
});
|
|
123
|
+
return result;
|
|
112
124
|
}
|
|
113
125
|
|
|
114
126
|
/**
|
|
@@ -185,80 +197,97 @@ export class ArchiverDataStoreUpdater {
|
|
|
185
197
|
}
|
|
186
198
|
|
|
187
199
|
/**
|
|
188
|
-
* Removes all blocks strictly after the specified block number and cleans up associated contract data.
|
|
200
|
+
* Removes all uncheckpointed blocks strictly after the specified block number and cleans up associated contract data.
|
|
189
201
|
* This handles removal of provisionally added blocks along with their contract classes/instances.
|
|
202
|
+
* Verifies that each block being removed is not part of a stored checkpoint.
|
|
190
203
|
*
|
|
191
204
|
* @param blockNumber - Remove all blocks with number greater than this.
|
|
192
205
|
* @returns The removed blocks.
|
|
206
|
+
* @throws Error if any block to be removed is checkpointed.
|
|
193
207
|
*/
|
|
194
|
-
public
|
|
195
|
-
|
|
196
|
-
//
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
]);
|
|
208
|
+
public async removeUncheckpointedBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
|
|
209
|
+
const result = await this.store.transactionAsync(async () => {
|
|
210
|
+
// Verify we're only removing uncheckpointed blocks
|
|
211
|
+
const lastCheckpointedBlockNumber = await this.store.getCheckpointedL2BlockNumber();
|
|
212
|
+
if (blockNumber < lastCheckpointedBlockNumber) {
|
|
213
|
+
throw new Error(
|
|
214
|
+
`Cannot remove blocks after ${blockNumber} because checkpointed blocks exist up to ${lastCheckpointedBlockNumber}`,
|
|
215
|
+
);
|
|
216
|
+
}
|
|
204
217
|
|
|
205
|
-
|
|
218
|
+
const result = await this.removeBlocksAfter(blockNumber);
|
|
219
|
+
await this.l2TipsCache?.refresh();
|
|
220
|
+
return result;
|
|
206
221
|
});
|
|
222
|
+
return result;
|
|
207
223
|
}
|
|
208
224
|
|
|
209
225
|
/**
|
|
210
|
-
*
|
|
226
|
+
* Removes all blocks strictly after the given block number along with any logs and contract data.
|
|
227
|
+
* Does not remove their checkpoints.
|
|
228
|
+
*/
|
|
229
|
+
private async removeBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
|
|
230
|
+
// First get the blocks to be removed so we can clean up contract data
|
|
231
|
+
const removedBlocks = await this.store.removeBlocksAfter(blockNumber);
|
|
232
|
+
|
|
233
|
+
// Clean up contract data and logs for the removed blocks
|
|
234
|
+
await Promise.all([
|
|
235
|
+
this.store.deleteLogs(removedBlocks),
|
|
236
|
+
...removedBlocks.map(block => this.removeContractDataFromDb(block)),
|
|
237
|
+
]);
|
|
238
|
+
|
|
239
|
+
return removedBlocks;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Removes all checkpoints after the given checkpoint number.
|
|
211
244
|
* Deletes ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated data
|
|
212
|
-
* that was stored for the
|
|
245
|
+
* that was stored for the removed checkpoints. Also removes ALL blocks (both checkpointed
|
|
246
|
+
* and uncheckpointed) after the last block of the given checkpoint.
|
|
213
247
|
*
|
|
214
|
-
* @param
|
|
215
|
-
* @param checkpointsToUnwind - The number of checkpoints to unwind.
|
|
248
|
+
* @param checkpointNumber - Remove all checkpoints strictly after this one.
|
|
216
249
|
* @returns True if the operation is successful.
|
|
217
250
|
*/
|
|
218
|
-
public async
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
}
|
|
251
|
+
public async removeCheckpointsAfter(checkpointNumber: CheckpointNumber): Promise<boolean> {
|
|
252
|
+
return await this.store.transactionAsync(async () => {
|
|
253
|
+
const { blocksRemoved = [] } = await this.store.removeCheckpointsAfter(checkpointNumber);
|
|
222
254
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
for (let checkpointNumber = from; checkpointNumber <= lastCheckpointNumber; checkpointNumber++) {
|
|
231
|
-
const blocksForCheckpoint = await this.store.getBlocksForCheckpoint(checkpointNumber);
|
|
232
|
-
if (!blocksForCheckpoint) {
|
|
233
|
-
continue;
|
|
234
|
-
}
|
|
235
|
-
blocks.push(...blocksForCheckpoint);
|
|
236
|
-
}
|
|
255
|
+
const opResults = await Promise.all([
|
|
256
|
+
// Prune rolls back to the last proven block, which is by definition valid
|
|
257
|
+
this.store.setPendingChainValidationStatus({ valid: true }),
|
|
258
|
+
// Remove contract data for all blocks being removed
|
|
259
|
+
...blocksRemoved.map(block => this.removeContractDataFromDb(block)),
|
|
260
|
+
this.store.deleteLogs(blocksRemoved),
|
|
261
|
+
]);
|
|
237
262
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
...blocks.map(block => this.removeBlockDataFromDB(block)),
|
|
243
|
-
this.store.deleteLogs(blocks),
|
|
244
|
-
this.store.unwindCheckpoints(from, checkpointsToUnwind),
|
|
245
|
-
]);
|
|
263
|
+
await this.l2TipsCache?.refresh();
|
|
264
|
+
return opResults.every(Boolean);
|
|
265
|
+
});
|
|
266
|
+
}
|
|
246
267
|
|
|
247
|
-
|
|
268
|
+
/**
|
|
269
|
+
* Updates the proven checkpoint number and refreshes the L2 tips cache.
|
|
270
|
+
* @param checkpointNumber - The checkpoint number to set as proven.
|
|
271
|
+
*/
|
|
272
|
+
public async setProvenCheckpointNumber(checkpointNumber: CheckpointNumber): Promise<void> {
|
|
273
|
+
await this.store.transactionAsync(async () => {
|
|
274
|
+
await this.store.setProvenCheckpointNumber(checkpointNumber);
|
|
275
|
+
await this.l2TipsCache?.refresh();
|
|
276
|
+
});
|
|
248
277
|
}
|
|
249
278
|
|
|
250
279
|
/** Extracts and stores contract data from a single block. */
|
|
251
|
-
private
|
|
252
|
-
return this.
|
|
280
|
+
private addContractDataToDb(block: L2Block): Promise<boolean> {
|
|
281
|
+
return this.updateContractDataOnDb(block, Operation.Store);
|
|
253
282
|
}
|
|
254
283
|
|
|
255
284
|
/** Removes contract data associated with a block. */
|
|
256
|
-
private
|
|
257
|
-
return this.
|
|
285
|
+
private removeContractDataFromDb(block: L2Block): Promise<boolean> {
|
|
286
|
+
return this.updateContractDataOnDb(block, Operation.Delete);
|
|
258
287
|
}
|
|
259
288
|
|
|
260
289
|
/** Adds or remove contract data associated with a block. */
|
|
261
|
-
private async
|
|
290
|
+
private async updateContractDataOnDb(block: L2Block, operation: Operation): Promise<boolean> {
|
|
262
291
|
const contractClassLogs = block.body.txEffects.flatMap(txEffect => txEffect.contractClassLogs);
|
|
263
292
|
const privateLogs = block.body.txEffects.flatMap(txEffect => txEffect.privateLogs);
|
|
264
293
|
const publicLogs = block.body.txEffects.flatMap(txEffect => txEffect.publicLogs);
|
|
@@ -1,5 +1,9 @@
|
|
|
1
|
+
import type { SlotNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import { createLogger } from '@aztec/foundation/log';
|
|
2
|
-
import type {
|
|
3
|
+
import type { L2Block } from '@aztec/stdlib/block';
|
|
4
|
+
import type { CheckpointData } from '@aztec/stdlib/checkpoint';
|
|
5
|
+
import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
|
|
6
|
+
import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
|
|
3
7
|
import {
|
|
4
8
|
Attributes,
|
|
5
9
|
type Gauge,
|
|
@@ -10,12 +14,14 @@ import {
|
|
|
10
14
|
type TelemetryClient,
|
|
11
15
|
type Tracer,
|
|
12
16
|
type UpDownCounter,
|
|
17
|
+
createUpDownCounterWithDefault,
|
|
13
18
|
} from '@aztec/telemetry-client';
|
|
14
19
|
|
|
15
20
|
export class ArchiverInstrumentation {
|
|
16
21
|
public readonly tracer: Tracer;
|
|
17
22
|
|
|
18
23
|
private blockHeight: Gauge;
|
|
24
|
+
private checkpointHeight: Gauge;
|
|
19
25
|
private txCount: UpDownCounter;
|
|
20
26
|
private l1BlockHeight: Gauge;
|
|
21
27
|
private proofsSubmittedDelay: Histogram;
|
|
@@ -35,6 +41,8 @@ export class ArchiverInstrumentation {
|
|
|
35
41
|
|
|
36
42
|
private blockProposalTxTargetCount: UpDownCounter;
|
|
37
43
|
|
|
44
|
+
private checkpointL1InclusionDelay: Histogram;
|
|
45
|
+
|
|
38
46
|
private log = createLogger('archiver:instrumentation');
|
|
39
47
|
|
|
40
48
|
private constructor(
|
|
@@ -46,17 +54,21 @@ export class ArchiverInstrumentation {
|
|
|
46
54
|
|
|
47
55
|
this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT);
|
|
48
56
|
|
|
57
|
+
this.checkpointHeight = meter.createGauge(Metrics.ARCHIVER_CHECKPOINT_HEIGHT);
|
|
58
|
+
|
|
49
59
|
this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT);
|
|
50
60
|
|
|
51
|
-
this.txCount = meter
|
|
61
|
+
this.txCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_TOTAL_TXS);
|
|
52
62
|
|
|
53
|
-
this.proofsSubmittedCount = meter
|
|
63
|
+
this.proofsSubmittedCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_ROLLUP_PROOF_COUNT, {
|
|
64
|
+
[Attributes.PROOF_TIMED_OUT]: [true, false],
|
|
65
|
+
});
|
|
54
66
|
|
|
55
67
|
this.proofsSubmittedDelay = meter.createHistogram(Metrics.ARCHIVER_ROLLUP_PROOF_DELAY);
|
|
56
68
|
|
|
57
69
|
this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK);
|
|
58
70
|
|
|
59
|
-
this.syncBlockCount = meter
|
|
71
|
+
this.syncBlockCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
|
|
60
72
|
|
|
61
73
|
this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK);
|
|
62
74
|
|
|
@@ -64,13 +76,21 @@ export class ArchiverInstrumentation {
|
|
|
64
76
|
|
|
65
77
|
this.syncDurationPerMessage = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_MESSAGE);
|
|
66
78
|
|
|
67
|
-
this.syncMessageCount = meter
|
|
79
|
+
this.syncMessageCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_MESSAGE_COUNT);
|
|
68
80
|
|
|
69
81
|
this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION);
|
|
70
82
|
|
|
71
|
-
this.pruneCount = meter
|
|
83
|
+
this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT);
|
|
72
84
|
|
|
73
|
-
this.blockProposalTxTargetCount =
|
|
85
|
+
this.blockProposalTxTargetCount = createUpDownCounterWithDefault(
|
|
86
|
+
meter,
|
|
87
|
+
Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT,
|
|
88
|
+
{
|
|
89
|
+
[Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: [true, false],
|
|
90
|
+
},
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
this.checkpointL1InclusionDelay = meter.createHistogram(Metrics.ARCHIVER_CHECKPOINT_L1_INCLUSION_DELAY);
|
|
74
94
|
|
|
75
95
|
this.dbMetrics = new LmdbMetrics(
|
|
76
96
|
meter,
|
|
@@ -84,10 +104,6 @@ export class ArchiverInstrumentation {
|
|
|
84
104
|
public static async new(telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
|
|
85
105
|
const instance = new ArchiverInstrumentation(telemetry, lmdbStats);
|
|
86
106
|
|
|
87
|
-
instance.syncBlockCount.add(0);
|
|
88
|
-
instance.syncMessageCount.add(0);
|
|
89
|
-
instance.pruneCount.add(0);
|
|
90
|
-
|
|
91
107
|
await instance.telemetry.flush();
|
|
92
108
|
|
|
93
109
|
return instance;
|
|
@@ -97,9 +113,10 @@ export class ArchiverInstrumentation {
|
|
|
97
113
|
return this.telemetry.isEnabled();
|
|
98
114
|
}
|
|
99
115
|
|
|
100
|
-
public processNewBlocks(syncTimePerBlock: number, blocks:
|
|
116
|
+
public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) {
|
|
101
117
|
this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock));
|
|
102
118
|
this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
|
|
119
|
+
this.checkpointHeight.record(Math.max(...blocks.map(b => b.checkpointNumber)));
|
|
103
120
|
this.syncBlockCount.add(blocks.length);
|
|
104
121
|
|
|
105
122
|
for (const block of blocks) {
|
|
@@ -122,8 +139,10 @@ export class ArchiverInstrumentation {
|
|
|
122
139
|
this.pruneDuration.record(Math.ceil(duration));
|
|
123
140
|
}
|
|
124
141
|
|
|
125
|
-
public
|
|
126
|
-
|
|
142
|
+
public updateLastProvenCheckpoint(checkpoint: CheckpointData) {
|
|
143
|
+
const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.blockCount - 1;
|
|
144
|
+
this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' });
|
|
145
|
+
this.checkpointHeight.record(checkpoint.checkpointNumber, { [Attributes.STATUS]: 'proven' });
|
|
127
146
|
}
|
|
128
147
|
|
|
129
148
|
public processProofsVerified(logs: { proverId: string; l2BlockNumber: bigint; delay: bigint }[]) {
|
|
@@ -149,4 +168,17 @@ export class ArchiverInstrumentation {
|
|
|
149
168
|
[Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: usedTrace,
|
|
150
169
|
});
|
|
151
170
|
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Records L1 inclusion timing for a checkpoint observed on L1 (seconds into the L2 slot).
|
|
174
|
+
*/
|
|
175
|
+
public processCheckpointL1Timing(data: {
|
|
176
|
+
slotNumber: SlotNumber;
|
|
177
|
+
l1Timestamp: bigint;
|
|
178
|
+
l1Constants: Pick<L1RollupConstants, 'l1GenesisTime' | 'slotDuration'>;
|
|
179
|
+
}): void {
|
|
180
|
+
const slotStartTs = getTimestampForSlot(data.slotNumber, data.l1Constants);
|
|
181
|
+
const inclusionDelaySeconds = Number(data.l1Timestamp - slotStartTs);
|
|
182
|
+
this.checkpointL1InclusionDelay.record(inclusionDelaySeconds);
|
|
183
|
+
}
|
|
152
184
|
}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { BlobClientInterface } from '@aztec/blob-client/client';
|
|
2
2
|
import { EpochCache } from '@aztec/epoch-cache';
|
|
3
3
|
import { InboxContract, RollupContract } from '@aztec/ethereum/contracts';
|
|
4
|
-
import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
|
|
5
4
|
import type { L1BlockId } from '@aztec/ethereum/l1-types';
|
|
6
5
|
import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
|
|
7
6
|
import { maxBigint } from '@aztec/foundation/bigint';
|
|
@@ -9,14 +8,13 @@ import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/br
|
|
|
9
8
|
import { Buffer32 } from '@aztec/foundation/buffer';
|
|
10
9
|
import { pick } from '@aztec/foundation/collection';
|
|
11
10
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
12
|
-
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
13
11
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
14
12
|
import { count } from '@aztec/foundation/string';
|
|
15
13
|
import { DateProvider, Timer, elapsed } from '@aztec/foundation/timer';
|
|
16
14
|
import { isDefined } from '@aztec/foundation/types';
|
|
17
15
|
import { type ArchiverEmitter, L2BlockSourceEvents, type ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
18
16
|
import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
19
|
-
import { type L1RollupConstants, getEpochAtSlot,
|
|
17
|
+
import { type L1RollupConstants, getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers';
|
|
20
18
|
import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
|
|
21
19
|
import { type Traceable, type Tracer, execInSpan, trackSpan } from '@aztec/telemetry-client';
|
|
22
20
|
|
|
@@ -28,6 +26,7 @@ import {
|
|
|
28
26
|
retrievedToPublishedCheckpoint,
|
|
29
27
|
} from '../l1/data_retrieval.js';
|
|
30
28
|
import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
|
|
29
|
+
import type { L2TipsCache } from '../store/l2_tips_cache.js';
|
|
31
30
|
import type { InboxMessage } from '../structs/inbox_message.js';
|
|
32
31
|
import { ArchiverDataStoreUpdater } from './data_store_updater.js';
|
|
33
32
|
import type { ArchiverInstrumentation } from './instrumentation.js';
|
|
@@ -60,10 +59,6 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
60
59
|
private readonly debugClient: ViemPublicDebugClient,
|
|
61
60
|
private readonly rollup: RollupContract,
|
|
62
61
|
private readonly inbox: InboxContract,
|
|
63
|
-
private readonly l1Addresses: Pick<
|
|
64
|
-
L1ContractAddresses,
|
|
65
|
-
'registryAddress' | 'governanceProposerAddress' | 'slashFactoryAddress'
|
|
66
|
-
> & { slashingProposerAddress: EthAddress },
|
|
67
62
|
private readonly store: KVArchiverDataStore,
|
|
68
63
|
private config: {
|
|
69
64
|
batchSize: number;
|
|
@@ -77,9 +72,10 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
77
72
|
private readonly l1Constants: L1RollupConstants & { l1StartBlockHash: Buffer32; genesisArchiveRoot: Fr },
|
|
78
73
|
private readonly events: ArchiverEmitter,
|
|
79
74
|
tracer: Tracer,
|
|
75
|
+
l2TipsCache?: L2TipsCache,
|
|
80
76
|
private readonly log: Logger = createLogger('archiver:l1-sync'),
|
|
81
77
|
) {
|
|
82
|
-
this.updater = new ArchiverDataStoreUpdater(this.store);
|
|
78
|
+
this.updater = new ArchiverDataStoreUpdater(this.store, l2TipsCache);
|
|
83
79
|
this.tracer = tracer;
|
|
84
80
|
}
|
|
85
81
|
|
|
@@ -249,8 +245,7 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
249
245
|
const firstUncheckpointedBlockSlot = firstUncheckpointedBlockHeader?.getSlot();
|
|
250
246
|
|
|
251
247
|
// What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
|
|
252
|
-
const
|
|
253
|
-
const slotAtNextL1Block = getSlotAtTimestamp(nextL1BlockTimestamp, this.l1Constants);
|
|
248
|
+
const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
|
|
254
249
|
|
|
255
250
|
// Prune provisional blocks from slots that have ended without being checkpointed
|
|
256
251
|
if (firstUncheckpointedBlockSlot !== undefined && firstUncheckpointedBlockSlot < slotAtNextL1Block) {
|
|
@@ -258,7 +253,7 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
258
253
|
`Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`,
|
|
259
254
|
{ firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(), slotAtNextL1Block },
|
|
260
255
|
);
|
|
261
|
-
const prunedBlocks = await this.updater.
|
|
256
|
+
const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
|
|
262
257
|
|
|
263
258
|
if (prunedBlocks.length > 0) {
|
|
264
259
|
this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
|
|
@@ -331,10 +326,10 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
331
326
|
this.log.debug(
|
|
332
327
|
`L2 prune from ${provenCheckpointNumber + 1} to ${localPendingCheckpointNumber} will occur on next checkpoint submission.`,
|
|
333
328
|
);
|
|
334
|
-
await this.updater.
|
|
329
|
+
await this.updater.removeCheckpointsAfter(provenCheckpointNumber);
|
|
335
330
|
this.log.warn(
|
|
336
|
-
`
|
|
337
|
-
`
|
|
331
|
+
`Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` +
|
|
332
|
+
`due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
|
|
338
333
|
`Updated latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
|
|
339
334
|
);
|
|
340
335
|
this.instrumentation.processPrune(timer.ms());
|
|
@@ -551,7 +546,7 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
551
546
|
if (provenCheckpointNumber === 0) {
|
|
552
547
|
const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
|
|
553
548
|
if (localProvenCheckpointNumber !== provenCheckpointNumber) {
|
|
554
|
-
await this.
|
|
549
|
+
await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
|
|
555
550
|
this.log.info(`Rolled back proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
|
|
556
551
|
}
|
|
557
552
|
}
|
|
@@ -583,13 +578,13 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
583
578
|
) {
|
|
584
579
|
const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
|
|
585
580
|
if (localProvenCheckpointNumber !== provenCheckpointNumber) {
|
|
586
|
-
await this.
|
|
581
|
+
await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
|
|
587
582
|
this.log.info(`Updated proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
|
|
588
583
|
const provenSlotNumber = localCheckpointForDestinationProvenCheckpointNumber.header.slotNumber;
|
|
589
584
|
const provenEpochNumber: EpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants);
|
|
590
585
|
const lastBlockNumberInCheckpoint =
|
|
591
586
|
localCheckpointForDestinationProvenCheckpointNumber.startBlock +
|
|
592
|
-
localCheckpointForDestinationProvenCheckpointNumber.
|
|
587
|
+
localCheckpointForDestinationProvenCheckpointNumber.blockCount -
|
|
593
588
|
1;
|
|
594
589
|
|
|
595
590
|
this.events.emit(L2BlockSourceEvents.L2BlockProven, {
|
|
@@ -598,7 +593,7 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
598
593
|
slotNumber: provenSlotNumber,
|
|
599
594
|
epochNumber: provenEpochNumber,
|
|
600
595
|
});
|
|
601
|
-
this.instrumentation.
|
|
596
|
+
this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber);
|
|
602
597
|
} else {
|
|
603
598
|
this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`);
|
|
604
599
|
}
|
|
@@ -675,11 +670,11 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
675
670
|
tipAfterUnwind--;
|
|
676
671
|
}
|
|
677
672
|
|
|
678
|
-
const
|
|
679
|
-
await this.updater.
|
|
673
|
+
const checkpointsToRemove = localPendingCheckpointNumber - tipAfterUnwind;
|
|
674
|
+
await this.updater.removeCheckpointsAfter(CheckpointNumber(tipAfterUnwind));
|
|
680
675
|
|
|
681
676
|
this.log.warn(
|
|
682
|
-
`
|
|
677
|
+
`Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` +
|
|
683
678
|
`due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` +
|
|
684
679
|
`Updated L2 latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
|
|
685
680
|
);
|
|
@@ -707,7 +702,6 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
707
702
|
this.blobClient,
|
|
708
703
|
searchStartBlock, // TODO(palla/reorg): If the L2 reorg was due to an L1 reorg, we need to start search earlier
|
|
709
704
|
searchEndBlock,
|
|
710
|
-
this.l1Addresses,
|
|
711
705
|
this.instrumentation,
|
|
712
706
|
this.log,
|
|
713
707
|
!initialSyncComplete, // isHistoricalSync
|
|
@@ -802,12 +796,20 @@ export class ArchiverL1Synchronizer implements Traceable {
|
|
|
802
796
|
);
|
|
803
797
|
}
|
|
804
798
|
|
|
799
|
+
for (const published of validCheckpoints) {
|
|
800
|
+
this.instrumentation.processCheckpointL1Timing({
|
|
801
|
+
slotNumber: published.checkpoint.header.slotNumber,
|
|
802
|
+
l1Timestamp: published.l1.timestamp,
|
|
803
|
+
l1Constants: this.l1Constants,
|
|
804
|
+
});
|
|
805
|
+
}
|
|
806
|
+
|
|
805
807
|
try {
|
|
806
808
|
const updatedValidationResult =
|
|
807
809
|
rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
|
|
808
810
|
const [processDuration, result] = await elapsed(() =>
|
|
809
|
-
execInSpan(this.tracer, 'Archiver.
|
|
810
|
-
this.updater.
|
|
811
|
+
execInSpan(this.tracer, 'Archiver.addCheckpoints', () =>
|
|
812
|
+
this.updater.addCheckpoints(validCheckpoints, updatedValidationResult),
|
|
811
813
|
),
|
|
812
814
|
);
|
|
813
815
|
this.instrumentation.processNewBlocks(
|