@aztec/archiver 0.0.1-commit.3469e52 → 0.0.1-commit.381b1a9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +9 -0
  2. package/dest/archiver.d.ts +12 -8
  3. package/dest/archiver.d.ts.map +1 -1
  4. package/dest/archiver.js +81 -120
  5. package/dest/errors.d.ts +6 -1
  6. package/dest/errors.d.ts.map +1 -1
  7. package/dest/errors.js +8 -0
  8. package/dest/factory.d.ts +3 -1
  9. package/dest/factory.d.ts.map +1 -1
  10. package/dest/factory.js +14 -11
  11. package/dest/index.d.ts +2 -1
  12. package/dest/index.d.ts.map +1 -1
  13. package/dest/index.js +1 -0
  14. package/dest/l1/bin/retrieve-calldata.js +35 -32
  15. package/dest/l1/calldata_retriever.d.ts +73 -50
  16. package/dest/l1/calldata_retriever.d.ts.map +1 -1
  17. package/dest/l1/calldata_retriever.js +190 -259
  18. package/dest/l1/data_retrieval.d.ts +4 -7
  19. package/dest/l1/data_retrieval.d.ts.map +1 -1
  20. package/dest/l1/data_retrieval.js +12 -16
  21. package/dest/l1/spire_proposer.d.ts +5 -5
  22. package/dest/l1/spire_proposer.d.ts.map +1 -1
  23. package/dest/l1/spire_proposer.js +9 -17
  24. package/dest/l1/validate_trace.d.ts +6 -3
  25. package/dest/l1/validate_trace.d.ts.map +1 -1
  26. package/dest/l1/validate_trace.js +13 -9
  27. package/dest/modules/data_source_base.d.ts +25 -21
  28. package/dest/modules/data_source_base.d.ts.map +1 -1
  29. package/dest/modules/data_source_base.js +45 -120
  30. package/dest/modules/data_store_updater.d.ts +40 -21
  31. package/dest/modules/data_store_updater.d.ts.map +1 -1
  32. package/dest/modules/data_store_updater.js +114 -64
  33. package/dest/modules/instrumentation.d.ts +6 -4
  34. package/dest/modules/instrumentation.d.ts.map +1 -1
  35. package/dest/modules/instrumentation.js +26 -12
  36. package/dest/modules/l1_synchronizer.d.ts +5 -8
  37. package/dest/modules/l1_synchronizer.d.ts.map +1 -1
  38. package/dest/modules/l1_synchronizer.js +46 -20
  39. package/dest/store/block_store.d.ts +49 -32
  40. package/dest/store/block_store.d.ts.map +1 -1
  41. package/dest/store/block_store.js +165 -54
  42. package/dest/store/contract_class_store.d.ts +1 -1
  43. package/dest/store/contract_class_store.d.ts.map +1 -1
  44. package/dest/store/contract_class_store.js +11 -7
  45. package/dest/store/kv_archiver_store.d.ts +53 -25
  46. package/dest/store/kv_archiver_store.d.ts.map +1 -1
  47. package/dest/store/kv_archiver_store.js +50 -17
  48. package/dest/store/l2_tips_cache.d.ts +19 -0
  49. package/dest/store/l2_tips_cache.d.ts.map +1 -0
  50. package/dest/store/l2_tips_cache.js +89 -0
  51. package/dest/store/log_store.d.ts +4 -4
  52. package/dest/store/log_store.d.ts.map +1 -1
  53. package/dest/store/log_store.js +105 -47
  54. package/dest/store/message_store.js +1 -1
  55. package/dest/test/fake_l1_state.d.ts +16 -4
  56. package/dest/test/fake_l1_state.d.ts.map +1 -1
  57. package/dest/test/fake_l1_state.js +84 -20
  58. package/dest/test/index.js +3 -1
  59. package/dest/test/mock_archiver.d.ts +1 -1
  60. package/dest/test/mock_archiver.d.ts.map +1 -1
  61. package/dest/test/mock_archiver.js +3 -2
  62. package/dest/test/mock_l2_block_source.d.ts +39 -23
  63. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  64. package/dest/test/mock_l2_block_source.js +157 -112
  65. package/dest/test/mock_structs.d.ts +6 -2
  66. package/dest/test/mock_structs.d.ts.map +1 -1
  67. package/dest/test/mock_structs.js +24 -10
  68. package/dest/test/noop_l1_archiver.d.ts +26 -0
  69. package/dest/test/noop_l1_archiver.d.ts.map +1 -0
  70. package/dest/test/noop_l1_archiver.js +72 -0
  71. package/package.json +14 -13
  72. package/src/archiver.ts +106 -149
  73. package/src/errors.ts +12 -0
  74. package/src/factory.ts +29 -12
  75. package/src/index.ts +1 -0
  76. package/src/l1/README.md +25 -68
  77. package/src/l1/bin/retrieve-calldata.ts +45 -33
  78. package/src/l1/calldata_retriever.ts +249 -379
  79. package/src/l1/data_retrieval.ts +10 -20
  80. package/src/l1/spire_proposer.ts +7 -15
  81. package/src/l1/validate_trace.ts +24 -6
  82. package/src/modules/data_source_base.ts +76 -166
  83. package/src/modules/data_store_updater.ts +130 -66
  84. package/src/modules/instrumentation.ts +26 -14
  85. package/src/modules/l1_synchronizer.ts +55 -26
  86. package/src/store/block_store.ts +216 -92
  87. package/src/store/contract_class_store.ts +11 -7
  88. package/src/store/kv_archiver_store.ts +88 -30
  89. package/src/store/l2_tips_cache.ts +89 -0
  90. package/src/store/log_store.ts +171 -55
  91. package/src/store/message_store.ts +1 -1
  92. package/src/test/fake_l1_state.ts +112 -23
  93. package/src/test/index.ts +3 -0
  94. package/src/test/mock_archiver.ts +3 -2
  95. package/src/test/mock_l2_block_source.ts +211 -129
  96. package/src/test/mock_structs.ts +45 -15
  97. package/src/test/noop_l1_archiver.ts +115 -0
@@ -1,4 +1,5 @@
1
- import { BlockNumber, type CheckpointNumber } from '@aztec/foundation/branded-types';
1
+ import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
2
+ import { filterAsync } from '@aztec/foundation/collection';
2
3
  import { Fr } from '@aztec/foundation/curves/bn254';
3
4
  import { createLogger } from '@aztec/foundation/log';
4
5
  import {
@@ -10,11 +11,12 @@ import {
10
11
  ContractInstancePublishedEvent,
11
12
  ContractInstanceUpdatedEvent,
12
13
  } from '@aztec/protocol-contracts/instance-registry';
13
- import type { L2BlockNew, ValidateCheckpointResult } from '@aztec/stdlib/block';
14
- import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
14
+ import type { L2Block, ValidateCheckpointResult } from '@aztec/stdlib/block';
15
+ import { type PublishedCheckpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
15
16
  import {
16
17
  type ExecutablePrivateFunctionWithMembershipProof,
17
18
  type UtilityFunctionWithMembershipProof,
19
+ computeContractAddressFromInstance,
18
20
  computePublicBytecodeCommitment,
19
21
  isValidPrivateFunctionMembershipProof,
20
22
  isValidUtilityFunctionMembershipProof,
@@ -25,6 +27,7 @@ import type { UInt64 } from '@aztec/stdlib/types';
25
27
  import groupBy from 'lodash.groupby';
26
28
 
27
29
  import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
30
+ import type { L2TipsCache } from '../store/l2_tips_cache.js';
28
31
 
29
32
  /** Operation type for contract data updates. */
30
33
  enum Operation {
@@ -35,7 +38,7 @@ enum Operation {
35
38
  /** Result of adding checkpoints with information about any pruned blocks. */
36
39
  type ReconcileCheckpointsResult = {
37
40
  /** Blocks that were pruned due to conflict with L1 checkpoints. */
38
- prunedBlocks: L2BlockNew[] | undefined;
41
+ prunedBlocks: L2Block[] | undefined;
39
42
  /** Last block number that was already inserted locally, or undefined if none. */
40
43
  lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
41
44
  };
@@ -44,20 +47,28 @@ type ReconcileCheckpointsResult = {
44
47
  export class ArchiverDataStoreUpdater {
45
48
  private readonly log = createLogger('archiver:store_updater');
46
49
 
47
- constructor(private store: KVArchiverDataStore) {}
50
+ constructor(
51
+ private store: KVArchiverDataStore,
52
+ private l2TipsCache?: L2TipsCache,
53
+ private opts: { rollupManaLimit?: number } = {},
54
+ ) {}
48
55
 
49
56
  /**
50
- * Adds blocks to the store with contract class/instance extraction from logs.
57
+ * Adds proposed blocks to the store with contract class/instance extraction from logs.
58
+ * These are uncheckpointed blocks that have been proposed by the sequencer but not yet included in a checkpoint on L1.
51
59
  * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
52
60
  * and individually broadcasted functions from the block logs.
53
61
  *
54
- * @param blocks - The L2 blocks to add.
62
+ * @param blocks - The proposed L2 blocks to add.
55
63
  * @param pendingChainValidationStatus - Optional validation status to set.
56
64
  * @returns True if the operation is successful.
57
65
  */
58
- public addBlocks(blocks: L2BlockNew[], pendingChainValidationStatus?: ValidateCheckpointResult): Promise<boolean> {
59
- return this.store.transactionAsync(async () => {
60
- await this.store.addBlocks(blocks);
66
+ public async addProposedBlocks(
67
+ blocks: L2Block[],
68
+ pendingChainValidationStatus?: ValidateCheckpointResult,
69
+ ): Promise<boolean> {
70
+ const result = await this.store.transactionAsync(async () => {
71
+ await this.store.addProposedBlocks(blocks);
61
72
 
62
73
  const opResults = await Promise.all([
63
74
  // Update the pending chain validation status if provided
@@ -65,16 +76,18 @@ export class ArchiverDataStoreUpdater {
65
76
  // Add any logs emitted during the retrieved blocks
66
77
  this.store.addLogs(blocks),
67
78
  // Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
68
- ...blocks.map(block => this.addBlockDataToDB(block)),
79
+ ...blocks.map(block => this.addContractDataToDb(block)),
69
80
  ]);
70
81
 
82
+ await this.l2TipsCache?.refresh();
71
83
  return opResults.every(Boolean);
72
84
  });
85
+ return result;
73
86
  }
74
87
 
75
88
  /**
76
89
  * Reconciles local blocks with incoming checkpoints from L1.
77
- * Adds checkpoints to the store with contract class/instance extraction from logs.
90
+ * Adds new checkpoints to the store with contract class/instance extraction from logs.
78
91
  * Prunes any local blocks that conflict with checkpoint data (by comparing archive roots).
79
92
  * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
80
93
  * and individually broadcasted functions from the checkpoint block logs.
@@ -83,17 +96,21 @@ export class ArchiverDataStoreUpdater {
83
96
  * @param pendingChainValidationStatus - Optional validation status to set.
84
97
  * @returns Result with information about any pruned blocks.
85
98
  */
86
- public setNewCheckpointData(
99
+ public async addCheckpoints(
87
100
  checkpoints: PublishedCheckpoint[],
88
101
  pendingChainValidationStatus?: ValidateCheckpointResult,
89
102
  ): Promise<ReconcileCheckpointsResult> {
90
- return this.store.transactionAsync(async () => {
103
+ for (const checkpoint of checkpoints) {
104
+ validateCheckpoint(checkpoint.checkpoint, { rollupManaLimit: this.opts?.rollupManaLimit });
105
+ }
106
+
107
+ const result = await this.store.transactionAsync(async () => {
91
108
  // Before adding checkpoints, check for conflicts with local blocks if any
92
109
  const { prunedBlocks, lastAlreadyInsertedBlockNumber } = await this.pruneMismatchingLocalBlocks(checkpoints);
93
110
 
94
111
  await this.store.addCheckpoints(checkpoints);
95
112
 
96
- // Filter out blocks that were already inserted via addBlocks() to avoid duplicating logs/contract data
113
+ // Filter out blocks that were already inserted via addProposedBlocks() to avoid duplicating logs/contract data
97
114
  const newBlocks = checkpoints
98
115
  .flatMap((ch: PublishedCheckpoint) => ch.checkpoint.blocks)
99
116
  .filter(b => lastAlreadyInsertedBlockNumber === undefined || b.number > lastAlreadyInsertedBlockNumber);
@@ -104,11 +121,13 @@ export class ArchiverDataStoreUpdater {
104
121
  // Add any logs emitted during the retrieved blocks
105
122
  this.store.addLogs(newBlocks),
106
123
  // Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
107
- ...newBlocks.map(block => this.addBlockDataToDB(block)),
124
+ ...newBlocks.map(block => this.addContractDataToDb(block)),
108
125
  ]);
109
126
 
127
+ await this.l2TipsCache?.refresh();
110
128
  return { prunedBlocks, lastAlreadyInsertedBlockNumber };
111
129
  });
130
+ return result;
112
131
  }
113
132
 
114
133
  /**
@@ -161,7 +180,7 @@ export class ArchiverDataStoreUpdater {
161
180
  this.log.verbose(`Block number ${blockNumber} already inserted and matches checkpoint`, blockInfos);
162
181
  lastAlreadyInsertedBlockNumber = blockNumber;
163
182
  } else {
164
- this.log.warn(`Conflict detected at block ${blockNumber} between checkpointed and local block`, blockInfos);
183
+ this.log.info(`Conflict detected at block ${blockNumber} between checkpointed and local block`, blockInfos);
165
184
  const prunedBlocks = await this.removeBlocksAfter(BlockNumber(blockNumber - 1));
166
185
  return { prunedBlocks, lastAlreadyInsertedBlockNumber };
167
186
  }
@@ -185,80 +204,108 @@ export class ArchiverDataStoreUpdater {
185
204
  }
186
205
 
187
206
  /**
188
- * Removes all blocks strictly after the specified block number and cleans up associated contract data.
207
+ * Removes all uncheckpointed blocks strictly after the specified block number and cleans up associated contract data.
189
208
  * This handles removal of provisionally added blocks along with their contract classes/instances.
209
+ * Verifies that each block being removed is not part of a stored checkpoint.
190
210
  *
191
211
  * @param blockNumber - Remove all blocks with number greater than this.
192
212
  * @returns The removed blocks.
213
+ * @throws Error if any block to be removed is checkpointed.
193
214
  */
194
- public removeBlocksAfter(blockNumber: BlockNumber): Promise<L2BlockNew[]> {
195
- return this.store.transactionAsync(async () => {
196
- // First get the blocks to be removed so we can clean up contract data
197
- const removedBlocks = await this.store.removeBlocksAfter(blockNumber);
198
-
199
- // Clean up contract data and logs for the removed blocks
200
- await Promise.all([
201
- this.store.deleteLogs(removedBlocks),
202
- ...removedBlocks.map(block => this.removeBlockDataFromDB(block)),
203
- ]);
215
+ public async removeUncheckpointedBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
216
+ const result = await this.store.transactionAsync(async () => {
217
+ // Verify we're only removing uncheckpointed blocks
218
+ const lastCheckpointedBlockNumber = await this.store.getCheckpointedL2BlockNumber();
219
+ if (blockNumber < lastCheckpointedBlockNumber) {
220
+ throw new Error(
221
+ `Cannot remove blocks after ${blockNumber} because checkpointed blocks exist up to ${lastCheckpointedBlockNumber}`,
222
+ );
223
+ }
204
224
 
205
- return removedBlocks;
225
+ const result = await this.removeBlocksAfter(blockNumber);
226
+ await this.l2TipsCache?.refresh();
227
+ return result;
206
228
  });
229
+ return result;
207
230
  }
208
231
 
209
232
  /**
210
- * Unwinds checkpoints from the store with reverse contract extraction.
233
+ * Removes all blocks strictly after the given block number along with any logs and contract data.
234
+ * Does not remove their checkpoints.
235
+ */
236
+ private async removeBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
237
+ // First get the blocks to be removed so we can clean up contract data
238
+ const removedBlocks = await this.store.removeBlocksAfter(blockNumber);
239
+
240
+ // Clean up contract data and logs for the removed blocks
241
+ await Promise.all([
242
+ this.store.deleteLogs(removedBlocks),
243
+ ...removedBlocks.map(block => this.removeContractDataFromDb(block)),
244
+ ]);
245
+
246
+ return removedBlocks;
247
+ }
248
+
249
+ /**
250
+ * Removes all checkpoints after the given checkpoint number.
211
251
  * Deletes ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated data
212
- * that was stored for the unwound checkpoints.
252
+ * that was stored for the removed checkpoints. Also removes ALL blocks (both checkpointed
253
+ * and uncheckpointed) after the last block of the given checkpoint.
213
254
  *
214
- * @param from - The checkpoint number to unwind from (must be the current tip).
215
- * @param checkpointsToUnwind - The number of checkpoints to unwind.
255
+ * @param checkpointNumber - Remove all checkpoints strictly after this one.
216
256
  * @returns True if the operation is successful.
217
257
  */
218
- public async unwindCheckpoints(from: CheckpointNumber, checkpointsToUnwind: number): Promise<boolean> {
219
- if (checkpointsToUnwind <= 0) {
220
- throw new Error(`Cannot unwind ${checkpointsToUnwind} blocks`);
221
- }
258
+ public async removeCheckpointsAfter(checkpointNumber: CheckpointNumber): Promise<boolean> {
259
+ return await this.store.transactionAsync(async () => {
260
+ const { blocksRemoved = [] } = await this.store.removeCheckpointsAfter(checkpointNumber);
222
261
 
223
- const last = await this.store.getSynchedCheckpointNumber();
224
- if (from != last) {
225
- throw new Error(`Cannot unwind checkpoints from checkpoint ${from} when the last checkpoint is ${last}`);
226
- }
262
+ const opResults = await Promise.all([
263
+ // Prune rolls back to the last proven block, which is by definition valid
264
+ this.store.setPendingChainValidationStatus({ valid: true }),
265
+ // Remove contract data for all blocks being removed
266
+ ...blocksRemoved.map(block => this.removeContractDataFromDb(block)),
267
+ this.store.deleteLogs(blocksRemoved),
268
+ ]);
227
269
 
228
- const blocks = [];
229
- const lastCheckpointNumber = from + checkpointsToUnwind - 1;
230
- for (let checkpointNumber = from; checkpointNumber <= lastCheckpointNumber; checkpointNumber++) {
231
- const blocksForCheckpoint = await this.store.getBlocksForCheckpoint(checkpointNumber);
232
- if (!blocksForCheckpoint) {
233
- continue;
234
- }
235
- blocks.push(...blocksForCheckpoint);
236
- }
270
+ await this.l2TipsCache?.refresh();
271
+ return opResults.every(Boolean);
272
+ });
273
+ }
237
274
 
238
- const opResults = await Promise.all([
239
- // Prune rolls back to the last proven block, which is by definition valid
240
- this.store.setPendingChainValidationStatus({ valid: true }),
241
- // Remove contract data for all blocks being unwound
242
- ...blocks.map(block => this.removeBlockDataFromDB(block)),
243
- this.store.deleteLogs(blocks),
244
- this.store.unwindCheckpoints(from, checkpointsToUnwind),
245
- ]);
275
+ /**
276
+ * Updates the proven checkpoint number and refreshes the L2 tips cache.
277
+ * @param checkpointNumber - The checkpoint number to set as proven.
278
+ */
279
+ public async setProvenCheckpointNumber(checkpointNumber: CheckpointNumber): Promise<void> {
280
+ await this.store.transactionAsync(async () => {
281
+ await this.store.setProvenCheckpointNumber(checkpointNumber);
282
+ await this.l2TipsCache?.refresh();
283
+ });
284
+ }
246
285
 
247
- return opResults.every(Boolean);
286
+ /**
287
+ * Updates the finalized checkpoint number and refreshes the L2 tips cache.
288
+ * @param checkpointNumber - The checkpoint number to set as finalized.
289
+ */
290
+ public async setFinalizedCheckpointNumber(checkpointNumber: CheckpointNumber): Promise<void> {
291
+ await this.store.transactionAsync(async () => {
292
+ await this.store.setFinalizedCheckpointNumber(checkpointNumber);
293
+ await this.l2TipsCache?.refresh();
294
+ });
248
295
  }
249
296
 
250
297
  /** Extracts and stores contract data from a single block. */
251
- private addBlockDataToDB(block: L2BlockNew): Promise<boolean> {
252
- return this.editContractBlockData(block, Operation.Store);
298
+ private addContractDataToDb(block: L2Block): Promise<boolean> {
299
+ return this.updateContractDataOnDb(block, Operation.Store);
253
300
  }
254
301
 
255
302
  /** Removes contract data associated with a block. */
256
- private removeBlockDataFromDB(block: L2BlockNew): Promise<boolean> {
257
- return this.editContractBlockData(block, Operation.Delete);
303
+ private removeContractDataFromDb(block: L2Block): Promise<boolean> {
304
+ return this.updateContractDataOnDb(block, Operation.Delete);
258
305
  }
259
306
 
260
307
  /** Adds or remove contract data associated with a block. */
261
- private async editContractBlockData(block: L2BlockNew, operation: Operation): Promise<boolean> {
308
+ private async updateContractDataOnDb(block: L2Block, operation: Operation): Promise<boolean> {
262
309
  const contractClassLogs = block.body.txEffects.flatMap(txEffect => txEffect.contractClassLogs);
263
310
  const privateLogs = block.body.txEffects.flatMap(txEffect => txEffect.privateLogs);
264
311
  const publicLogs = block.body.txEffects.flatMap(txEffect => txEffect.publicLogs);
@@ -311,10 +358,27 @@ export class ArchiverDataStoreUpdater {
311
358
  blockNum: BlockNumber,
312
359
  operation: Operation,
313
360
  ): Promise<boolean> {
314
- const contractInstances = allLogs
361
+ const allInstances = allLogs
315
362
  .filter(log => ContractInstancePublishedEvent.isContractInstancePublishedEvent(log))
316
363
  .map(log => ContractInstancePublishedEvent.fromLog(log))
317
364
  .map(e => e.toContractInstance());
365
+
366
+ // Verify that each instance's address matches the one derived from its fields if we're adding
367
+ const contractInstances =
368
+ operation === Operation.Delete
369
+ ? allInstances
370
+ : await filterAsync(allInstances, async instance => {
371
+ const computedAddress = await computeContractAddressFromInstance(instance);
372
+ if (!computedAddress.equals(instance.address)) {
373
+ this.log.warn(
374
+ `Found contract instance with mismatched address at block ${blockNum}. Claimed ${instance.address} but computed ${computedAddress}.`,
375
+ { instanceAddress: instance.address.toString(), computedAddress: computedAddress.toString(), blockNum },
376
+ );
377
+ return false;
378
+ }
379
+ return true;
380
+ });
381
+
318
382
  if (contractInstances.length > 0) {
319
383
  contractInstances.forEach(c =>
320
384
  this.log.verbose(`${Operation[operation]} contract instance at ${c.address.toString()}`),
@@ -412,7 +476,7 @@ export class ArchiverDataStoreUpdater {
412
476
  if (validFnCount > 0) {
413
477
  this.log.verbose(`Storing ${validFnCount} functions for contract class ${contractClassId.toString()}`);
414
478
  }
415
- return await this.store.addFunctions(contractClassId, validPrivateFns, validUtilityFns);
479
+ await this.store.addFunctions(contractClassId, validPrivateFns, validUtilityFns);
416
480
  }
417
481
  return true;
418
482
  }
@@ -1,5 +1,6 @@
1
1
  import { createLogger } from '@aztec/foundation/log';
2
- import type { L2BlockNew } from '@aztec/stdlib/block';
2
+ import type { L2Block } from '@aztec/stdlib/block';
3
+ import type { CheckpointData } from '@aztec/stdlib/checkpoint';
3
4
  import {
4
5
  Attributes,
5
6
  type Gauge,
@@ -10,12 +11,14 @@ import {
10
11
  type TelemetryClient,
11
12
  type Tracer,
12
13
  type UpDownCounter,
14
+ createUpDownCounterWithDefault,
13
15
  } from '@aztec/telemetry-client';
14
16
 
15
17
  export class ArchiverInstrumentation {
16
18
  public readonly tracer: Tracer;
17
19
 
18
20
  private blockHeight: Gauge;
21
+ private checkpointHeight: Gauge;
19
22
  private txCount: UpDownCounter;
20
23
  private l1BlockHeight: Gauge;
21
24
  private proofsSubmittedDelay: Histogram;
@@ -46,17 +49,21 @@ export class ArchiverInstrumentation {
46
49
 
47
50
  this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT);
48
51
 
52
+ this.checkpointHeight = meter.createGauge(Metrics.ARCHIVER_CHECKPOINT_HEIGHT);
53
+
49
54
  this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT);
50
55
 
51
- this.txCount = meter.createUpDownCounter(Metrics.ARCHIVER_TOTAL_TXS);
56
+ this.txCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_TOTAL_TXS);
52
57
 
53
- this.proofsSubmittedCount = meter.createUpDownCounter(Metrics.ARCHIVER_ROLLUP_PROOF_COUNT);
58
+ this.proofsSubmittedCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_ROLLUP_PROOF_COUNT, {
59
+ [Attributes.PROOF_TIMED_OUT]: [true, false],
60
+ });
54
61
 
55
62
  this.proofsSubmittedDelay = meter.createHistogram(Metrics.ARCHIVER_ROLLUP_PROOF_DELAY);
56
63
 
57
64
  this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK);
58
65
 
59
- this.syncBlockCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
66
+ this.syncBlockCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
60
67
 
61
68
  this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK);
62
69
 
@@ -64,13 +71,19 @@ export class ArchiverInstrumentation {
64
71
 
65
72
  this.syncDurationPerMessage = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_MESSAGE);
66
73
 
67
- this.syncMessageCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_MESSAGE_COUNT);
74
+ this.syncMessageCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_MESSAGE_COUNT);
68
75
 
69
76
  this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION);
70
77
 
71
- this.pruneCount = meter.createUpDownCounter(Metrics.ARCHIVER_PRUNE_COUNT);
78
+ this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT);
72
79
 
73
- this.blockProposalTxTargetCount = meter.createUpDownCounter(Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT);
80
+ this.blockProposalTxTargetCount = createUpDownCounterWithDefault(
81
+ meter,
82
+ Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT,
83
+ {
84
+ [Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: [true, false],
85
+ },
86
+ );
74
87
 
75
88
  this.dbMetrics = new LmdbMetrics(
76
89
  meter,
@@ -84,10 +97,6 @@ export class ArchiverInstrumentation {
84
97
  public static async new(telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
85
98
  const instance = new ArchiverInstrumentation(telemetry, lmdbStats);
86
99
 
87
- instance.syncBlockCount.add(0);
88
- instance.syncMessageCount.add(0);
89
- instance.pruneCount.add(0);
90
-
91
100
  await instance.telemetry.flush();
92
101
 
93
102
  return instance;
@@ -97,9 +106,10 @@ export class ArchiverInstrumentation {
97
106
  return this.telemetry.isEnabled();
98
107
  }
99
108
 
100
- public processNewBlocks(syncTimePerBlock: number, blocks: L2BlockNew[]) {
109
+ public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) {
101
110
  this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock));
102
111
  this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
112
+ this.checkpointHeight.record(Math.max(...blocks.map(b => b.checkpointNumber)));
103
113
  this.syncBlockCount.add(blocks.length);
104
114
 
105
115
  for (const block of blocks) {
@@ -122,8 +132,10 @@ export class ArchiverInstrumentation {
122
132
  this.pruneDuration.record(Math.ceil(duration));
123
133
  }
124
134
 
125
- public updateLastProvenBlock(blockNumber: number) {
126
- this.blockHeight.record(blockNumber, { [Attributes.STATUS]: 'proven' });
135
+ public updateLastProvenCheckpoint(checkpoint: CheckpointData) {
136
+ const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.blockCount - 1;
137
+ this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' });
138
+ this.checkpointHeight.record(checkpoint.checkpointNumber, { [Attributes.STATUS]: 'proven' });
127
139
  }
128
140
 
129
141
  public processProofsVerified(logs: { proverId: string; l2BlockNumber: bigint; delay: bigint }[]) {
@@ -1,7 +1,6 @@
1
1
  import type { BlobClientInterface } from '@aztec/blob-client/client';
2
2
  import { EpochCache } from '@aztec/epoch-cache';
3
3
  import { InboxContract, RollupContract } from '@aztec/ethereum/contracts';
4
- import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
5
4
  import type { L1BlockId } from '@aztec/ethereum/l1-types';
6
5
  import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
7
6
  import { maxBigint } from '@aztec/foundation/bigint';
@@ -9,14 +8,13 @@ import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/br
9
8
  import { Buffer32 } from '@aztec/foundation/buffer';
10
9
  import { pick } from '@aztec/foundation/collection';
11
10
  import { Fr } from '@aztec/foundation/curves/bn254';
12
- import { EthAddress } from '@aztec/foundation/eth-address';
13
11
  import { type Logger, createLogger } from '@aztec/foundation/log';
14
12
  import { count } from '@aztec/foundation/string';
15
13
  import { DateProvider, Timer, elapsed } from '@aztec/foundation/timer';
16
14
  import { isDefined } from '@aztec/foundation/types';
17
15
  import { type ArchiverEmitter, L2BlockSourceEvents, type ValidateCheckpointResult } from '@aztec/stdlib/block';
18
16
  import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
19
- import { type L1RollupConstants, getEpochAtSlot, getSlotAtTimestamp } from '@aztec/stdlib/epoch-helpers';
17
+ import { type L1RollupConstants, getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers';
20
18
  import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
21
19
  import { type Traceable, type Tracer, execInSpan, trackSpan } from '@aztec/telemetry-client';
22
20
 
@@ -28,6 +26,7 @@ import {
28
26
  retrievedToPublishedCheckpoint,
29
27
  } from '../l1/data_retrieval.js';
30
28
  import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
29
+ import type { L2TipsCache } from '../store/l2_tips_cache.js';
31
30
  import type { InboxMessage } from '../structs/inbox_message.js';
32
31
  import { ArchiverDataStoreUpdater } from './data_store_updater.js';
33
32
  import type { ArchiverInstrumentation } from './instrumentation.js';
@@ -60,10 +59,6 @@ export class ArchiverL1Synchronizer implements Traceable {
60
59
  private readonly debugClient: ViemPublicDebugClient,
61
60
  private readonly rollup: RollupContract,
62
61
  private readonly inbox: InboxContract,
63
- private readonly l1Addresses: Pick<
64
- L1ContractAddresses,
65
- 'registryAddress' | 'governanceProposerAddress' | 'slashFactoryAddress'
66
- > & { slashingProposerAddress: EthAddress },
67
62
  private readonly store: KVArchiverDataStore,
68
63
  private config: {
69
64
  batchSize: number;
@@ -74,12 +69,18 @@ export class ArchiverL1Synchronizer implements Traceable {
74
69
  private readonly epochCache: EpochCache,
75
70
  private readonly dateProvider: DateProvider,
76
71
  private readonly instrumentation: ArchiverInstrumentation,
77
- private readonly l1Constants: L1RollupConstants & { l1StartBlockHash: Buffer32; genesisArchiveRoot: Fr },
72
+ private readonly l1Constants: L1RollupConstants & {
73
+ l1StartBlockHash: Buffer32;
74
+ genesisArchiveRoot: Fr;
75
+ },
78
76
  private readonly events: ArchiverEmitter,
79
77
  tracer: Tracer,
78
+ l2TipsCache?: L2TipsCache,
80
79
  private readonly log: Logger = createLogger('archiver:l1-sync'),
81
80
  ) {
82
- this.updater = new ArchiverDataStoreUpdater(this.store);
81
+ this.updater = new ArchiverDataStoreUpdater(this.store, l2TipsCache, {
82
+ rollupManaLimit: l1Constants.rollupManaLimit,
83
+ });
83
84
  this.tracer = tracer;
84
85
  }
85
86
 
@@ -215,6 +216,9 @@ export class ArchiverL1Synchronizer implements Traceable {
215
216
  this.instrumentation.updateL1BlockHeight(currentL1BlockNumber);
216
217
  }
217
218
 
219
+ // Update the finalized L2 checkpoint based on L1 finality.
220
+ await this.updateFinalizedCheckpoint();
221
+
218
222
  // After syncing has completed, update the current l1 block number and timestamp,
219
223
  // otherwise we risk announcing to the world that we've synced to a given point,
220
224
  // but the corresponding blocks have not been processed (see #12631).
@@ -230,6 +234,33 @@ export class ArchiverL1Synchronizer implements Traceable {
230
234
  });
231
235
  }
232
236
 
237
+ /** Query L1 for its finalized block and update the finalized checkpoint accordingly. */
238
+ private async updateFinalizedCheckpoint(): Promise<void> {
239
+ try {
240
+ const finalizedL1Block = await this.publicClient.getBlock({ blockTag: 'finalized', includeTransactions: false });
241
+ const finalizedL1BlockNumber = finalizedL1Block.number;
242
+ const finalizedCheckpointNumber = await this.rollup.getProvenCheckpointNumber({
243
+ blockNumber: finalizedL1BlockNumber,
244
+ });
245
+ const localFinalizedCheckpointNumber = await this.store.getFinalizedCheckpointNumber();
246
+ if (localFinalizedCheckpointNumber !== finalizedCheckpointNumber) {
247
+ await this.updater.setFinalizedCheckpointNumber(finalizedCheckpointNumber);
248
+ const finalizedL2BlockNumber = await this.store.getFinalizedL2BlockNumber();
249
+ this.log.info(
250
+ `Updated finalized chain to checkpoint ${finalizedCheckpointNumber} (L2 block ${finalizedL2BlockNumber})`,
251
+ {
252
+ finalizedCheckpointNumber,
253
+ previousFinalizedCheckpointNumber: localFinalizedCheckpointNumber,
254
+ finalizedL2BlockNumber,
255
+ finalizedL1BlockNumber,
256
+ },
257
+ );
258
+ }
259
+ } catch (err) {
260
+ this.log.warn(`Failed to update finalized checkpoint: ${err}`);
261
+ }
262
+ }
263
+
233
264
  /** Prune all proposed local blocks that should have been checkpointed by now. */
234
265
  private async pruneUncheckpointedBlocks(currentL1Timestamp: bigint) {
235
266
  const [lastCheckpointedBlockNumber, lastProposedBlockNumber] = await Promise.all([
@@ -249,8 +280,7 @@ export class ArchiverL1Synchronizer implements Traceable {
249
280
  const firstUncheckpointedBlockSlot = firstUncheckpointedBlockHeader?.getSlot();
250
281
 
251
282
  // What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
252
- const nextL1BlockTimestamp = currentL1Timestamp + BigInt(this.l1Constants.ethereumSlotDuration);
253
- const slotAtNextL1Block = getSlotAtTimestamp(nextL1BlockTimestamp, this.l1Constants);
283
+ const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
254
284
 
255
285
  // Prune provisional blocks from slots that have ended without being checkpointed
256
286
  if (firstUncheckpointedBlockSlot !== undefined && firstUncheckpointedBlockSlot < slotAtNextL1Block) {
@@ -258,7 +288,7 @@ export class ArchiverL1Synchronizer implements Traceable {
258
288
  `Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`,
259
289
  { firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(), slotAtNextL1Block },
260
290
  );
261
- const prunedBlocks = await this.updater.removeBlocksAfter(lastCheckpointedBlockNumber);
291
+ const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
262
292
 
263
293
  if (prunedBlocks.length > 0) {
264
294
  this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
@@ -331,10 +361,10 @@ export class ArchiverL1Synchronizer implements Traceable {
331
361
  this.log.debug(
332
362
  `L2 prune from ${provenCheckpointNumber + 1} to ${localPendingCheckpointNumber} will occur on next checkpoint submission.`,
333
363
  );
334
- await this.updater.unwindCheckpoints(localPendingCheckpointNumber, checkpointsToUnwind);
364
+ await this.updater.removeCheckpointsAfter(provenCheckpointNumber);
335
365
  this.log.warn(
336
- `Unwound ${count(checkpointsToUnwind, 'checkpoint')} from checkpoint ${localPendingCheckpointNumber} ` +
337
- `to ${provenCheckpointNumber} due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
366
+ `Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` +
367
+ `due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
338
368
  `Updated latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
339
369
  );
340
370
  this.instrumentation.processPrune(timer.ms());
@@ -551,7 +581,7 @@ export class ArchiverL1Synchronizer implements Traceable {
551
581
  if (provenCheckpointNumber === 0) {
552
582
  const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
553
583
  if (localProvenCheckpointNumber !== provenCheckpointNumber) {
554
- await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
584
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
555
585
  this.log.info(`Rolled back proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
556
586
  }
557
587
  }
@@ -583,13 +613,13 @@ export class ArchiverL1Synchronizer implements Traceable {
583
613
  ) {
584
614
  const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
585
615
  if (localProvenCheckpointNumber !== provenCheckpointNumber) {
586
- await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
616
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
587
617
  this.log.info(`Updated proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
588
618
  const provenSlotNumber = localCheckpointForDestinationProvenCheckpointNumber.header.slotNumber;
589
619
  const provenEpochNumber: EpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants);
590
620
  const lastBlockNumberInCheckpoint =
591
621
  localCheckpointForDestinationProvenCheckpointNumber.startBlock +
592
- localCheckpointForDestinationProvenCheckpointNumber.numBlocks -
622
+ localCheckpointForDestinationProvenCheckpointNumber.blockCount -
593
623
  1;
594
624
 
595
625
  this.events.emit(L2BlockSourceEvents.L2BlockProven, {
@@ -598,7 +628,7 @@ export class ArchiverL1Synchronizer implements Traceable {
598
628
  slotNumber: provenSlotNumber,
599
629
  epochNumber: provenEpochNumber,
600
630
  });
601
- this.instrumentation.updateLastProvenBlock(lastBlockNumberInCheckpoint);
631
+ this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber);
602
632
  } else {
603
633
  this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`);
604
634
  }
@@ -675,11 +705,11 @@ export class ArchiverL1Synchronizer implements Traceable {
675
705
  tipAfterUnwind--;
676
706
  }
677
707
 
678
- const checkpointsToUnwind = localPendingCheckpointNumber - tipAfterUnwind;
679
- await this.updater.unwindCheckpoints(localPendingCheckpointNumber, checkpointsToUnwind);
708
+ const checkpointsToRemove = localPendingCheckpointNumber - tipAfterUnwind;
709
+ await this.updater.removeCheckpointsAfter(CheckpointNumber(tipAfterUnwind));
680
710
 
681
711
  this.log.warn(
682
- `Unwound ${count(checkpointsToUnwind, 'checkpoint')} from checkpoint ${localPendingCheckpointNumber} ` +
712
+ `Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` +
683
713
  `due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` +
684
714
  `Updated L2 latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
685
715
  );
@@ -707,7 +737,6 @@ export class ArchiverL1Synchronizer implements Traceable {
707
737
  this.blobClient,
708
738
  searchStartBlock, // TODO(palla/reorg): If the L2 reorg was due to an L1 reorg, we need to start search earlier
709
739
  searchEndBlock,
710
- this.l1Addresses,
711
740
  this.instrumentation,
712
741
  this.log,
713
742
  !initialSyncComplete, // isHistoricalSync
@@ -806,8 +835,8 @@ export class ArchiverL1Synchronizer implements Traceable {
806
835
  const updatedValidationResult =
807
836
  rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
808
837
  const [processDuration, result] = await elapsed(() =>
809
- execInSpan(this.tracer, 'Archiver.setCheckpointData', () =>
810
- this.updater.setNewCheckpointData(validCheckpoints, updatedValidationResult),
838
+ execInSpan(this.tracer, 'Archiver.addCheckpoints', () =>
839
+ this.updater.addCheckpoints(validCheckpoints, updatedValidationResult),
811
840
  ),
812
841
  );
813
842
  this.instrumentation.processNewBlocks(
@@ -820,7 +849,7 @@ export class ArchiverL1Synchronizer implements Traceable {
820
849
  const prunedCheckpointNumber = result.prunedBlocks[0].checkpointNumber;
821
850
  const prunedSlotNumber = result.prunedBlocks[0].header.globalVariables.slotNumber;
822
851
 
823
- this.log.warn(
852
+ this.log.info(
824
853
  `Pruned ${result.prunedBlocks.length} mismatching blocks for checkpoint ${prunedCheckpointNumber}`,
825
854
  { prunedBlocks: result.prunedBlocks.map(b => b.toBlockInfo()), prunedSlotNumber, prunedCheckpointNumber },
826
855
  );