@aztec/archiver 0.0.1-commit.1142ef1 → 0.0.1-commit.11bf3dd6e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/README.md +162 -22
  2. package/dest/archiver.d.ts +140 -0
  3. package/dest/archiver.d.ts.map +1 -0
  4. package/dest/archiver.js +750 -0
  5. package/dest/config.d.ts +30 -0
  6. package/dest/config.d.ts.map +1 -0
  7. package/dest/{archiver/config.js → config.js} +12 -2
  8. package/dest/errors.d.ts +65 -0
  9. package/dest/errors.d.ts.map +1 -0
  10. package/dest/errors.js +91 -0
  11. package/dest/factory.d.ts +8 -7
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +96 -13
  14. package/dest/index.d.ts +11 -4
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +9 -3
  17. package/dest/interfaces.d.ts +9 -0
  18. package/dest/interfaces.d.ts.map +1 -0
  19. package/dest/interfaces.js +3 -0
  20. package/dest/{archiver/l1 → l1}/bin/retrieve-calldata.d.ts +1 -1
  21. package/dest/l1/bin/retrieve-calldata.d.ts.map +1 -0
  22. package/dest/{archiver/l1 → l1}/bin/retrieve-calldata.js +35 -32
  23. package/dest/l1/calldata_retriever.d.ts +135 -0
  24. package/dest/l1/calldata_retriever.d.ts.map +1 -0
  25. package/dest/l1/calldata_retriever.js +403 -0
  26. package/dest/l1/data_retrieval.d.ts +88 -0
  27. package/dest/l1/data_retrieval.d.ts.map +1 -0
  28. package/dest/{archiver/l1 → l1}/data_retrieval.js +59 -77
  29. package/dest/{archiver/l1 → l1}/debug_tx.d.ts +1 -1
  30. package/dest/l1/debug_tx.d.ts.map +1 -0
  31. package/dest/{archiver/l1 → l1}/spire_proposer.d.ts +5 -5
  32. package/dest/l1/spire_proposer.d.ts.map +1 -0
  33. package/dest/{archiver/l1 → l1}/spire_proposer.js +9 -17
  34. package/dest/{archiver/l1 → l1}/trace_tx.d.ts +1 -1
  35. package/dest/l1/trace_tx.d.ts.map +1 -0
  36. package/dest/l1/types.d.ts +12 -0
  37. package/dest/l1/types.d.ts.map +1 -0
  38. package/dest/{archiver/l1 → l1}/validate_trace.d.ts +6 -3
  39. package/dest/l1/validate_trace.d.ts.map +1 -0
  40. package/dest/{archiver/l1 → l1}/validate_trace.js +13 -9
  41. package/dest/modules/data_source_base.d.ts +91 -0
  42. package/dest/modules/data_source_base.d.ts.map +1 -0
  43. package/dest/modules/data_source_base.js +222 -0
  44. package/dest/modules/data_store_updater.d.ts +86 -0
  45. package/dest/modules/data_store_updater.d.ts.map +1 -0
  46. package/dest/modules/data_store_updater.js +333 -0
  47. package/dest/modules/instrumentation.d.ts +52 -0
  48. package/dest/modules/instrumentation.d.ts.map +1 -0
  49. package/dest/{archiver → modules}/instrumentation.js +53 -18
  50. package/dest/modules/l1_synchronizer.d.ts +73 -0
  51. package/dest/modules/l1_synchronizer.d.ts.map +1 -0
  52. package/dest/modules/l1_synchronizer.js +1154 -0
  53. package/dest/{archiver → modules}/validation.d.ts +1 -1
  54. package/dest/modules/validation.d.ts.map +1 -0
  55. package/dest/{archiver → modules}/validation.js +8 -2
  56. package/dest/store/block_store.d.ts +230 -0
  57. package/dest/store/block_store.d.ts.map +1 -0
  58. package/dest/store/block_store.js +942 -0
  59. package/dest/store/contract_class_store.d.ts +17 -0
  60. package/dest/store/contract_class_store.d.ts.map +1 -0
  61. package/dest/store/contract_class_store.js +64 -0
  62. package/dest/store/contract_instance_store.d.ts +24 -0
  63. package/dest/store/contract_instance_store.d.ts.map +1 -0
  64. package/dest/{archiver/kv_archiver_store → store}/contract_instance_store.js +7 -3
  65. package/dest/store/kv_archiver_store.d.ts +377 -0
  66. package/dest/store/kv_archiver_store.d.ts.map +1 -0
  67. package/dest/store/kv_archiver_store.js +494 -0
  68. package/dest/store/l2_tips_cache.d.ts +20 -0
  69. package/dest/store/l2_tips_cache.d.ts.map +1 -0
  70. package/dest/store/l2_tips_cache.js +109 -0
  71. package/dest/store/log_store.d.ts +57 -0
  72. package/dest/store/log_store.d.ts.map +1 -0
  73. package/dest/store/log_store.js +531 -0
  74. package/dest/store/message_store.d.ts +44 -0
  75. package/dest/store/message_store.d.ts.map +1 -0
  76. package/dest/{archiver/kv_archiver_store → store}/message_store.js +21 -9
  77. package/dest/{archiver/structs → structs}/data_retrieval.d.ts +1 -1
  78. package/dest/structs/data_retrieval.d.ts.map +1 -0
  79. package/dest/structs/inbox_message.d.ts +15 -0
  80. package/dest/structs/inbox_message.d.ts.map +1 -0
  81. package/dest/{archiver/structs → structs}/published.d.ts +1 -1
  82. package/dest/structs/published.d.ts.map +1 -0
  83. package/dest/test/fake_l1_state.d.ts +210 -0
  84. package/dest/test/fake_l1_state.d.ts.map +1 -0
  85. package/dest/test/fake_l1_state.js +490 -0
  86. package/dest/test/index.d.ts +2 -1
  87. package/dest/test/index.d.ts.map +1 -1
  88. package/dest/test/index.js +4 -1
  89. package/dest/test/mock_archiver.d.ts +2 -2
  90. package/dest/test/mock_archiver.d.ts.map +1 -1
  91. package/dest/test/mock_archiver.js +3 -3
  92. package/dest/test/mock_l1_to_l2_message_source.d.ts +1 -1
  93. package/dest/test/mock_l1_to_l2_message_source.d.ts.map +1 -1
  94. package/dest/test/mock_l1_to_l2_message_source.js +2 -1
  95. package/dest/test/mock_l2_block_source.d.ts +43 -19
  96. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  97. package/dest/test/mock_l2_block_source.js +211 -80
  98. package/dest/test/mock_structs.d.ts +81 -3
  99. package/dest/test/mock_structs.d.ts.map +1 -1
  100. package/dest/test/mock_structs.js +152 -7
  101. package/dest/test/noop_l1_archiver.d.ts +26 -0
  102. package/dest/test/noop_l1_archiver.d.ts.map +1 -0
  103. package/dest/test/noop_l1_archiver.js +71 -0
  104. package/package.json +16 -17
  105. package/src/archiver.ts +510 -0
  106. package/src/{archiver/config.ts → config.ts} +20 -2
  107. package/src/errors.ts +146 -0
  108. package/src/factory.ts +152 -16
  109. package/src/index.ts +11 -3
  110. package/src/interfaces.ts +9 -0
  111. package/src/l1/README.md +55 -0
  112. package/src/{archiver/l1 → l1}/bin/retrieve-calldata.ts +45 -33
  113. package/src/l1/calldata_retriever.ts +512 -0
  114. package/src/{archiver/l1 → l1}/data_retrieval.ts +79 -102
  115. package/src/{archiver/l1 → l1}/spire_proposer.ts +7 -15
  116. package/src/{archiver/l1 → l1}/validate_trace.ts +24 -6
  117. package/src/modules/data_source_base.ts +347 -0
  118. package/src/modules/data_store_updater.ts +437 -0
  119. package/src/{archiver → modules}/instrumentation.ts +64 -20
  120. package/src/modules/l1_synchronizer.ts +961 -0
  121. package/src/{archiver → modules}/validation.ts +7 -2
  122. package/src/store/block_store.ts +1245 -0
  123. package/src/store/contract_class_store.ts +82 -0
  124. package/src/{archiver/kv_archiver_store → store}/contract_instance_store.ts +9 -6
  125. package/src/store/kv_archiver_store.ts +713 -0
  126. package/src/store/l2_tips_cache.ts +134 -0
  127. package/src/store/log_store.ts +733 -0
  128. package/src/{archiver/kv_archiver_store → store}/message_store.ts +27 -10
  129. package/src/{archiver/structs → structs}/inbox_message.ts +1 -1
  130. package/src/test/fake_l1_state.ts +747 -0
  131. package/src/test/index.ts +4 -0
  132. package/src/test/mock_archiver.ts +4 -3
  133. package/src/test/mock_l1_to_l2_message_source.ts +1 -0
  134. package/src/test/mock_l2_block_source.ts +269 -94
  135. package/src/test/mock_structs.ts +283 -8
  136. package/src/test/noop_l1_archiver.ts +114 -0
  137. package/dest/archiver/archiver.d.ts +0 -307
  138. package/dest/archiver/archiver.d.ts.map +0 -1
  139. package/dest/archiver/archiver.js +0 -2102
  140. package/dest/archiver/archiver_store.d.ts +0 -315
  141. package/dest/archiver/archiver_store.d.ts.map +0 -1
  142. package/dest/archiver/archiver_store.js +0 -4
  143. package/dest/archiver/archiver_store_test_suite.d.ts +0 -8
  144. package/dest/archiver/archiver_store_test_suite.d.ts.map +0 -1
  145. package/dest/archiver/archiver_store_test_suite.js +0 -2770
  146. package/dest/archiver/config.d.ts +0 -22
  147. package/dest/archiver/config.d.ts.map +0 -1
  148. package/dest/archiver/errors.d.ts +0 -36
  149. package/dest/archiver/errors.d.ts.map +0 -1
  150. package/dest/archiver/errors.js +0 -54
  151. package/dest/archiver/index.d.ts +0 -7
  152. package/dest/archiver/index.d.ts.map +0 -1
  153. package/dest/archiver/index.js +0 -4
  154. package/dest/archiver/instrumentation.d.ts +0 -37
  155. package/dest/archiver/instrumentation.d.ts.map +0 -1
  156. package/dest/archiver/kv_archiver_store/block_store.d.ts +0 -164
  157. package/dest/archiver/kv_archiver_store/block_store.d.ts.map +0 -1
  158. package/dest/archiver/kv_archiver_store/block_store.js +0 -626
  159. package/dest/archiver/kv_archiver_store/contract_class_store.d.ts +0 -18
  160. package/dest/archiver/kv_archiver_store/contract_class_store.d.ts.map +0 -1
  161. package/dest/archiver/kv_archiver_store/contract_class_store.js +0 -120
  162. package/dest/archiver/kv_archiver_store/contract_instance_store.d.ts +0 -24
  163. package/dest/archiver/kv_archiver_store/contract_instance_store.d.ts.map +0 -1
  164. package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts +0 -159
  165. package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts.map +0 -1
  166. package/dest/archiver/kv_archiver_store/kv_archiver_store.js +0 -316
  167. package/dest/archiver/kv_archiver_store/log_store.d.ts +0 -45
  168. package/dest/archiver/kv_archiver_store/log_store.d.ts.map +0 -1
  169. package/dest/archiver/kv_archiver_store/log_store.js +0 -401
  170. package/dest/archiver/kv_archiver_store/message_store.d.ts +0 -40
  171. package/dest/archiver/kv_archiver_store/message_store.d.ts.map +0 -1
  172. package/dest/archiver/l1/bin/retrieve-calldata.d.ts.map +0 -1
  173. package/dest/archiver/l1/calldata_retriever.d.ts +0 -112
  174. package/dest/archiver/l1/calldata_retriever.d.ts.map +0 -1
  175. package/dest/archiver/l1/calldata_retriever.js +0 -471
  176. package/dest/archiver/l1/data_retrieval.d.ts +0 -90
  177. package/dest/archiver/l1/data_retrieval.d.ts.map +0 -1
  178. package/dest/archiver/l1/debug_tx.d.ts.map +0 -1
  179. package/dest/archiver/l1/spire_proposer.d.ts.map +0 -1
  180. package/dest/archiver/l1/trace_tx.d.ts.map +0 -1
  181. package/dest/archiver/l1/types.d.ts +0 -12
  182. package/dest/archiver/l1/types.d.ts.map +0 -1
  183. package/dest/archiver/l1/validate_trace.d.ts.map +0 -1
  184. package/dest/archiver/structs/data_retrieval.d.ts.map +0 -1
  185. package/dest/archiver/structs/inbox_message.d.ts +0 -15
  186. package/dest/archiver/structs/inbox_message.d.ts.map +0 -1
  187. package/dest/archiver/structs/published.d.ts.map +0 -1
  188. package/dest/archiver/validation.d.ts.map +0 -1
  189. package/dest/rpc/index.d.ts +0 -9
  190. package/dest/rpc/index.d.ts.map +0 -1
  191. package/dest/rpc/index.js +0 -15
  192. package/src/archiver/archiver.ts +0 -2265
  193. package/src/archiver/archiver_store.ts +0 -380
  194. package/src/archiver/archiver_store_test_suite.ts +0 -2842
  195. package/src/archiver/errors.ts +0 -90
  196. package/src/archiver/index.ts +0 -6
  197. package/src/archiver/kv_archiver_store/block_store.ts +0 -850
  198. package/src/archiver/kv_archiver_store/contract_class_store.ts +0 -176
  199. package/src/archiver/kv_archiver_store/kv_archiver_store.ts +0 -442
  200. package/src/archiver/kv_archiver_store/log_store.ts +0 -516
  201. package/src/archiver/l1/README.md +0 -98
  202. package/src/archiver/l1/calldata_retriever.ts +0 -641
  203. package/src/rpc/index.ts +0 -16
  204. /package/dest/{archiver/l1 → l1}/debug_tx.js +0 -0
  205. /package/dest/{archiver/l1 → l1}/trace_tx.js +0 -0
  206. /package/dest/{archiver/l1 → l1}/types.js +0 -0
  207. /package/dest/{archiver/structs → structs}/data_retrieval.js +0 -0
  208. /package/dest/{archiver/structs → structs}/inbox_message.js +0 -0
  209. /package/dest/{archiver/structs → structs}/published.js +0 -0
  210. /package/src/{archiver/l1 → l1}/debug_tx.ts +0 -0
  211. /package/src/{archiver/l1 → l1}/trace_tx.ts +0 -0
  212. /package/src/{archiver/l1 → l1}/types.ts +0 -0
  213. /package/src/{archiver/structs → structs}/data_retrieval.ts +0 -0
  214. /package/src/{archiver/structs → structs}/published.ts +0 -0
@@ -0,0 +1,437 @@
1
+ import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
2
+ import { filterAsync } from '@aztec/foundation/collection';
3
+ import { createLogger } from '@aztec/foundation/log';
4
+ import { ContractClassPublishedEvent } from '@aztec/protocol-contracts/class-registry';
5
+ import {
6
+ ContractInstancePublishedEvent,
7
+ ContractInstanceUpdatedEvent,
8
+ } from '@aztec/protocol-contracts/instance-registry';
9
+ import type { L2Block, ValidateCheckpointResult } from '@aztec/stdlib/block';
10
+ import { type ProposedCheckpointInput, type PublishedCheckpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
11
+ import {
12
+ type ContractClassPublicWithCommitment,
13
+ computeContractAddressFromInstance,
14
+ computeContractClassId,
15
+ } from '@aztec/stdlib/contract';
16
+ import type { ContractClassLog, PrivateLog, PublicLog } from '@aztec/stdlib/logs';
17
+ import type { UInt64 } from '@aztec/stdlib/types';
18
+
19
+ import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
20
+ import type { L2TipsCache } from '../store/l2_tips_cache.js';
21
+
22
+ /** Operation type for contract data updates. */
23
+ enum Operation {
24
+ Store,
25
+ Delete,
26
+ }
27
+
28
+ /** Result of adding checkpoints with information about any pruned blocks. */
29
+ type ReconcileCheckpointsResult = {
30
+ /** Blocks that were pruned due to conflict with L1 checkpoints. */
31
+ prunedBlocks: L2Block[] | undefined;
32
+ /** Last block number that was already inserted locally, or undefined if none. */
33
+ lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
34
+ };
35
+
36
+ /** Archiver helper module to handle updates to the data store. */
37
+ export class ArchiverDataStoreUpdater {
38
+ private readonly log = createLogger('archiver:store_updater');
39
+
40
+ constructor(
41
+ private store: KVArchiverDataStore,
42
+ private l2TipsCache?: L2TipsCache,
43
+ private opts: { rollupManaLimit?: number } = {},
44
+ ) {}
45
+
46
+ /**
47
+ * Adds a proposed block to the store with contract class/instance extraction from logs.
48
+ * This is an uncheckpointed block that has been proposed by the sequencer but not yet included in a checkpoint on L1.
49
+ * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events from the block logs.
50
+ *
51
+ * @param block - The proposed L2 block to add.
52
+ * @param pendingChainValidationStatus - Optional validation status to set.
53
+ * @returns True if the operation is successful.
54
+ */
55
+ public async addProposedBlock(
56
+ block: L2Block,
57
+ pendingChainValidationStatus?: ValidateCheckpointResult,
58
+ ): Promise<boolean> {
59
+ const result = await this.store.transactionAsync(async () => {
60
+ await this.store.addProposedBlock(block);
61
+
62
+ const opResults = await Promise.all([
63
+ // Update the pending chain validation status if provided
64
+ pendingChainValidationStatus && this.store.setPendingChainValidationStatus(pendingChainValidationStatus),
65
+ // Add any logs emitted during the retrieved block
66
+ this.store.addLogs([block]),
67
+ // Unroll all logs emitted during the retrieved block and extract any contract classes and instances from it
68
+ this.addContractDataToDb(block),
69
+ ]);
70
+
71
+ await this.l2TipsCache?.refresh();
72
+ return opResults.every(Boolean);
73
+ });
74
+ return result;
75
+ }
76
+
77
+ /**
78
+ * Reconciles local blocks with incoming checkpoints from L1.
79
+ * Adds new checkpoints to the store with contract class/instance extraction from logs.
80
+ * Prunes any local blocks that conflict with checkpoint data (by comparing archive roots).
81
+ * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events from the checkpoint block logs.
82
+ *
83
+ * @param checkpoints - The published checkpoints to add.
84
+ * @param pendingChainValidationStatus - Optional validation status to set.
85
+ * @returns Result with information about any pruned blocks.
86
+ */
87
+ public async addCheckpoints(
88
+ checkpoints: PublishedCheckpoint[],
89
+ pendingChainValidationStatus?: ValidateCheckpointResult,
90
+ ): Promise<ReconcileCheckpointsResult> {
91
+ for (const checkpoint of checkpoints) {
92
+ validateCheckpoint(checkpoint.checkpoint, { rollupManaLimit: this.opts?.rollupManaLimit });
93
+ }
94
+
95
+ const result = await this.store.transactionAsync(async () => {
96
+ // Before adding checkpoints, check for conflicts with local blocks if any
97
+ const { prunedBlocks, lastAlreadyInsertedBlockNumber } = await this.pruneMismatchingLocalBlocks(checkpoints);
98
+
99
+ await this.store.addCheckpoints(checkpoints);
100
+
101
+ // Filter out blocks that were already inserted via addProposedBlock() to avoid duplicating logs/contract data
102
+ const newBlocks = checkpoints
103
+ .flatMap((ch: PublishedCheckpoint) => ch.checkpoint.blocks)
104
+ .filter(b => lastAlreadyInsertedBlockNumber === undefined || b.number > lastAlreadyInsertedBlockNumber);
105
+
106
+ await Promise.all([
107
+ // Update the pending chain validation status if provided
108
+ pendingChainValidationStatus && this.store.setPendingChainValidationStatus(pendingChainValidationStatus),
109
+ // Add any logs emitted during the retrieved blocks
110
+ this.store.addLogs(newBlocks),
111
+ // Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
112
+ ...newBlocks.map(block => this.addContractDataToDb(block)),
113
+ ]);
114
+
115
+ await this.l2TipsCache?.refresh();
116
+ return { prunedBlocks, lastAlreadyInsertedBlockNumber };
117
+ });
118
+ return result;
119
+ }
120
+
121
+ public async setProposedCheckpoint(proposedCheckpoint: ProposedCheckpointInput) {
122
+ const result = await this.store.transactionAsync(async () => {
123
+ await this.store.setProposedCheckpoint(proposedCheckpoint);
124
+ await this.l2TipsCache?.refresh();
125
+ });
126
+
127
+ return result;
128
+ }
129
+
130
+ /**
131
+ * Checks for local proposed blocks that do not match the ones to be checkpointed and prunes them.
132
+ * This method handles multiple checkpoints but returns after pruning the first conflict found.
133
+ * This is correct because pruning from the first conflict point removes all subsequent blocks,
134
+ * and when checkpoints are added afterward, they include all the correct blocks.
135
+ */
136
+ private async pruneMismatchingLocalBlocks(checkpoints: PublishedCheckpoint[]): Promise<ReconcileCheckpointsResult> {
137
+ const [lastCheckpointedBlockNumber, lastBlockNumber] = await Promise.all([
138
+ this.store.getCheckpointedL2BlockNumber(),
139
+ this.store.getLatestBlockNumber(),
140
+ ]);
141
+
142
+ // Exit early if there are no local uncheckpointed blocks
143
+ if (lastBlockNumber === lastCheckpointedBlockNumber) {
144
+ return { prunedBlocks: undefined, lastAlreadyInsertedBlockNumber: undefined };
145
+ }
146
+
147
+ // Get all uncheckpointed local blocks
148
+ const uncheckpointedLocalBlocks = await this.store.getBlocks(
149
+ BlockNumber.add(lastCheckpointedBlockNumber, 1),
150
+ lastBlockNumber - lastCheckpointedBlockNumber,
151
+ );
152
+
153
+ let lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
154
+
155
+ for (const publishedCheckpoint of checkpoints) {
156
+ const checkpointBlocks = publishedCheckpoint.checkpoint.blocks;
157
+ const slot = publishedCheckpoint.checkpoint.slot;
158
+ const localBlocksInSlot = uncheckpointedLocalBlocks.filter(b => b.slot === slot);
159
+
160
+ if (checkpointBlocks.length === 0) {
161
+ this.log.warn(`Checkpoint ${publishedCheckpoint.checkpoint.number} for slot ${slot} has no blocks`);
162
+ continue;
163
+ }
164
+
165
+ // Find the first checkpoint block that conflicts with an existing local block and prune local afterwards
166
+ for (const checkpointBlock of checkpointBlocks) {
167
+ const blockNumber = checkpointBlock.number;
168
+ const existingBlock = localBlocksInSlot.find(b => b.number === blockNumber);
169
+ const blockInfos = {
170
+ existingBlock: existingBlock?.toBlockInfo(),
171
+ checkpointBlock: checkpointBlock.toBlockInfo(),
172
+ };
173
+
174
+ if (!existingBlock) {
175
+ this.log.verbose(`No local block found for checkpointed block number ${blockNumber}`, blockInfos);
176
+ } else if (existingBlock.archive.root.equals(checkpointBlock.archive.root)) {
177
+ this.log.verbose(`Block number ${blockNumber} already inserted and matches checkpoint`, blockInfos);
178
+ lastAlreadyInsertedBlockNumber = blockNumber;
179
+ } else {
180
+ this.log.info(`Conflict detected at block ${blockNumber} between checkpointed and local block`, blockInfos);
181
+ const prunedBlocks = await this.removeBlocksAfter(BlockNumber(blockNumber - 1));
182
+ return { prunedBlocks, lastAlreadyInsertedBlockNumber };
183
+ }
184
+ }
185
+
186
+ // If local has more blocks than the checkpoint (e.g., local has [2,3,4] but checkpoint has [2,3]),
187
+ // we need to prune the extra local blocks so they match what was checkpointed
188
+ const lastCheckpointBlockNumber = checkpointBlocks.at(-1)!.number;
189
+ const lastLocalBlockNumber = localBlocksInSlot.at(-1)?.number;
190
+
191
+ if (lastLocalBlockNumber !== undefined && lastLocalBlockNumber > lastCheckpointBlockNumber) {
192
+ this.log.warn(
193
+ `Local chain for slot ${slot} ends at block ${lastLocalBlockNumber} but checkpoint ends at ${lastCheckpointBlockNumber}. Pruning blocks after block ${lastCheckpointBlockNumber}.`,
194
+ );
195
+ const prunedBlocks = await this.removeBlocksAfter(lastCheckpointBlockNumber);
196
+ return { prunedBlocks, lastAlreadyInsertedBlockNumber };
197
+ }
198
+ }
199
+
200
+ return { prunedBlocks: undefined, lastAlreadyInsertedBlockNumber };
201
+ }
202
+
203
+ /**
204
+ * Removes all uncheckpointed blocks strictly after the specified block number and cleans up associated contract data.
205
+ * This handles removal of provisionally added blocks along with their contract classes/instances.
206
+ * Verifies that each block being removed is not part of a stored checkpoint.
207
+ *
208
+ * @param blockNumber - Remove all blocks with number greater than this.
209
+ * @returns The removed blocks.
210
+ * @throws Error if any block to be removed is checkpointed.
211
+ */
212
+ public async removeUncheckpointedBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
213
+ const result = await this.store.transactionAsync(async () => {
214
+ // Verify we're only removing uncheckpointed blocks
215
+ const lastCheckpointedBlockNumber = await this.store.getCheckpointedL2BlockNumber();
216
+ if (blockNumber < lastCheckpointedBlockNumber) {
217
+ throw new Error(
218
+ `Cannot remove blocks after ${blockNumber} because checkpointed blocks exist up to ${lastCheckpointedBlockNumber}`,
219
+ );
220
+ }
221
+
222
+ const result = await this.removeBlocksAfter(blockNumber);
223
+
224
+ // Clear the proposed checkpoint if it exists, since its blocks have been pruned
225
+ await this.store.deleteProposedCheckpoint();
226
+
227
+ await this.l2TipsCache?.refresh();
228
+ return result;
229
+ });
230
+ return result;
231
+ }
232
+
233
+ /**
234
+ * Removes all blocks strictly after the given block number along with any logs and contract data.
235
+ * Does not remove their checkpoints.
236
+ */
237
+ private async removeBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
238
+ // First get the blocks to be removed so we can clean up contract data
239
+ const removedBlocks = await this.store.removeBlocksAfter(blockNumber);
240
+
241
+ // Clean up contract data and logs for the removed blocks
242
+ await Promise.all([
243
+ this.store.deleteLogs(removedBlocks),
244
+ ...removedBlocks.map(block => this.removeContractDataFromDb(block)),
245
+ ]);
246
+
247
+ return removedBlocks;
248
+ }
249
+
250
+ /**
251
+ * Removes all checkpoints after the given checkpoint number.
252
+ * Deletes ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated data
253
+ * that was stored for the removed checkpoints. Also removes ALL blocks (both checkpointed
254
+ * and uncheckpointed) after the last block of the given checkpoint.
255
+ *
256
+ * @param checkpointNumber - Remove all checkpoints strictly after this one.
257
+ * @returns True if the operation is successful.
258
+ */
259
+ public async removeCheckpointsAfter(checkpointNumber: CheckpointNumber): Promise<boolean> {
260
+ return await this.store.transactionAsync(async () => {
261
+ const { blocksRemoved = [] } = await this.store.removeCheckpointsAfter(checkpointNumber);
262
+
263
+ const opResults = await Promise.all([
264
+ // Prune rolls back to the last proven block, which is by definition valid
265
+ this.store.setPendingChainValidationStatus({ valid: true }),
266
+ // Remove contract data for all blocks being removed
267
+ ...blocksRemoved.map(block => this.removeContractDataFromDb(block)),
268
+ this.store.deleteLogs(blocksRemoved),
269
+ ]);
270
+
271
+ await this.l2TipsCache?.refresh();
272
+ return opResults.every(Boolean);
273
+ });
274
+ }
275
+
276
+ /**
277
+ * Updates the proven checkpoint number and refreshes the L2 tips cache.
278
+ * @param checkpointNumber - The checkpoint number to set as proven.
279
+ */
280
+ public async setProvenCheckpointNumber(checkpointNumber: CheckpointNumber): Promise<void> {
281
+ await this.store.transactionAsync(async () => {
282
+ await this.store.setProvenCheckpointNumber(checkpointNumber);
283
+ await this.l2TipsCache?.refresh();
284
+ });
285
+ }
286
+
287
+ /**
288
+ * Updates the finalized checkpoint number and refreshes the L2 tips cache.
289
+ * @param checkpointNumber - The checkpoint number to set as finalized.
290
+ */
291
+ public async setFinalizedCheckpointNumber(checkpointNumber: CheckpointNumber): Promise<void> {
292
+ await this.store.transactionAsync(async () => {
293
+ await this.store.setFinalizedCheckpointNumber(checkpointNumber);
294
+ await this.l2TipsCache?.refresh();
295
+ });
296
+ }
297
+
298
+ /** Extracts and stores contract data from a single block. */
299
+ private addContractDataToDb(block: L2Block): Promise<boolean> {
300
+ return this.updateContractDataOnDb(block, Operation.Store);
301
+ }
302
+
303
+ /** Removes contract data associated with a block. */
304
+ private removeContractDataFromDb(block: L2Block): Promise<boolean> {
305
+ return this.updateContractDataOnDb(block, Operation.Delete);
306
+ }
307
+
308
+ /** Adds or remove contract data associated with a block. */
309
+ private async updateContractDataOnDb(block: L2Block, operation: Operation): Promise<boolean> {
310
+ const contractClassLogs = block.body.txEffects.flatMap(txEffect => txEffect.contractClassLogs);
311
+ const privateLogs = block.body.txEffects.flatMap(txEffect => txEffect.privateLogs);
312
+ const publicLogs = block.body.txEffects.flatMap(txEffect => txEffect.publicLogs);
313
+
314
+ return (
315
+ await Promise.all([
316
+ this.updatePublishedContractClasses(contractClassLogs, block.number, operation),
317
+ this.updateDeployedContractInstances(privateLogs, block.number, operation),
318
+ this.updateUpdatedContractInstances(publicLogs, block.header.globalVariables.timestamp, operation),
319
+ ])
320
+ ).every(Boolean);
321
+ }
322
+
323
+ /**
324
+ * Extracts and stores contract classes out of ContractClassPublished events emitted by the class registry contract.
325
+ */
326
+ private async updatePublishedContractClasses(
327
+ allLogs: ContractClassLog[],
328
+ blockNum: BlockNumber,
329
+ operation: Operation,
330
+ ): Promise<boolean> {
331
+ const contractClassPublishedEvents = allLogs
332
+ .filter(log => ContractClassPublishedEvent.isContractClassPublishedEvent(log))
333
+ .map(log => ContractClassPublishedEvent.fromLog(log));
334
+
335
+ if (operation == Operation.Delete) {
336
+ const contractClasses = contractClassPublishedEvents.map(e => e.toContractClassPublic());
337
+ if (contractClasses.length > 0) {
338
+ contractClasses.forEach(c => this.log.verbose(`${Operation[operation]} contract class ${c.id.toString()}`));
339
+ return await this.store.deleteContractClasses(contractClasses, blockNum);
340
+ }
341
+ return true;
342
+ }
343
+
344
+ // Compute bytecode commitments and validate class IDs in a single pass.
345
+ const contractClasses: ContractClassPublicWithCommitment[] = [];
346
+ for (const event of contractClassPublishedEvents) {
347
+ const contractClass = await event.toContractClassPublicWithBytecodeCommitment();
348
+ const computedClassId = await computeContractClassId({
349
+ artifactHash: contractClass.artifactHash,
350
+ privateFunctionsRoot: contractClass.privateFunctionsRoot,
351
+ publicBytecodeCommitment: contractClass.publicBytecodeCommitment,
352
+ });
353
+ if (!computedClassId.equals(contractClass.id)) {
354
+ this.log.warn(
355
+ `Skipping contract class with mismatched id at block ${blockNum}. Claimed ${contractClass.id}, computed ${computedClassId}`,
356
+ { blockNum, contractClassId: event.contractClassId.toString() },
357
+ );
358
+ continue;
359
+ }
360
+ contractClasses.push(contractClass);
361
+ }
362
+
363
+ if (contractClasses.length > 0) {
364
+ contractClasses.forEach(c => this.log.verbose(`${Operation[operation]} contract class ${c.id.toString()}`));
365
+ return await this.store.addContractClasses(contractClasses, blockNum);
366
+ }
367
+ return true;
368
+ }
369
+
370
+ /**
371
+ * Extracts and stores contract instances out of ContractInstancePublished events emitted by the canonical deployer contract.
372
+ */
373
+ private async updateDeployedContractInstances(
374
+ allLogs: PrivateLog[],
375
+ blockNum: BlockNumber,
376
+ operation: Operation,
377
+ ): Promise<boolean> {
378
+ const allInstances = allLogs
379
+ .filter(log => ContractInstancePublishedEvent.isContractInstancePublishedEvent(log))
380
+ .map(log => ContractInstancePublishedEvent.fromLog(log))
381
+ .map(e => e.toContractInstance());
382
+
383
+ // Verify that each instance's address matches the one derived from its fields if we're adding
384
+ const contractInstances =
385
+ operation === Operation.Delete
386
+ ? allInstances
387
+ : await filterAsync(allInstances, async instance => {
388
+ const computedAddress = await computeContractAddressFromInstance(instance);
389
+ if (!computedAddress.equals(instance.address)) {
390
+ this.log.warn(
391
+ `Found contract instance with mismatched address at block ${blockNum}. Claimed ${instance.address} but computed ${computedAddress}.`,
392
+ { instanceAddress: instance.address.toString(), computedAddress: computedAddress.toString(), blockNum },
393
+ );
394
+ return false;
395
+ }
396
+ return true;
397
+ });
398
+
399
+ if (contractInstances.length > 0) {
400
+ contractInstances.forEach(c =>
401
+ this.log.verbose(`${Operation[operation]} contract instance at ${c.address.toString()}`),
402
+ );
403
+ if (operation == Operation.Store) {
404
+ return await this.store.addContractInstances(contractInstances, blockNum);
405
+ } else if (operation == Operation.Delete) {
406
+ return await this.store.deleteContractInstances(contractInstances, blockNum);
407
+ }
408
+ }
409
+ return true;
410
+ }
411
+
412
+ /**
413
+ * Extracts and stores contract instance updates out of ContractInstanceUpdated events.
414
+ */
415
+ private async updateUpdatedContractInstances(
416
+ allLogs: PublicLog[],
417
+ timestamp: UInt64,
418
+ operation: Operation,
419
+ ): Promise<boolean> {
420
+ const contractUpdates = allLogs
421
+ .filter(log => ContractInstanceUpdatedEvent.isContractInstanceUpdatedEvent(log))
422
+ .map(log => ContractInstanceUpdatedEvent.fromLog(log))
423
+ .map(e => e.toContractInstanceUpdate());
424
+
425
+ if (contractUpdates.length > 0) {
426
+ contractUpdates.forEach(c =>
427
+ this.log.verbose(`${Operation[operation]} contract instance update at ${c.address.toString()}`),
428
+ );
429
+ if (operation == Operation.Store) {
430
+ return await this.store.addContractInstanceUpdates(contractUpdates, timestamp);
431
+ } else if (operation == Operation.Delete) {
432
+ return await this.store.deleteContractInstanceUpdates(contractUpdates, timestamp);
433
+ }
434
+ }
435
+ return true;
436
+ }
437
+ }
@@ -1,5 +1,9 @@
1
+ import type { SlotNumber } from '@aztec/foundation/branded-types';
1
2
  import { createLogger } from '@aztec/foundation/log';
2
- import type { L2BlockNew } from '@aztec/stdlib/block';
3
+ import type { L2Block } from '@aztec/stdlib/block';
4
+ import type { CheckpointData } from '@aztec/stdlib/checkpoint';
5
+ import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
6
+ import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
3
7
  import {
4
8
  Attributes,
5
9
  type Gauge,
@@ -10,12 +14,14 @@ import {
10
14
  type TelemetryClient,
11
15
  type Tracer,
12
16
  type UpDownCounter,
17
+ createUpDownCounterWithDefault,
13
18
  } from '@aztec/telemetry-client';
14
19
 
15
20
  export class ArchiverInstrumentation {
16
21
  public readonly tracer: Tracer;
17
22
 
18
23
  private blockHeight: Gauge;
24
+ private checkpointHeight: Gauge;
19
25
  private txCount: UpDownCounter;
20
26
  private l1BlockHeight: Gauge;
21
27
  private proofsSubmittedDelay: Histogram;
@@ -26,6 +32,7 @@ export class ArchiverInstrumentation {
26
32
  private pruneCount: UpDownCounter;
27
33
 
28
34
  private syncDurationPerBlock: Histogram;
35
+ private syncDurationPerCheckpoint: Histogram;
29
36
  private syncBlockCount: UpDownCounter;
30
37
  private manaPerBlock: Histogram;
31
38
  private txsPerBlock: Histogram;
@@ -35,6 +42,8 @@ export class ArchiverInstrumentation {
35
42
 
36
43
  private blockProposalTxTargetCount: UpDownCounter;
37
44
 
45
+ private checkpointL1InclusionDelay: Histogram;
46
+
38
47
  private log = createLogger('archiver:instrumentation');
39
48
 
40
49
  private constructor(
@@ -46,17 +55,23 @@ export class ArchiverInstrumentation {
46
55
 
47
56
  this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT);
48
57
 
58
+ this.checkpointHeight = meter.createGauge(Metrics.ARCHIVER_CHECKPOINT_HEIGHT);
59
+
49
60
  this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT);
50
61
 
51
- this.txCount = meter.createUpDownCounter(Metrics.ARCHIVER_TOTAL_TXS);
62
+ this.txCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_TOTAL_TXS);
52
63
 
53
- this.proofsSubmittedCount = meter.createUpDownCounter(Metrics.ARCHIVER_ROLLUP_PROOF_COUNT);
64
+ this.proofsSubmittedCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_ROLLUP_PROOF_COUNT, {
65
+ [Attributes.PROOF_TIMED_OUT]: [true, false],
66
+ });
54
67
 
55
68
  this.proofsSubmittedDelay = meter.createHistogram(Metrics.ARCHIVER_ROLLUP_PROOF_DELAY);
56
69
 
57
70
  this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK);
58
71
 
59
- this.syncBlockCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
72
+ this.syncDurationPerCheckpoint = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_CHECKPOINT);
73
+
74
+ this.syncBlockCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
60
75
 
61
76
  this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK);
62
77
 
@@ -64,13 +79,21 @@ export class ArchiverInstrumentation {
64
79
 
65
80
  this.syncDurationPerMessage = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_MESSAGE);
66
81
 
67
- this.syncMessageCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_MESSAGE_COUNT);
82
+ this.syncMessageCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_MESSAGE_COUNT);
68
83
 
69
84
  this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION);
70
85
 
71
- this.pruneCount = meter.createUpDownCounter(Metrics.ARCHIVER_PRUNE_COUNT);
86
+ this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT);
87
+
88
+ this.blockProposalTxTargetCount = createUpDownCounterWithDefault(
89
+ meter,
90
+ Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT,
91
+ {
92
+ [Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: [true, false],
93
+ },
94
+ );
72
95
 
73
- this.blockProposalTxTargetCount = meter.createUpDownCounter(Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT);
96
+ this.checkpointL1InclusionDelay = meter.createHistogram(Metrics.ARCHIVER_CHECKPOINT_L1_INCLUSION_DELAY);
74
97
 
75
98
  this.dbMetrics = new LmdbMetrics(
76
99
  meter,
@@ -84,10 +107,6 @@ export class ArchiverInstrumentation {
84
107
  public static async new(telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
85
108
  const instance = new ArchiverInstrumentation(telemetry, lmdbStats);
86
109
 
87
- instance.syncBlockCount.add(0);
88
- instance.syncMessageCount.add(0);
89
- instance.pruneCount.add(0);
90
-
91
110
  await instance.telemetry.flush();
92
111
 
93
112
  return instance;
@@ -97,16 +116,26 @@ export class ArchiverInstrumentation {
97
116
  return this.telemetry.isEnabled();
98
117
  }
99
118
 
100
- public processNewBlocks(syncTimePerBlock: number, blocks: L2BlockNew[]) {
119
+ public processNewProposedBlock(syncTimePerBlock: number, block: L2Block) {
120
+ const attrs = { [Attributes.STATUS]: 'proposed' };
121
+ this.blockHeight.record(block.number, attrs);
101
122
  this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock));
102
- this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
103
- this.syncBlockCount.add(blocks.length);
104
123
 
105
- for (const block of blocks) {
106
- this.txCount.add(block.body.txEffects.length);
107
- this.txsPerBlock.record(block.body.txEffects.length);
108
- this.manaPerBlock.record(block.header.totalManaUsed.toNumber() / 1e6);
124
+ // Per block metrics
125
+ this.txCount.add(block.body.txEffects.length);
126
+ this.txsPerBlock.record(block.body.txEffects.length);
127
+ this.manaPerBlock.record(block.header.totalManaUsed.toNumber() / 1e6);
128
+ }
129
+
130
+ public processNewCheckpointedBlocks(syncTimePerCheckpoint: number, blocks: L2Block[]) {
131
+ if (blocks.length === 0) {
132
+ return;
109
133
  }
134
+
135
+ this.syncDurationPerCheckpoint.record(Math.ceil(syncTimePerCheckpoint));
136
+ this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
137
+ this.checkpointHeight.record(Math.max(...blocks.map(b => b.checkpointNumber)));
138
+ this.syncBlockCount.add(blocks.length);
110
139
  }
111
140
 
112
141
  public processNewMessages(count: number, syncPerMessageMs: number) {
@@ -122,8 +151,10 @@ export class ArchiverInstrumentation {
122
151
  this.pruneDuration.record(Math.ceil(duration));
123
152
  }
124
153
 
125
- public updateLastProvenBlock(blockNumber: number) {
126
- this.blockHeight.record(blockNumber, { [Attributes.STATUS]: 'proven' });
154
+ public updateLastProvenCheckpoint(checkpoint: CheckpointData) {
155
+ const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.blockCount - 1;
156
+ this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' });
157
+ this.checkpointHeight.record(checkpoint.checkpointNumber, { [Attributes.STATUS]: 'proven' });
127
158
  }
128
159
 
129
160
  public processProofsVerified(logs: { proverId: string; l2BlockNumber: bigint; delay: bigint }[]) {
@@ -149,4 +180,17 @@ export class ArchiverInstrumentation {
149
180
  [Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: usedTrace,
150
181
  });
151
182
  }
183
+
184
+ /**
185
+ * Records L1 inclusion timing for a checkpoint observed on L1 (seconds into the L2 slot).
186
+ */
187
+ public processCheckpointL1Timing(data: {
188
+ slotNumber: SlotNumber;
189
+ l1Timestamp: bigint;
190
+ l1Constants: Pick<L1RollupConstants, 'l1GenesisTime' | 'slotDuration'>;
191
+ }): void {
192
+ const slotStartTs = getTimestampForSlot(data.slotNumber, data.l1Constants);
193
+ const inclusionDelaySeconds = Number(data.l1Timestamp - slotStartTs);
194
+ this.checkpointL1InclusionDelay.record(inclusionDelaySeconds);
195
+ }
152
196
  }