@aztec/archiver 0.0.1-commit.03f7ef2 → 0.0.1-commit.04852196a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/README.md +156 -22
  2. package/dest/archiver.d.ts +141 -0
  3. package/dest/archiver.d.ts.map +1 -0
  4. package/dest/archiver.js +710 -0
  5. package/dest/{archiver/config.d.ts → config.d.ts} +9 -1
  6. package/dest/config.d.ts.map +1 -0
  7. package/dest/{archiver/config.js → config.js} +11 -2
  8. package/dest/errors.d.ts +41 -0
  9. package/dest/errors.d.ts.map +1 -0
  10. package/dest/{archiver/errors.js → errors.js} +8 -0
  11. package/dest/factory.d.ts +10 -8
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +93 -11
  14. package/dest/index.d.ts +11 -4
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +9 -3
  17. package/dest/interfaces.d.ts +9 -0
  18. package/dest/interfaces.d.ts.map +1 -0
  19. package/dest/interfaces.js +3 -0
  20. package/dest/{archiver/l1 → l1}/bin/retrieve-calldata.d.ts +1 -1
  21. package/dest/l1/bin/retrieve-calldata.d.ts.map +1 -0
  22. package/dest/{archiver/l1 → l1}/bin/retrieve-calldata.js +35 -32
  23. package/dest/l1/calldata_retriever.d.ts +135 -0
  24. package/dest/l1/calldata_retriever.d.ts.map +1 -0
  25. package/dest/l1/calldata_retriever.js +402 -0
  26. package/dest/l1/data_retrieval.d.ts +88 -0
  27. package/dest/l1/data_retrieval.d.ts.map +1 -0
  28. package/dest/{archiver/l1 → l1}/data_retrieval.js +54 -71
  29. package/dest/{archiver/l1 → l1}/debug_tx.d.ts +1 -1
  30. package/dest/l1/debug_tx.d.ts.map +1 -0
  31. package/dest/{archiver/l1 → l1}/spire_proposer.d.ts +5 -5
  32. package/dest/l1/spire_proposer.d.ts.map +1 -0
  33. package/dest/{archiver/l1 → l1}/spire_proposer.js +9 -17
  34. package/dest/{archiver/l1 → l1}/trace_tx.d.ts +1 -1
  35. package/dest/l1/trace_tx.d.ts.map +1 -0
  36. package/dest/l1/types.d.ts +12 -0
  37. package/dest/l1/types.d.ts.map +1 -0
  38. package/dest/{archiver/l1 → l1}/validate_trace.d.ts +6 -3
  39. package/dest/l1/validate_trace.d.ts.map +1 -0
  40. package/dest/{archiver/l1 → l1}/validate_trace.js +14 -10
  41. package/dest/modules/data_source_base.d.ts +89 -0
  42. package/dest/modules/data_source_base.d.ts.map +1 -0
  43. package/dest/modules/data_source_base.js +216 -0
  44. package/dest/modules/data_store_updater.d.ts +83 -0
  45. package/dest/modules/data_store_updater.d.ts.map +1 -0
  46. package/dest/modules/data_store_updater.js +331 -0
  47. package/dest/modules/instrumentation.d.ts +50 -0
  48. package/dest/modules/instrumentation.d.ts.map +1 -0
  49. package/dest/{archiver → modules}/instrumentation.js +44 -68
  50. package/dest/modules/l1_synchronizer.d.ts +72 -0
  51. package/dest/modules/l1_synchronizer.d.ts.map +1 -0
  52. package/dest/modules/l1_synchronizer.js +1119 -0
  53. package/dest/modules/validation.d.ts +17 -0
  54. package/dest/modules/validation.d.ts.map +1 -0
  55. package/dest/{archiver → modules}/validation.js +7 -1
  56. package/dest/store/block_store.d.ts +196 -0
  57. package/dest/store/block_store.d.ts.map +1 -0
  58. package/dest/{archiver/kv_archiver_store → store}/block_store.js +228 -62
  59. package/dest/store/contract_class_store.d.ts +18 -0
  60. package/dest/store/contract_class_store.d.ts.map +1 -0
  61. package/dest/{archiver/kv_archiver_store → store}/contract_class_store.js +12 -8
  62. package/dest/store/contract_instance_store.d.ts +24 -0
  63. package/dest/store/contract_instance_store.d.ts.map +1 -0
  64. package/dest/{archiver/kv_archiver_store → store}/contract_instance_store.js +1 -1
  65. package/dest/store/kv_archiver_store.d.ts +354 -0
  66. package/dest/store/kv_archiver_store.d.ts.map +1 -0
  67. package/dest/store/kv_archiver_store.js +464 -0
  68. package/dest/store/l2_tips_cache.d.ts +19 -0
  69. package/dest/store/l2_tips_cache.d.ts.map +1 -0
  70. package/dest/store/l2_tips_cache.js +89 -0
  71. package/dest/store/log_store.d.ts +54 -0
  72. package/dest/store/log_store.d.ts.map +1 -0
  73. package/dest/{archiver/kv_archiver_store → store}/log_store.js +156 -104
  74. package/dest/{archiver/kv_archiver_store → store}/message_store.d.ts +1 -1
  75. package/dest/store/message_store.d.ts.map +1 -0
  76. package/dest/{archiver/kv_archiver_store → store}/message_store.js +1 -1
  77. package/dest/{archiver/structs → structs}/data_retrieval.d.ts +1 -1
  78. package/dest/structs/data_retrieval.d.ts.map +1 -0
  79. package/dest/structs/inbox_message.d.ts +15 -0
  80. package/dest/structs/inbox_message.d.ts.map +1 -0
  81. package/dest/{archiver/structs → structs}/published.d.ts +1 -1
  82. package/dest/structs/published.d.ts.map +1 -0
  83. package/dest/test/fake_l1_state.d.ts +195 -0
  84. package/dest/test/fake_l1_state.d.ts.map +1 -0
  85. package/dest/test/fake_l1_state.js +421 -0
  86. package/dest/test/index.d.ts +2 -1
  87. package/dest/test/index.d.ts.map +1 -1
  88. package/dest/test/index.js +4 -1
  89. package/dest/test/mock_archiver.d.ts +2 -2
  90. package/dest/test/mock_archiver.d.ts.map +1 -1
  91. package/dest/test/mock_archiver.js +3 -3
  92. package/dest/test/mock_l1_to_l2_message_source.d.ts +2 -2
  93. package/dest/test/mock_l1_to_l2_message_source.d.ts.map +1 -1
  94. package/dest/test/mock_l1_to_l2_message_source.js +12 -3
  95. package/dest/test/mock_l2_block_source.d.ts +40 -17
  96. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  97. package/dest/test/mock_l2_block_source.js +235 -83
  98. package/dest/test/mock_structs.d.ts +81 -3
  99. package/dest/test/mock_structs.d.ts.map +1 -1
  100. package/dest/test/mock_structs.js +152 -7
  101. package/dest/test/noop_l1_archiver.d.ts +23 -0
  102. package/dest/test/noop_l1_archiver.d.ts.map +1 -0
  103. package/dest/test/noop_l1_archiver.js +68 -0
  104. package/package.json +16 -17
  105. package/src/archiver.ts +466 -0
  106. package/src/{archiver/config.ts → config.ts} +13 -2
  107. package/src/{archiver/errors.ts → errors.ts} +12 -0
  108. package/src/factory.ts +143 -12
  109. package/src/index.ts +11 -3
  110. package/src/interfaces.ts +9 -0
  111. package/src/l1/README.md +55 -0
  112. package/src/{archiver/l1 → l1}/bin/retrieve-calldata.ts +45 -33
  113. package/src/l1/calldata_retriever.ts +511 -0
  114. package/src/{archiver/l1 → l1}/data_retrieval.ts +75 -94
  115. package/src/{archiver/l1 → l1}/spire_proposer.ts +7 -15
  116. package/src/{archiver/l1 → l1}/validate_trace.ts +25 -7
  117. package/src/modules/data_source_base.ts +328 -0
  118. package/src/modules/data_store_updater.ts +453 -0
  119. package/src/{archiver → modules}/instrumentation.ts +53 -70
  120. package/src/modules/l1_synchronizer.ts +939 -0
  121. package/src/{archiver → modules}/validation.ts +11 -6
  122. package/src/{archiver/kv_archiver_store → store}/block_store.ts +293 -100
  123. package/src/{archiver/kv_archiver_store → store}/contract_class_store.ts +12 -8
  124. package/src/{archiver/kv_archiver_store → store}/contract_instance_store.ts +1 -1
  125. package/src/{archiver/kv_archiver_store → store}/kv_archiver_store.ts +273 -40
  126. package/src/store/l2_tips_cache.ts +89 -0
  127. package/src/{archiver/kv_archiver_store → store}/log_store.ts +256 -141
  128. package/src/{archiver/kv_archiver_store → store}/message_store.ts +1 -1
  129. package/src/test/fake_l1_state.ts +657 -0
  130. package/src/test/index.ts +4 -0
  131. package/src/test/mock_archiver.ts +4 -3
  132. package/src/test/mock_l1_to_l2_message_source.ts +10 -4
  133. package/src/test/mock_l2_block_source.ts +282 -90
  134. package/src/test/mock_structs.ts +283 -8
  135. package/src/test/noop_l1_archiver.ts +109 -0
  136. package/dest/archiver/archiver.d.ts +0 -304
  137. package/dest/archiver/archiver.d.ts.map +0 -1
  138. package/dest/archiver/archiver.js +0 -1645
  139. package/dest/archiver/archiver_store.d.ts +0 -308
  140. package/dest/archiver/archiver_store.d.ts.map +0 -1
  141. package/dest/archiver/archiver_store.js +0 -4
  142. package/dest/archiver/archiver_store_test_suite.d.ts +0 -8
  143. package/dest/archiver/archiver_store_test_suite.d.ts.map +0 -1
  144. package/dest/archiver/archiver_store_test_suite.js +0 -2790
  145. package/dest/archiver/config.d.ts.map +0 -1
  146. package/dest/archiver/errors.d.ts +0 -36
  147. package/dest/archiver/errors.d.ts.map +0 -1
  148. package/dest/archiver/index.d.ts +0 -7
  149. package/dest/archiver/index.d.ts.map +0 -1
  150. package/dest/archiver/index.js +0 -4
  151. package/dest/archiver/instrumentation.d.ts +0 -37
  152. package/dest/archiver/instrumentation.d.ts.map +0 -1
  153. package/dest/archiver/kv_archiver_store/block_store.d.ts +0 -157
  154. package/dest/archiver/kv_archiver_store/block_store.d.ts.map +0 -1
  155. package/dest/archiver/kv_archiver_store/contract_class_store.d.ts +0 -18
  156. package/dest/archiver/kv_archiver_store/contract_class_store.d.ts.map +0 -1
  157. package/dest/archiver/kv_archiver_store/contract_instance_store.d.ts +0 -24
  158. package/dest/archiver/kv_archiver_store/contract_instance_store.d.ts.map +0 -1
  159. package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts +0 -158
  160. package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts.map +0 -1
  161. package/dest/archiver/kv_archiver_store/kv_archiver_store.js +0 -313
  162. package/dest/archiver/kv_archiver_store/log_store.d.ts +0 -45
  163. package/dest/archiver/kv_archiver_store/log_store.d.ts.map +0 -1
  164. package/dest/archiver/kv_archiver_store/message_store.d.ts.map +0 -1
  165. package/dest/archiver/l1/bin/retrieve-calldata.d.ts.map +0 -1
  166. package/dest/archiver/l1/calldata_retriever.d.ts +0 -112
  167. package/dest/archiver/l1/calldata_retriever.d.ts.map +0 -1
  168. package/dest/archiver/l1/calldata_retriever.js +0 -471
  169. package/dest/archiver/l1/data_retrieval.d.ts +0 -90
  170. package/dest/archiver/l1/data_retrieval.d.ts.map +0 -1
  171. package/dest/archiver/l1/debug_tx.d.ts.map +0 -1
  172. package/dest/archiver/l1/spire_proposer.d.ts.map +0 -1
  173. package/dest/archiver/l1/trace_tx.d.ts.map +0 -1
  174. package/dest/archiver/l1/types.d.ts +0 -12
  175. package/dest/archiver/l1/types.d.ts.map +0 -1
  176. package/dest/archiver/l1/validate_trace.d.ts.map +0 -1
  177. package/dest/archiver/structs/data_retrieval.d.ts.map +0 -1
  178. package/dest/archiver/structs/inbox_message.d.ts +0 -15
  179. package/dest/archiver/structs/inbox_message.d.ts.map +0 -1
  180. package/dest/archiver/structs/published.d.ts.map +0 -1
  181. package/dest/archiver/validation.d.ts +0 -17
  182. package/dest/archiver/validation.d.ts.map +0 -1
  183. package/dest/rpc/index.d.ts +0 -9
  184. package/dest/rpc/index.d.ts.map +0 -1
  185. package/dest/rpc/index.js +0 -15
  186. package/src/archiver/archiver.ts +0 -2157
  187. package/src/archiver/archiver_store.ts +0 -372
  188. package/src/archiver/archiver_store_test_suite.ts +0 -2863
  189. package/src/archiver/index.ts +0 -6
  190. package/src/archiver/l1/README.md +0 -98
  191. package/src/archiver/l1/calldata_retriever.ts +0 -641
  192. package/src/rpc/index.ts +0 -16
  193. /package/dest/{archiver/l1 → l1}/debug_tx.js +0 -0
  194. /package/dest/{archiver/l1 → l1}/trace_tx.js +0 -0
  195. /package/dest/{archiver/l1 → l1}/types.js +0 -0
  196. /package/dest/{archiver/structs → structs}/data_retrieval.js +0 -0
  197. /package/dest/{archiver/structs → structs}/inbox_message.js +0 -0
  198. /package/dest/{archiver/structs → structs}/published.js +0 -0
  199. /package/src/{archiver/l1 → l1}/debug_tx.ts +0 -0
  200. /package/src/{archiver/l1 → l1}/trace_tx.ts +0 -0
  201. /package/src/{archiver/l1 → l1}/types.ts +0 -0
  202. /package/src/{archiver/structs → structs}/data_retrieval.ts +0 -0
  203. /package/src/{archiver/structs → structs}/inbox_message.ts +0 -0
  204. /package/src/{archiver/structs → structs}/published.ts +0 -0
@@ -0,0 +1,453 @@
1
+ import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
2
+ import { Fr } from '@aztec/foundation/curves/bn254';
3
+ import { createLogger } from '@aztec/foundation/log';
4
+ import {
5
+ ContractClassPublishedEvent,
6
+ PrivateFunctionBroadcastedEvent,
7
+ UtilityFunctionBroadcastedEvent,
8
+ } from '@aztec/protocol-contracts/class-registry';
9
+ import {
10
+ ContractInstancePublishedEvent,
11
+ ContractInstanceUpdatedEvent,
12
+ } from '@aztec/protocol-contracts/instance-registry';
13
+ import type { L2Block, ValidateCheckpointResult } from '@aztec/stdlib/block';
14
+ import { type PublishedCheckpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
15
+ import {
16
+ type ExecutablePrivateFunctionWithMembershipProof,
17
+ type UtilityFunctionWithMembershipProof,
18
+ computePublicBytecodeCommitment,
19
+ isValidPrivateFunctionMembershipProof,
20
+ isValidUtilityFunctionMembershipProof,
21
+ } from '@aztec/stdlib/contract';
22
+ import type { ContractClassLog, PrivateLog, PublicLog } from '@aztec/stdlib/logs';
23
+ import type { UInt64 } from '@aztec/stdlib/types';
24
+
25
+ import groupBy from 'lodash.groupby';
26
+
27
+ import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
28
+ import type { L2TipsCache } from '../store/l2_tips_cache.js';
29
+
30
+ /** Operation type for contract data updates. */
31
+ enum Operation {
32
+ Store,
33
+ Delete,
34
+ }
35
+
36
+ /** Result of adding checkpoints with information about any pruned blocks. */
37
+ type ReconcileCheckpointsResult = {
38
+ /** Blocks that were pruned due to conflict with L1 checkpoints. */
39
+ prunedBlocks: L2Block[] | undefined;
40
+ /** Last block number that was already inserted locally, or undefined if none. */
41
+ lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
42
+ };
43
+
44
+ /** Archiver helper module to handle updates to the data store. */
45
+ export class ArchiverDataStoreUpdater {
46
+ private readonly log = createLogger('archiver:store_updater');
47
+
48
+ constructor(
49
+ private store: KVArchiverDataStore,
50
+ private l2TipsCache?: L2TipsCache,
51
+ private opts: { rollupManaLimit?: number } = {},
52
+ ) {}
53
+
54
+ /**
55
+ * Adds proposed blocks to the store with contract class/instance extraction from logs.
56
+ * These are uncheckpointed blocks that have been proposed by the sequencer but not yet included in a checkpoint on L1.
57
+ * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
58
+ * and individually broadcasted functions from the block logs.
59
+ *
60
+ * @param blocks - The proposed L2 blocks to add.
61
+ * @param pendingChainValidationStatus - Optional validation status to set.
62
+ * @returns True if the operation is successful.
63
+ */
64
+ public async addProposedBlocks(
65
+ blocks: L2Block[],
66
+ pendingChainValidationStatus?: ValidateCheckpointResult,
67
+ ): Promise<boolean> {
68
+ const result = await this.store.transactionAsync(async () => {
69
+ await this.store.addProposedBlocks(blocks);
70
+
71
+ const opResults = await Promise.all([
72
+ // Update the pending chain validation status if provided
73
+ pendingChainValidationStatus && this.store.setPendingChainValidationStatus(pendingChainValidationStatus),
74
+ // Add any logs emitted during the retrieved blocks
75
+ this.store.addLogs(blocks),
76
+ // Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
77
+ ...blocks.map(block => this.addContractDataToDb(block)),
78
+ ]);
79
+
80
+ await this.l2TipsCache?.refresh();
81
+ return opResults.every(Boolean);
82
+ });
83
+ return result;
84
+ }
85
+
86
+ /**
87
+ * Reconciles local blocks with incoming checkpoints from L1.
88
+ * Adds new checkpoints to the store with contract class/instance extraction from logs.
89
+ * Prunes any local blocks that conflict with checkpoint data (by comparing archive roots).
90
+ * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
91
+ * and individually broadcasted functions from the checkpoint block logs.
92
+ *
93
+ * @param checkpoints - The published checkpoints to add.
94
+ * @param pendingChainValidationStatus - Optional validation status to set.
95
+ * @returns Result with information about any pruned blocks.
96
+ */
97
+ public async addCheckpoints(
98
+ checkpoints: PublishedCheckpoint[],
99
+ pendingChainValidationStatus?: ValidateCheckpointResult,
100
+ ): Promise<ReconcileCheckpointsResult> {
101
+ for (const checkpoint of checkpoints) {
102
+ validateCheckpoint(checkpoint.checkpoint, { rollupManaLimit: this.opts?.rollupManaLimit });
103
+ }
104
+
105
+ const result = await this.store.transactionAsync(async () => {
106
+ // Before adding checkpoints, check for conflicts with local blocks if any
107
+ const { prunedBlocks, lastAlreadyInsertedBlockNumber } = await this.pruneMismatchingLocalBlocks(checkpoints);
108
+
109
+ await this.store.addCheckpoints(checkpoints);
110
+
111
+ // Filter out blocks that were already inserted via addProposedBlocks() to avoid duplicating logs/contract data
112
+ const newBlocks = checkpoints
113
+ .flatMap((ch: PublishedCheckpoint) => ch.checkpoint.blocks)
114
+ .filter(b => lastAlreadyInsertedBlockNumber === undefined || b.number > lastAlreadyInsertedBlockNumber);
115
+
116
+ await Promise.all([
117
+ // Update the pending chain validation status if provided
118
+ pendingChainValidationStatus && this.store.setPendingChainValidationStatus(pendingChainValidationStatus),
119
+ // Add any logs emitted during the retrieved blocks
120
+ this.store.addLogs(newBlocks),
121
+ // Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
122
+ ...newBlocks.map(block => this.addContractDataToDb(block)),
123
+ ]);
124
+
125
+ await this.l2TipsCache?.refresh();
126
+ return { prunedBlocks, lastAlreadyInsertedBlockNumber };
127
+ });
128
+ return result;
129
+ }
130
+
131
+ /**
132
+ * Checks for local proposed blocks that do not match the ones to be checkpointed and prunes them.
133
+ * This method handles multiple checkpoints but returns after pruning the first conflict found.
134
+ * This is correct because pruning from the first conflict point removes all subsequent blocks,
135
+ * and when checkpoints are added afterward, they include all the correct blocks.
136
+ */
137
+ private async pruneMismatchingLocalBlocks(checkpoints: PublishedCheckpoint[]): Promise<ReconcileCheckpointsResult> {
138
+ const [lastCheckpointedBlockNumber, lastBlockNumber] = await Promise.all([
139
+ this.store.getCheckpointedL2BlockNumber(),
140
+ this.store.getLatestBlockNumber(),
141
+ ]);
142
+
143
+ // Exit early if there are no local uncheckpointed blocks
144
+ if (lastBlockNumber === lastCheckpointedBlockNumber) {
145
+ return { prunedBlocks: undefined, lastAlreadyInsertedBlockNumber: undefined };
146
+ }
147
+
148
+ // Get all uncheckpointed local blocks
149
+ const uncheckpointedLocalBlocks = await this.store.getBlocks(
150
+ BlockNumber.add(lastCheckpointedBlockNumber, 1),
151
+ lastBlockNumber - lastCheckpointedBlockNumber,
152
+ );
153
+
154
+ let lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
155
+
156
+ for (const publishedCheckpoint of checkpoints) {
157
+ const checkpointBlocks = publishedCheckpoint.checkpoint.blocks;
158
+ const slot = publishedCheckpoint.checkpoint.slot;
159
+ const localBlocksInSlot = uncheckpointedLocalBlocks.filter(b => b.slot === slot);
160
+
161
+ if (checkpointBlocks.length === 0) {
162
+ this.log.warn(`Checkpoint ${publishedCheckpoint.checkpoint.number} for slot ${slot} has no blocks`);
163
+ continue;
164
+ }
165
+
166
+ // Find the first checkpoint block that conflicts with an existing local block and prune local afterwards
167
+ for (const checkpointBlock of checkpointBlocks) {
168
+ const blockNumber = checkpointBlock.number;
169
+ const existingBlock = localBlocksInSlot.find(b => b.number === blockNumber);
170
+ const blockInfos = {
171
+ existingBlock: existingBlock?.toBlockInfo(),
172
+ checkpointBlock: checkpointBlock.toBlockInfo(),
173
+ };
174
+
175
+ if (!existingBlock) {
176
+ this.log.verbose(`No local block found for checkpointed block number ${blockNumber}`, blockInfos);
177
+ } else if (existingBlock.archive.root.equals(checkpointBlock.archive.root)) {
178
+ this.log.verbose(`Block number ${blockNumber} already inserted and matches checkpoint`, blockInfos);
179
+ lastAlreadyInsertedBlockNumber = blockNumber;
180
+ } else {
181
+ this.log.warn(`Conflict detected at block ${blockNumber} between checkpointed and local block`, blockInfos);
182
+ const prunedBlocks = await this.removeBlocksAfter(BlockNumber(blockNumber - 1));
183
+ return { prunedBlocks, lastAlreadyInsertedBlockNumber };
184
+ }
185
+ }
186
+
187
+ // If local has more blocks than the checkpoint (e.g., local has [2,3,4] but checkpoint has [2,3]),
188
+ // we need to prune the extra local blocks so they match what was checkpointed
189
+ const lastCheckpointBlockNumber = checkpointBlocks.at(-1)!.number;
190
+ const lastLocalBlockNumber = localBlocksInSlot.at(-1)?.number;
191
+
192
+ if (lastLocalBlockNumber !== undefined && lastLocalBlockNumber > lastCheckpointBlockNumber) {
193
+ this.log.warn(
194
+ `Local chain for slot ${slot} ends at block ${lastLocalBlockNumber} but checkpoint ends at ${lastCheckpointBlockNumber}. Pruning blocks after block ${lastCheckpointBlockNumber}.`,
195
+ );
196
+ const prunedBlocks = await this.removeBlocksAfter(lastCheckpointBlockNumber);
197
+ return { prunedBlocks, lastAlreadyInsertedBlockNumber };
198
+ }
199
+ }
200
+
201
+ return { prunedBlocks: undefined, lastAlreadyInsertedBlockNumber };
202
+ }
203
+
204
+ /**
205
+ * Removes all uncheckpointed blocks strictly after the specified block number and cleans up associated contract data.
206
+ * This handles removal of provisionally added blocks along with their contract classes/instances.
207
+ * Verifies that each block being removed is not part of a stored checkpoint.
208
+ *
209
+ * @param blockNumber - Remove all blocks with number greater than this.
210
+ * @returns The removed blocks.
211
+ * @throws Error if any block to be removed is checkpointed.
212
+ */
213
+ public async removeUncheckpointedBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
214
+ const result = await this.store.transactionAsync(async () => {
215
+ // Verify we're only removing uncheckpointed blocks
216
+ const lastCheckpointedBlockNumber = await this.store.getCheckpointedL2BlockNumber();
217
+ if (blockNumber < lastCheckpointedBlockNumber) {
218
+ throw new Error(
219
+ `Cannot remove blocks after ${blockNumber} because checkpointed blocks exist up to ${lastCheckpointedBlockNumber}`,
220
+ );
221
+ }
222
+
223
+ const result = await this.removeBlocksAfter(blockNumber);
224
+ await this.l2TipsCache?.refresh();
225
+ return result;
226
+ });
227
+ return result;
228
+ }
229
+
230
+ /**
231
+ * Removes all blocks strictly after the given block number along with any logs and contract data.
232
+ * Does not remove their checkpoints.
233
+ */
234
+ private async removeBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
235
+ // First get the blocks to be removed so we can clean up contract data
236
+ const removedBlocks = await this.store.removeBlocksAfter(blockNumber);
237
+
238
+ // Clean up contract data and logs for the removed blocks
239
+ await Promise.all([
240
+ this.store.deleteLogs(removedBlocks),
241
+ ...removedBlocks.map(block => this.removeContractDataFromDb(block)),
242
+ ]);
243
+
244
+ return removedBlocks;
245
+ }
246
+
247
+ /**
248
+ * Removes all checkpoints after the given checkpoint number.
249
+ * Deletes ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated data
250
+ * that was stored for the removed checkpoints. Also removes ALL blocks (both checkpointed
251
+ * and uncheckpointed) after the last block of the given checkpoint.
252
+ *
253
+ * @param checkpointNumber - Remove all checkpoints strictly after this one.
254
+ * @returns True if the operation is successful.
255
+ */
256
+ public async removeCheckpointsAfter(checkpointNumber: CheckpointNumber): Promise<boolean> {
257
+ return await this.store.transactionAsync(async () => {
258
+ const { blocksRemoved = [] } = await this.store.removeCheckpointsAfter(checkpointNumber);
259
+
260
+ const opResults = await Promise.all([
261
+ // Prune rolls back to the last proven block, which is by definition valid
262
+ this.store.setPendingChainValidationStatus({ valid: true }),
263
+ // Remove contract data for all blocks being removed
264
+ ...blocksRemoved.map(block => this.removeContractDataFromDb(block)),
265
+ this.store.deleteLogs(blocksRemoved),
266
+ ]);
267
+
268
+ await this.l2TipsCache?.refresh();
269
+ return opResults.every(Boolean);
270
+ });
271
+ }
272
+
273
+ /**
274
+ * Updates the proven checkpoint number and refreshes the L2 tips cache.
275
+ * @param checkpointNumber - The checkpoint number to set as proven.
276
+ */
277
+ public async setProvenCheckpointNumber(checkpointNumber: CheckpointNumber): Promise<void> {
278
+ await this.store.transactionAsync(async () => {
279
+ await this.store.setProvenCheckpointNumber(checkpointNumber);
280
+ await this.l2TipsCache?.refresh();
281
+ });
282
+ }
283
+
284
+ /** Extracts and stores contract data from a single block. */
285
+ private addContractDataToDb(block: L2Block): Promise<boolean> {
286
+ return this.updateContractDataOnDb(block, Operation.Store);
287
+ }
288
+
289
+ /** Removes contract data associated with a block. */
290
+ private removeContractDataFromDb(block: L2Block): Promise<boolean> {
291
+ return this.updateContractDataOnDb(block, Operation.Delete);
292
+ }
293
+
294
+ /** Adds or remove contract data associated with a block. */
295
+ private async updateContractDataOnDb(block: L2Block, operation: Operation): Promise<boolean> {
296
+ const contractClassLogs = block.body.txEffects.flatMap(txEffect => txEffect.contractClassLogs);
297
+ const privateLogs = block.body.txEffects.flatMap(txEffect => txEffect.privateLogs);
298
+ const publicLogs = block.body.txEffects.flatMap(txEffect => txEffect.publicLogs);
299
+
300
+ return (
301
+ await Promise.all([
302
+ this.updatePublishedContractClasses(contractClassLogs, block.number, operation),
303
+ this.updateDeployedContractInstances(privateLogs, block.number, operation),
304
+ this.updateUpdatedContractInstances(publicLogs, block.header.globalVariables.timestamp, operation),
305
+ operation === Operation.Store
306
+ ? this.storeBroadcastedIndividualFunctions(contractClassLogs, block.number)
307
+ : Promise.resolve(true),
308
+ ])
309
+ ).every(Boolean);
310
+ }
311
+
312
+ /**
313
+ * Extracts and stores contract classes out of ContractClassPublished events emitted by the class registry contract.
314
+ */
315
+ private async updatePublishedContractClasses(
316
+ allLogs: ContractClassLog[],
317
+ blockNum: BlockNumber,
318
+ operation: Operation,
319
+ ): Promise<boolean> {
320
+ const contractClassPublishedEvents = allLogs
321
+ .filter(log => ContractClassPublishedEvent.isContractClassPublishedEvent(log))
322
+ .map(log => ContractClassPublishedEvent.fromLog(log));
323
+
324
+ const contractClasses = await Promise.all(contractClassPublishedEvents.map(e => e.toContractClassPublic()));
325
+ if (contractClasses.length > 0) {
326
+ contractClasses.forEach(c => this.log.verbose(`${Operation[operation]} contract class ${c.id.toString()}`));
327
+ if (operation == Operation.Store) {
328
+ // TODO: Will probably want to create some worker threads to compute these bytecode commitments as they are expensive
329
+ const commitments = await Promise.all(
330
+ contractClasses.map(c => computePublicBytecodeCommitment(c.packedBytecode)),
331
+ );
332
+ return await this.store.addContractClasses(contractClasses, commitments, blockNum);
333
+ } else if (operation == Operation.Delete) {
334
+ return await this.store.deleteContractClasses(contractClasses, blockNum);
335
+ }
336
+ }
337
+ return true;
338
+ }
339
+
340
+ /**
341
+ * Extracts and stores contract instances out of ContractInstancePublished events emitted by the canonical deployer contract.
342
+ */
343
+ private async updateDeployedContractInstances(
344
+ allLogs: PrivateLog[],
345
+ blockNum: BlockNumber,
346
+ operation: Operation,
347
+ ): Promise<boolean> {
348
+ const contractInstances = allLogs
349
+ .filter(log => ContractInstancePublishedEvent.isContractInstancePublishedEvent(log))
350
+ .map(log => ContractInstancePublishedEvent.fromLog(log))
351
+ .map(e => e.toContractInstance());
352
+ if (contractInstances.length > 0) {
353
+ contractInstances.forEach(c =>
354
+ this.log.verbose(`${Operation[operation]} contract instance at ${c.address.toString()}`),
355
+ );
356
+ if (operation == Operation.Store) {
357
+ return await this.store.addContractInstances(contractInstances, blockNum);
358
+ } else if (operation == Operation.Delete) {
359
+ return await this.store.deleteContractInstances(contractInstances, blockNum);
360
+ }
361
+ }
362
+ return true;
363
+ }
364
+
365
+ /**
366
+ * Extracts and stores contract instance updates out of ContractInstanceUpdated events.
367
+ */
368
+ private async updateUpdatedContractInstances(
369
+ allLogs: PublicLog[],
370
+ timestamp: UInt64,
371
+ operation: Operation,
372
+ ): Promise<boolean> {
373
+ const contractUpdates = allLogs
374
+ .filter(log => ContractInstanceUpdatedEvent.isContractInstanceUpdatedEvent(log))
375
+ .map(log => ContractInstanceUpdatedEvent.fromLog(log))
376
+ .map(e => e.toContractInstanceUpdate());
377
+
378
+ if (contractUpdates.length > 0) {
379
+ contractUpdates.forEach(c =>
380
+ this.log.verbose(`${Operation[operation]} contract instance update at ${c.address.toString()}`),
381
+ );
382
+ if (operation == Operation.Store) {
383
+ return await this.store.addContractInstanceUpdates(contractUpdates, timestamp);
384
+ } else if (operation == Operation.Delete) {
385
+ return await this.store.deleteContractInstanceUpdates(contractUpdates, timestamp);
386
+ }
387
+ }
388
+ return true;
389
+ }
390
+
391
+ /**
392
+ * Stores the functions that were broadcasted individually.
393
+ *
394
+ * @dev Beware that there is not a delete variant of this, since they are added to contract classes
395
+ * and will be deleted as part of the class if needed.
396
+ */
397
+ private async storeBroadcastedIndividualFunctions(
398
+ allLogs: ContractClassLog[],
399
+ _blockNum: BlockNumber,
400
+ ): Promise<boolean> {
401
+ // Filter out private and utility function broadcast events
402
+ const privateFnEvents = allLogs
403
+ .filter(log => PrivateFunctionBroadcastedEvent.isPrivateFunctionBroadcastedEvent(log))
404
+ .map(log => PrivateFunctionBroadcastedEvent.fromLog(log));
405
+ const utilityFnEvents = allLogs
406
+ .filter(log => UtilityFunctionBroadcastedEvent.isUtilityFunctionBroadcastedEvent(log))
407
+ .map(log => UtilityFunctionBroadcastedEvent.fromLog(log));
408
+
409
+ // Group all events by contract class id
410
+ for (const [classIdString, classEvents] of Object.entries(
411
+ groupBy([...privateFnEvents, ...utilityFnEvents], e => e.contractClassId.toString()),
412
+ )) {
413
+ const contractClassId = Fr.fromHexString(classIdString);
414
+ const contractClass = await this.store.getContractClass(contractClassId);
415
+ if (!contractClass) {
416
+ this.log.warn(`Skipping broadcasted functions as contract class ${contractClassId.toString()} was not found`);
417
+ continue;
418
+ }
419
+
420
+ // Split private and utility functions, and filter out invalid ones
421
+ const allFns = classEvents.map(e => e.toFunctionWithMembershipProof());
422
+ const privateFns = allFns.filter(
423
+ (fn): fn is ExecutablePrivateFunctionWithMembershipProof => 'utilityFunctionsTreeRoot' in fn,
424
+ );
425
+ const utilityFns = allFns.filter(
426
+ (fn): fn is UtilityFunctionWithMembershipProof => 'privateFunctionsArtifactTreeRoot' in fn,
427
+ );
428
+
429
+ const privateFunctionsWithValidity = await Promise.all(
430
+ privateFns.map(async fn => ({ fn, valid: await isValidPrivateFunctionMembershipProof(fn, contractClass) })),
431
+ );
432
+ const validPrivateFns = privateFunctionsWithValidity.filter(({ valid }) => valid).map(({ fn }) => fn);
433
+ const utilityFunctionsWithValidity = await Promise.all(
434
+ utilityFns.map(async fn => ({
435
+ fn,
436
+ valid: await isValidUtilityFunctionMembershipProof(fn, contractClass),
437
+ })),
438
+ );
439
+ const validUtilityFns = utilityFunctionsWithValidity.filter(({ valid }) => valid).map(({ fn }) => fn);
440
+ const validFnCount = validPrivateFns.length + validUtilityFns.length;
441
+ if (validFnCount !== allFns.length) {
442
+ this.log.warn(`Skipping ${allFns.length - validFnCount} invalid functions`);
443
+ }
444
+
445
+ // Store the functions in the contract class in a single operation
446
+ if (validFnCount > 0) {
447
+ this.log.verbose(`Storing ${validFnCount} functions for contract class ${contractClassId.toString()}`);
448
+ }
449
+ return await this.store.addFunctions(contractClassId, validPrivateFns, validUtilityFns);
450
+ }
451
+ return true;
452
+ }
453
+ }
@@ -1,5 +1,9 @@
1
+ import type { SlotNumber } from '@aztec/foundation/branded-types';
1
2
  import { createLogger } from '@aztec/foundation/log';
2
- import type { L2BlockNew } from '@aztec/stdlib/block';
3
+ import type { L2Block } from '@aztec/stdlib/block';
4
+ import type { CheckpointData } from '@aztec/stdlib/checkpoint';
5
+ import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
6
+ import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
3
7
  import {
4
8
  Attributes,
5
9
  type Gauge,
@@ -10,13 +14,14 @@ import {
10
14
  type TelemetryClient,
11
15
  type Tracer,
12
16
  type UpDownCounter,
13
- ValueType,
17
+ createUpDownCounterWithDefault,
14
18
  } from '@aztec/telemetry-client';
15
19
 
16
20
  export class ArchiverInstrumentation {
17
21
  public readonly tracer: Tracer;
18
22
 
19
23
  private blockHeight: Gauge;
24
+ private checkpointHeight: Gauge;
20
25
  private txCount: UpDownCounter;
21
26
  private l1BlockHeight: Gauge;
22
27
  private proofsSubmittedDelay: Histogram;
@@ -36,6 +41,8 @@ export class ArchiverInstrumentation {
36
41
 
37
42
  private blockProposalTxTargetCount: UpDownCounter;
38
43
 
44
+ private checkpointL1InclusionDelay: Histogram;
45
+
39
46
  private log = createLogger('archiver:instrumentation');
40
47
 
41
48
  private constructor(
@@ -45,81 +52,45 @@ export class ArchiverInstrumentation {
45
52
  this.tracer = telemetry.getTracer('Archiver');
46
53
  const meter = telemetry.getMeter('Archiver');
47
54
 
48
- this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT, {
49
- description: 'The height of the latest block processed by the archiver',
50
- valueType: ValueType.INT,
51
- });
55
+ this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT);
52
56
 
53
- this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT, {
54
- description: 'The height of the latest L1 block processed by the archiver',
55
- valueType: ValueType.INT,
56
- });
57
+ this.checkpointHeight = meter.createGauge(Metrics.ARCHIVER_CHECKPOINT_HEIGHT);
57
58
 
58
- this.txCount = meter.createUpDownCounter(Metrics.ARCHIVER_TOTAL_TXS, {
59
- description: 'The total number of transactions',
60
- valueType: ValueType.INT,
61
- });
59
+ this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT);
62
60
 
63
- this.proofsSubmittedCount = meter.createUpDownCounter(Metrics.ARCHIVER_ROLLUP_PROOF_COUNT, {
64
- description: 'Number of proofs submitted',
65
- valueType: ValueType.INT,
66
- });
61
+ this.txCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_TOTAL_TXS);
67
62
 
68
- this.proofsSubmittedDelay = meter.createHistogram(Metrics.ARCHIVER_ROLLUP_PROOF_DELAY, {
69
- unit: 'ms',
70
- description: 'Time after a block is submitted until its proof is published',
71
- valueType: ValueType.INT,
63
+ this.proofsSubmittedCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_ROLLUP_PROOF_COUNT, {
64
+ [Attributes.PROOF_TIMED_OUT]: [true, false],
72
65
  });
73
66
 
74
- this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK, {
75
- unit: 'ms',
76
- description: 'Duration to sync a block',
77
- valueType: ValueType.INT,
78
- });
67
+ this.proofsSubmittedDelay = meter.createHistogram(Metrics.ARCHIVER_ROLLUP_PROOF_DELAY);
79
68
 
80
- this.syncBlockCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_BLOCK_COUNT, {
81
- description: 'Number of blocks synced from L1',
82
- valueType: ValueType.INT,
83
- });
69
+ this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK);
84
70
 
85
- this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK, {
86
- description: 'The mana consumed by blocks',
87
- valueType: ValueType.DOUBLE,
88
- unit: 'Mmana',
89
- });
71
+ this.syncBlockCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
90
72
 
91
- this.txsPerBlock = meter.createHistogram(Metrics.ARCHIVER_TXS_PER_BLOCK, {
92
- description: 'The block tx count',
93
- valueType: ValueType.INT,
94
- unit: 'tx',
95
- });
73
+ this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK);
96
74
 
97
- this.syncDurationPerMessage = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_MESSAGE, {
98
- unit: 'ms',
99
- description: 'Duration to sync a message',
100
- valueType: ValueType.INT,
101
- });
75
+ this.txsPerBlock = meter.createHistogram(Metrics.ARCHIVER_TXS_PER_BLOCK);
102
76
 
103
- this.syncMessageCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_MESSAGE_COUNT, {
104
- description: 'Number of L1 to L2 messages synced',
105
- valueType: ValueType.INT,
106
- });
77
+ this.syncDurationPerMessage = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_MESSAGE);
107
78
 
108
- this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION, {
109
- unit: 'ms',
110
- description: 'Duration to sync a message',
111
- valueType: ValueType.INT,
112
- });
79
+ this.syncMessageCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_MESSAGE_COUNT);
113
80
 
114
- this.pruneCount = meter.createUpDownCounter(Metrics.ARCHIVER_PRUNE_COUNT, {
115
- description: 'Number of prunes detected',
116
- valueType: ValueType.INT,
117
- });
81
+ this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION);
118
82
 
119
- this.blockProposalTxTargetCount = meter.createUpDownCounter(Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT, {
120
- description: 'Number of block proposals by tx target',
121
- valueType: ValueType.INT,
122
- });
83
+ this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT);
84
+
85
+ this.blockProposalTxTargetCount = createUpDownCounterWithDefault(
86
+ meter,
87
+ Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT,
88
+ {
89
+ [Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: [true, false],
90
+ },
91
+ );
92
+
93
+ this.checkpointL1InclusionDelay = meter.createHistogram(Metrics.ARCHIVER_CHECKPOINT_L1_INCLUSION_DELAY);
123
94
 
124
95
  this.dbMetrics = new LmdbMetrics(
125
96
  meter,
@@ -133,10 +104,6 @@ export class ArchiverInstrumentation {
133
104
  public static async new(telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
134
105
  const instance = new ArchiverInstrumentation(telemetry, lmdbStats);
135
106
 
136
- instance.syncBlockCount.add(0);
137
- instance.syncMessageCount.add(0);
138
- instance.pruneCount.add(0);
139
-
140
107
  await instance.telemetry.flush();
141
108
 
142
109
  return instance;
@@ -146,9 +113,10 @@ export class ArchiverInstrumentation {
146
113
  return this.telemetry.isEnabled();
147
114
  }
148
115
 
149
- public processNewBlocks(syncTimePerBlock: number, blocks: L2BlockNew[]) {
116
+ public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) {
150
117
  this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock));
151
118
  this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
119
+ this.checkpointHeight.record(Math.max(...blocks.map(b => b.checkpointNumber)));
152
120
  this.syncBlockCount.add(blocks.length);
153
121
 
154
122
  for (const block of blocks) {
@@ -171,8 +139,10 @@ export class ArchiverInstrumentation {
171
139
  this.pruneDuration.record(Math.ceil(duration));
172
140
  }
173
141
 
174
- public updateLastProvenBlock(blockNumber: number) {
175
- this.blockHeight.record(blockNumber, { [Attributes.STATUS]: 'proven' });
142
+ public updateLastProvenCheckpoint(checkpoint: CheckpointData) {
143
+ const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.blockCount - 1;
144
+ this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' });
145
+ this.checkpointHeight.record(checkpoint.checkpointNumber, { [Attributes.STATUS]: 'proven' });
176
146
  }
177
147
 
178
148
  public processProofsVerified(logs: { proverId: string; l2BlockNumber: bigint; delay: bigint }[]) {
@@ -198,4 +168,17 @@ export class ArchiverInstrumentation {
198
168
  [Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: usedTrace,
199
169
  });
200
170
  }
171
+
172
+ /**
173
+ * Records L1 inclusion timing for a checkpoint observed on L1 (seconds into the L2 slot).
174
+ */
175
+ public processCheckpointL1Timing(data: {
176
+ slotNumber: SlotNumber;
177
+ l1Timestamp: bigint;
178
+ l1Constants: Pick<L1RollupConstants, 'l1GenesisTime' | 'slotDuration'>;
179
+ }): void {
180
+ const slotStartTs = getTimestampForSlot(data.slotNumber, data.l1Constants);
181
+ const inclusionDelaySeconds = Number(data.l1Timestamp - slotStartTs);
182
+ this.checkpointL1InclusionDelay.record(inclusionDelaySeconds);
183
+ }
201
184
  }