@aztec/archiver 0.0.1-commit.9b94fc1 → 0.0.1-commit.9badcec54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/README.md +162 -22
  2. package/dest/archiver.d.ts +140 -0
  3. package/dest/archiver.d.ts.map +1 -0
  4. package/dest/archiver.js +750 -0
  5. package/dest/config.d.ts +30 -0
  6. package/dest/config.d.ts.map +1 -0
  7. package/dest/{archiver/config.js → config.js} +24 -7
  8. package/dest/errors.d.ts +65 -0
  9. package/dest/errors.d.ts.map +1 -0
  10. package/dest/errors.js +91 -0
  11. package/dest/factory.d.ts +8 -7
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +99 -15
  14. package/dest/index.d.ts +11 -4
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +9 -3
  17. package/dest/interfaces.d.ts +9 -0
  18. package/dest/interfaces.d.ts.map +1 -0
  19. package/dest/interfaces.js +3 -0
  20. package/dest/l1/bin/retrieve-calldata.d.ts +3 -0
  21. package/dest/l1/bin/retrieve-calldata.d.ts.map +1 -0
  22. package/dest/l1/bin/retrieve-calldata.js +152 -0
  23. package/dest/l1/calldata_retriever.d.ts +135 -0
  24. package/dest/l1/calldata_retriever.d.ts.map +1 -0
  25. package/dest/l1/calldata_retriever.js +403 -0
  26. package/dest/l1/data_retrieval.d.ts +88 -0
  27. package/dest/l1/data_retrieval.d.ts.map +1 -0
  28. package/dest/{archiver → l1}/data_retrieval.js +87 -160
  29. package/dest/l1/debug_tx.d.ts +19 -0
  30. package/dest/l1/debug_tx.d.ts.map +1 -0
  31. package/dest/l1/debug_tx.js +73 -0
  32. package/dest/l1/spire_proposer.d.ts +70 -0
  33. package/dest/l1/spire_proposer.d.ts.map +1 -0
  34. package/dest/l1/spire_proposer.js +149 -0
  35. package/dest/l1/trace_tx.d.ts +97 -0
  36. package/dest/l1/trace_tx.d.ts.map +1 -0
  37. package/dest/l1/trace_tx.js +91 -0
  38. package/dest/l1/types.d.ts +12 -0
  39. package/dest/l1/types.d.ts.map +1 -0
  40. package/dest/l1/types.js +3 -0
  41. package/dest/l1/validate_trace.d.ts +32 -0
  42. package/dest/l1/validate_trace.d.ts.map +1 -0
  43. package/dest/l1/validate_trace.js +154 -0
  44. package/dest/modules/data_source_base.d.ts +91 -0
  45. package/dest/modules/data_source_base.d.ts.map +1 -0
  46. package/dest/modules/data_source_base.js +222 -0
  47. package/dest/modules/data_store_updater.d.ts +86 -0
  48. package/dest/modules/data_store_updater.d.ts.map +1 -0
  49. package/dest/modules/data_store_updater.js +333 -0
  50. package/dest/modules/instrumentation.d.ts +52 -0
  51. package/dest/modules/instrumentation.d.ts.map +1 -0
  52. package/dest/modules/instrumentation.js +138 -0
  53. package/dest/modules/l1_synchronizer.d.ts +73 -0
  54. package/dest/modules/l1_synchronizer.d.ts.map +1 -0
  55. package/dest/modules/l1_synchronizer.js +1154 -0
  56. package/dest/modules/validation.d.ts +17 -0
  57. package/dest/modules/validation.d.ts.map +1 -0
  58. package/dest/{archiver → modules}/validation.js +9 -3
  59. package/dest/store/block_store.d.ts +230 -0
  60. package/dest/store/block_store.d.ts.map +1 -0
  61. package/dest/store/block_store.js +942 -0
  62. package/dest/store/contract_class_store.d.ts +17 -0
  63. package/dest/store/contract_class_store.d.ts.map +1 -0
  64. package/dest/store/contract_class_store.js +64 -0
  65. package/dest/store/contract_instance_store.d.ts +24 -0
  66. package/dest/store/contract_instance_store.d.ts.map +1 -0
  67. package/dest/{archiver/kv_archiver_store → store}/contract_instance_store.js +7 -3
  68. package/dest/store/kv_archiver_store.d.ts +377 -0
  69. package/dest/store/kv_archiver_store.d.ts.map +1 -0
  70. package/dest/store/kv_archiver_store.js +494 -0
  71. package/dest/store/l2_tips_cache.d.ts +20 -0
  72. package/dest/store/l2_tips_cache.d.ts.map +1 -0
  73. package/dest/store/l2_tips_cache.js +109 -0
  74. package/dest/store/log_store.d.ts +57 -0
  75. package/dest/store/log_store.d.ts.map +1 -0
  76. package/dest/store/log_store.js +531 -0
  77. package/dest/store/message_store.d.ts +44 -0
  78. package/dest/store/message_store.d.ts.map +1 -0
  79. package/dest/{archiver/kv_archiver_store → store}/message_store.js +36 -23
  80. package/dest/{archiver/structs → structs}/data_retrieval.d.ts +1 -1
  81. package/dest/structs/data_retrieval.d.ts.map +1 -0
  82. package/dest/structs/inbox_message.d.ts +15 -0
  83. package/dest/structs/inbox_message.d.ts.map +1 -0
  84. package/dest/{archiver/structs → structs}/inbox_message.js +6 -5
  85. package/dest/structs/published.d.ts +2 -0
  86. package/dest/structs/published.d.ts.map +1 -0
  87. package/dest/test/fake_l1_state.d.ts +210 -0
  88. package/dest/test/fake_l1_state.d.ts.map +1 -0
  89. package/dest/test/fake_l1_state.js +490 -0
  90. package/dest/test/index.d.ts +2 -1
  91. package/dest/test/index.d.ts.map +1 -1
  92. package/dest/test/index.js +4 -1
  93. package/dest/test/mock_archiver.d.ts +16 -8
  94. package/dest/test/mock_archiver.d.ts.map +1 -1
  95. package/dest/test/mock_archiver.js +19 -14
  96. package/dest/test/mock_l1_to_l2_message_source.d.ts +7 -6
  97. package/dest/test/mock_l1_to_l2_message_source.d.ts.map +1 -1
  98. package/dest/test/mock_l1_to_l2_message_source.js +23 -12
  99. package/dest/test/mock_l2_block_source.d.ts +59 -20
  100. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  101. package/dest/test/mock_l2_block_source.js +276 -85
  102. package/dest/test/mock_structs.d.ts +83 -4
  103. package/dest/test/mock_structs.d.ts.map +1 -1
  104. package/dest/test/mock_structs.js +157 -11
  105. package/dest/test/noop_l1_archiver.d.ts +26 -0
  106. package/dest/test/noop_l1_archiver.d.ts.map +1 -0
  107. package/dest/test/noop_l1_archiver.js +71 -0
  108. package/package.json +20 -20
  109. package/src/archiver.ts +510 -0
  110. package/src/{archiver/config.ts → config.ts} +37 -14
  111. package/src/errors.ts +146 -0
  112. package/src/factory.ts +154 -17
  113. package/src/index.ts +11 -3
  114. package/src/interfaces.ts +9 -0
  115. package/src/l1/README.md +55 -0
  116. package/src/l1/bin/retrieve-calldata.ts +194 -0
  117. package/src/l1/calldata_retriever.ts +512 -0
  118. package/src/{archiver → l1}/data_retrieval.ts +146 -233
  119. package/src/l1/debug_tx.ts +99 -0
  120. package/src/l1/spire_proposer.ts +152 -0
  121. package/src/l1/trace_tx.ts +128 -0
  122. package/src/l1/types.ts +13 -0
  123. package/src/l1/validate_trace.ts +229 -0
  124. package/src/modules/data_source_base.ts +347 -0
  125. package/src/modules/data_store_updater.ts +437 -0
  126. package/src/modules/instrumentation.ts +196 -0
  127. package/src/modules/l1_synchronizer.ts +961 -0
  128. package/src/{archiver → modules}/validation.ts +13 -8
  129. package/src/store/block_store.ts +1245 -0
  130. package/src/store/contract_class_store.ts +82 -0
  131. package/src/{archiver/kv_archiver_store → store}/contract_instance_store.ts +10 -7
  132. package/src/store/kv_archiver_store.ts +713 -0
  133. package/src/store/l2_tips_cache.ts +134 -0
  134. package/src/store/log_store.ts +733 -0
  135. package/src/{archiver/kv_archiver_store → store}/message_store.ts +48 -28
  136. package/src/{archiver/structs → structs}/inbox_message.ts +7 -7
  137. package/src/{archiver/structs → structs}/published.ts +0 -1
  138. package/src/test/fake_l1_state.ts +747 -0
  139. package/src/test/fixtures/debug_traceTransaction-multicall3.json +88 -0
  140. package/src/test/fixtures/debug_traceTransaction-multiplePropose.json +153 -0
  141. package/src/test/fixtures/debug_traceTransaction-proxied.json +122 -0
  142. package/src/test/fixtures/trace_transaction-multicall3.json +65 -0
  143. package/src/test/fixtures/trace_transaction-multiplePropose.json +319 -0
  144. package/src/test/fixtures/trace_transaction-proxied.json +128 -0
  145. package/src/test/fixtures/trace_transaction-randomRevert.json +216 -0
  146. package/src/test/index.ts +4 -0
  147. package/src/test/mock_archiver.ts +23 -16
  148. package/src/test/mock_l1_to_l2_message_source.ts +19 -11
  149. package/src/test/mock_l2_block_source.ts +346 -93
  150. package/src/test/mock_structs.ts +289 -13
  151. package/src/test/noop_l1_archiver.ts +114 -0
  152. package/dest/archiver/archiver.d.ts +0 -287
  153. package/dest/archiver/archiver.d.ts.map +0 -1
  154. package/dest/archiver/archiver.js +0 -1408
  155. package/dest/archiver/archiver_store.d.ts +0 -255
  156. package/dest/archiver/archiver_store.d.ts.map +0 -1
  157. package/dest/archiver/archiver_store.js +0 -4
  158. package/dest/archiver/archiver_store_test_suite.d.ts +0 -8
  159. package/dest/archiver/archiver_store_test_suite.d.ts.map +0 -1
  160. package/dest/archiver/archiver_store_test_suite.js +0 -1289
  161. package/dest/archiver/config.d.ts +0 -21
  162. package/dest/archiver/config.d.ts.map +0 -1
  163. package/dest/archiver/data_retrieval.d.ts +0 -79
  164. package/dest/archiver/data_retrieval.d.ts.map +0 -1
  165. package/dest/archiver/errors.d.ts +0 -12
  166. package/dest/archiver/errors.d.ts.map +0 -1
  167. package/dest/archiver/errors.js +0 -17
  168. package/dest/archiver/index.d.ts +0 -7
  169. package/dest/archiver/index.d.ts.map +0 -1
  170. package/dest/archiver/index.js +0 -4
  171. package/dest/archiver/instrumentation.d.ts +0 -35
  172. package/dest/archiver/instrumentation.d.ts.map +0 -1
  173. package/dest/archiver/instrumentation.js +0 -140
  174. package/dest/archiver/kv_archiver_store/block_store.d.ts +0 -124
  175. package/dest/archiver/kv_archiver_store/block_store.d.ts.map +0 -1
  176. package/dest/archiver/kv_archiver_store/block_store.js +0 -370
  177. package/dest/archiver/kv_archiver_store/contract_class_store.d.ts +0 -18
  178. package/dest/archiver/kv_archiver_store/contract_class_store.d.ts.map +0 -1
  179. package/dest/archiver/kv_archiver_store/contract_class_store.js +0 -120
  180. package/dest/archiver/kv_archiver_store/contract_instance_store.d.ts +0 -24
  181. package/dest/archiver/kv_archiver_store/contract_instance_store.d.ts.map +0 -1
  182. package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts +0 -168
  183. package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts.map +0 -1
  184. package/dest/archiver/kv_archiver_store/kv_archiver_store.js +0 -296
  185. package/dest/archiver/kv_archiver_store/log_store.d.ts +0 -49
  186. package/dest/archiver/kv_archiver_store/log_store.d.ts.map +0 -1
  187. package/dest/archiver/kv_archiver_store/log_store.js +0 -336
  188. package/dest/archiver/kv_archiver_store/message_store.d.ts +0 -39
  189. package/dest/archiver/kv_archiver_store/message_store.d.ts.map +0 -1
  190. package/dest/archiver/structs/data_retrieval.d.ts.map +0 -1
  191. package/dest/archiver/structs/inbox_message.d.ts +0 -15
  192. package/dest/archiver/structs/inbox_message.d.ts.map +0 -1
  193. package/dest/archiver/structs/published.d.ts +0 -3
  194. package/dest/archiver/structs/published.d.ts.map +0 -1
  195. package/dest/archiver/validation.d.ts +0 -17
  196. package/dest/archiver/validation.d.ts.map +0 -1
  197. package/dest/rpc/index.d.ts +0 -9
  198. package/dest/rpc/index.d.ts.map +0 -1
  199. package/dest/rpc/index.js +0 -15
  200. package/src/archiver/archiver.ts +0 -1858
  201. package/src/archiver/archiver_store.ts +0 -305
  202. package/src/archiver/archiver_store_test_suite.ts +0 -1264
  203. package/src/archiver/errors.ts +0 -26
  204. package/src/archiver/index.ts +0 -6
  205. package/src/archiver/instrumentation.ts +0 -187
  206. package/src/archiver/kv_archiver_store/block_store.ts +0 -481
  207. package/src/archiver/kv_archiver_store/contract_class_store.ts +0 -176
  208. package/src/archiver/kv_archiver_store/kv_archiver_store.ts +0 -422
  209. package/src/archiver/kv_archiver_store/log_store.ts +0 -406
  210. package/src/rpc/index.ts +0 -16
  211. /package/dest/{archiver/structs → structs}/data_retrieval.js +0 -0
  212. /package/dest/{archiver/structs → structs}/published.js +0 -0
  213. /package/src/{archiver/structs → structs}/data_retrieval.ts +0 -0
@@ -0,0 +1,961 @@
1
+ import type { BlobClientInterface } from '@aztec/blob-client/client';
2
+ import { EpochCache } from '@aztec/epoch-cache';
3
+ import { InboxContract, type InboxContractState, RollupContract } from '@aztec/ethereum/contracts';
4
+ import type { L1BlockId } from '@aztec/ethereum/l1-types';
5
+ import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
6
+ import { asyncPool } from '@aztec/foundation/async-pool';
7
+ import { maxBigint } from '@aztec/foundation/bigint';
8
+ import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
9
+ import { Buffer16, Buffer32 } from '@aztec/foundation/buffer';
10
+ import { pick } from '@aztec/foundation/collection';
11
+ import { Fr } from '@aztec/foundation/curves/bn254';
12
+ import { type Logger, createLogger } from '@aztec/foundation/log';
13
+ import { retryTimes } from '@aztec/foundation/retry';
14
+ import { count } from '@aztec/foundation/string';
15
+ import { DateProvider, Timer, elapsed } from '@aztec/foundation/timer';
16
+ import { isDefined, isErrorClass } from '@aztec/foundation/types';
17
+ import { type ArchiverEmitter, L2BlockSourceEvents, type ValidateCheckpointResult } from '@aztec/stdlib/block';
18
+ import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
19
+ import { type L1RollupConstants, getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers';
20
+ import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
21
+ import { type Traceable, type Tracer, execInSpan, trackSpan } from '@aztec/telemetry-client';
22
+
23
+ import { InitialCheckpointNumberNotSequentialError } from '../errors.js';
24
+ import {
25
+ retrieveCheckpointsFromRollup,
26
+ retrieveL1ToL2Message,
27
+ retrieveL1ToL2Messages,
28
+ retrievedToPublishedCheckpoint,
29
+ } from '../l1/data_retrieval.js';
30
+ import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
31
+ import type { L2TipsCache } from '../store/l2_tips_cache.js';
32
+ import { MessageStoreError } from '../store/message_store.js';
33
+ import type { InboxMessage } from '../structs/inbox_message.js';
34
+ import { ArchiverDataStoreUpdater } from './data_store_updater.js';
35
+ import type { ArchiverInstrumentation } from './instrumentation.js';
36
+ import { validateCheckpointAttestations } from './validation.js';
37
+
38
+ type RollupStatus = {
39
+ provenCheckpointNumber: CheckpointNumber;
40
+ provenArchive: string;
41
+ pendingCheckpointNumber: CheckpointNumber;
42
+ pendingArchive: string;
43
+ validationResult: ValidateCheckpointResult | undefined;
44
+ lastRetrievedCheckpoint?: PublishedCheckpoint;
45
+ lastL1BlockWithCheckpoint?: bigint;
46
+ };
47
+
48
+ /**
49
+ * Handles L1 synchronization for the archiver.
50
+ * Responsible for fetching checkpoints, L1→L2 messages, and handling L1 reorgs.
51
+ */
52
+ export class ArchiverL1Synchronizer implements Traceable {
53
+ private l1BlockNumber: bigint | undefined;
54
+ private l1BlockHash: Buffer32 | undefined;
55
+ private l1Timestamp: bigint | undefined;
56
+
57
+ private readonly updater: ArchiverDataStoreUpdater;
58
+ public readonly tracer: Tracer;
59
+
60
+ constructor(
61
+ private readonly publicClient: ViemPublicClient,
62
+ private readonly debugClient: ViemPublicDebugClient,
63
+ private readonly rollup: RollupContract,
64
+ private readonly inbox: InboxContract,
65
+ private readonly store: KVArchiverDataStore,
66
+ private config: {
67
+ batchSize: number;
68
+ skipValidateCheckpointAttestations?: boolean;
69
+ maxAllowedEthClientDriftSeconds: number;
70
+ },
71
+ private readonly blobClient: BlobClientInterface,
72
+ private readonly epochCache: EpochCache,
73
+ private readonly dateProvider: DateProvider,
74
+ private readonly instrumentation: ArchiverInstrumentation,
75
+ private readonly l1Constants: L1RollupConstants & {
76
+ l1StartBlockHash: Buffer32;
77
+ genesisArchiveRoot: Fr;
78
+ },
79
+ private readonly events: ArchiverEmitter,
80
+ tracer: Tracer,
81
+ l2TipsCache?: L2TipsCache,
82
+ private readonly log: Logger = createLogger('archiver:l1-sync'),
83
+ ) {
84
+ this.updater = new ArchiverDataStoreUpdater(this.store, l2TipsCache, {
85
+ rollupManaLimit: l1Constants.rollupManaLimit,
86
+ });
87
+ this.tracer = tracer;
88
+ }
89
+
90
+ /** Sets new config */
91
+ public setConfig(newConfig: {
92
+ batchSize: number;
93
+ skipValidateCheckpointAttestations?: boolean;
94
+ maxAllowedEthClientDriftSeconds: number;
95
+ }) {
96
+ this.config = newConfig;
97
+ }
98
+
99
+ /** Returns the last L1 block number that was synced. */
100
+ public getL1BlockNumber(): bigint | undefined {
101
+ return this.l1BlockNumber;
102
+ }
103
+
104
+ /** Returns the last L1 timestamp that was synced. */
105
+ public getL1Timestamp(): bigint | undefined {
106
+ return this.l1Timestamp;
107
+ }
108
+
109
+ /** Checks that the ethereum node we are connected to has a latest timestamp no more than the allowed drift. Throw if not. */
110
+ public async testEthereumNodeSynced(): Promise<void> {
111
+ const maxAllowedDelay = this.config.maxAllowedEthClientDriftSeconds;
112
+ if (maxAllowedDelay === 0) {
113
+ return;
114
+ }
115
+ const { number, timestamp: l1Timestamp } = await this.publicClient.getBlock({ includeTransactions: false });
116
+ const currentTime = BigInt(this.dateProvider.nowInSeconds());
117
+ if (currentTime - l1Timestamp > BigInt(maxAllowedDelay)) {
118
+ throw new Error(
119
+ `Ethereum node is out of sync (last block synced ${number} at ${l1Timestamp} vs current time ${currentTime})`,
120
+ );
121
+ }
122
+ }
123
+
124
+ @trackSpan('Archiver.syncFromL1')
125
+ public async syncFromL1(initialSyncComplete: boolean): Promise<void> {
126
+ // In between the various calls to L1, the block number can move meaning some of the following
127
+ // calls will return data for blocks that were not present during earlier calls. To combat this
128
+ // we ensure that all data retrieval methods only retrieve data up to the currentBlockNumber
129
+ // captured at the top of this function.
130
+ const currentL1Block = await this.publicClient.getBlock({ includeTransactions: false });
131
+ const currentL1BlockNumber = currentL1Block.number;
132
+ const currentL1BlockHash = Buffer32.fromString(currentL1Block.hash);
133
+ const currentL1Timestamp = currentL1Block.timestamp;
134
+ const currentL1BlockData = { l1BlockNumber: currentL1BlockNumber, l1BlockHash: currentL1BlockHash };
135
+
136
+ if (this.l1BlockHash && currentL1BlockHash.equals(this.l1BlockHash)) {
137
+ this.log.trace(`No new L1 blocks since last sync at L1 block ${this.l1BlockNumber}`);
138
+ return;
139
+ }
140
+
141
+ // Warn if the latest L1 block timestamp is too old
142
+ const maxAllowedDelay = this.config.maxAllowedEthClientDriftSeconds;
143
+ const now = this.dateProvider.nowInSeconds();
144
+ if (maxAllowedDelay > 0 && Number(currentL1Timestamp) <= now - maxAllowedDelay) {
145
+ this.log.warn(
146
+ `Latest L1 block ${currentL1BlockNumber} timestamp ${currentL1Timestamp} is too old. Make sure your Ethereum node is synced.`,
147
+ { currentL1BlockNumber, currentL1Timestamp, now, maxAllowedDelay },
148
+ );
149
+ }
150
+
151
+ // Load sync point for blocks defaulting to start block
152
+ const { blocksSynchedTo = this.l1Constants.l1StartBlock } = await this.store.getSynchPoint();
153
+ this.log.debug(`Starting new archiver sync iteration`, { blocksSynchedTo, currentL1BlockData });
154
+
155
+ // Sync L1 to L2 messages. We retry this a few times since there are error conditions that reset the sync point, requiring a new iteration.
156
+ // Note that we cannot just wait for the l1 synchronizer to loop again, since the synchronizer would report as synced up to the current L1
157
+ // block, when that wouldn't be the case, since L1 to L2 messages would need another iteration.
158
+ await retryTimes(() => this.handleL1ToL2Messages(currentL1BlockData), 'Handling L1 to L2 messages', 3, 0.1);
159
+
160
+ if (currentL1BlockNumber > blocksSynchedTo) {
161
+ // First we retrieve new checkpoints and L2 blocks and store them in the DB. This will also update the
162
+ // pending chain validation status, proven checkpoint number, and synched L1 block number.
163
+ const rollupStatus = await this.handleCheckpoints(blocksSynchedTo, currentL1BlockNumber, initialSyncComplete);
164
+
165
+ // Then we try pruning uncheckpointed blocks if a new slot was mined without checkpoints
166
+ await this.pruneUncheckpointedBlocks(currentL1Timestamp);
167
+
168
+ // Then we prune the current epoch if it'd reorg on next submission.
169
+ // Note that we don't do this before retrieving checkpoints because we may need to retrieve
170
+ // checkpoints from more than 2 epochs ago, so we want to make sure we have the latest view of
171
+ // the chain locally before we start unwinding stuff. This can be optimized by figuring out
172
+ // up to which point we're pruning, and then requesting checkpoints up to that point only.
173
+ const { rollupCanPrune } = await this.handleEpochPrune(
174
+ rollupStatus.provenCheckpointNumber,
175
+ currentL1BlockNumber,
176
+ currentL1Timestamp,
177
+ );
178
+
179
+ // If the last checkpoint we processed had an invalid attestation, we manually advance the L1 syncpoint
180
+ // past it, since otherwise we'll keep downloading it and reprocessing it on every iteration until
181
+ // we get a valid checkpoint to advance the syncpoint.
182
+ if (!rollupStatus.validationResult?.valid && rollupStatus.lastL1BlockWithCheckpoint !== undefined) {
183
+ await this.store.setCheckpointSynchedL1BlockNumber(rollupStatus.lastL1BlockWithCheckpoint);
184
+ }
185
+
186
+ // And lastly we check if we are missing any checkpoints behind us due to a possible L1 reorg.
187
+ // We only do this if rollup cant prune on the next submission. Otherwise we will end up
188
+ // re-syncing the checkpoints we have just unwound above. We also dont do this if the last checkpoint is invalid,
189
+ // since the archiver will rightfully refuse to sync up to it.
190
+ if (!rollupCanPrune && rollupStatus.validationResult?.valid) {
191
+ await this.checkForNewCheckpointsBeforeL1SyncPoint(rollupStatus, blocksSynchedTo, currentL1BlockNumber);
192
+ }
193
+
194
+ this.instrumentation.updateL1BlockHeight(currentL1BlockNumber);
195
+ }
196
+
197
+ // Update the finalized L2 checkpoint based on L1 finality.
198
+ await this.updateFinalizedCheckpoint();
199
+
200
+ // After syncing has completed, update the current l1 block number and timestamp,
201
+ // otherwise we risk announcing to the world that we've synced to a given point,
202
+ // but the corresponding blocks have not been processed (see #12631).
203
+ this.l1Timestamp = currentL1Timestamp;
204
+ this.l1BlockNumber = currentL1BlockNumber;
205
+ this.l1BlockHash = currentL1BlockHash;
206
+
207
+ const l1BlockNumberAtEnd = await this.publicClient.getBlockNumber();
208
+ this.log.debug(`Archiver sync iteration complete`, {
209
+ l1BlockNumberAtStart: currentL1BlockNumber,
210
+ l1TimestampAtStart: currentL1Timestamp,
211
+ l1BlockNumberAtEnd,
212
+ });
213
+ }
214
+
215
+ /** Query L1 for its finalized block and update the finalized checkpoint accordingly. */
216
+ private async updateFinalizedCheckpoint(): Promise<void> {
217
+ try {
218
+ const finalizedL1Block = await this.publicClient.getBlock({ blockTag: 'finalized', includeTransactions: false });
219
+ const finalizedL1BlockNumber = finalizedL1Block.number;
220
+ const finalizedCheckpointNumber = await this.rollup.getProvenCheckpointNumber({
221
+ blockNumber: finalizedL1BlockNumber,
222
+ });
223
+ const localFinalizedCheckpointNumber = await this.store.getFinalizedCheckpointNumber();
224
+ if (localFinalizedCheckpointNumber !== finalizedCheckpointNumber) {
225
+ await this.updater.setFinalizedCheckpointNumber(finalizedCheckpointNumber);
226
+ this.log.info(`Updated finalized chain to checkpoint ${finalizedCheckpointNumber}`, {
227
+ finalizedCheckpointNumber,
228
+ finalizedL1BlockNumber,
229
+ });
230
+ }
231
+ } catch (err: any) {
232
+ // The rollup contract may not exist at the finalized L1 block right after deployment.
233
+ if (!err?.message?.includes('returned no data')) {
234
+ this.log.warn(`Failed to update finalized checkpoint: ${err}`);
235
+ }
236
+ }
237
+ }
238
+
239
+ /** Prune all proposed local blocks that should have been checkpointed by now. */
240
+ private async pruneUncheckpointedBlocks(currentL1Timestamp: bigint) {
241
+ const [lastCheckpointedBlockNumber, lastProposedBlockNumber] = await Promise.all([
242
+ this.store.getCheckpointedL2BlockNumber(),
243
+ this.store.getLatestBlockNumber(),
244
+ ]);
245
+
246
+ // If there are no uncheckpointed blocks, we got nothing to do
247
+ if (lastProposedBlockNumber === lastCheckpointedBlockNumber) {
248
+ this.log.trace(`No uncheckpointed blocks to prune.`);
249
+ return;
250
+ }
251
+
252
+ // What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
253
+ const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
254
+ const firstUncheckpointedBlockNumber = BlockNumber(lastCheckpointedBlockNumber + 1);
255
+
256
+ // What's the slot of the first uncheckpointed block?
257
+ const [firstUncheckpointedBlockHeader] = await this.store.getBlockHeaders(firstUncheckpointedBlockNumber, 1);
258
+ const firstUncheckpointedBlockSlot = firstUncheckpointedBlockHeader?.getSlot();
259
+
260
+ if (firstUncheckpointedBlockSlot === undefined || firstUncheckpointedBlockSlot >= slotAtNextL1Block) {
261
+ return;
262
+ }
263
+
264
+ // Prune provisional blocks from slots that have ended without being checkpointed.
265
+ // This also clears any proposed checkpoint whose blocks are being pruned.
266
+ this.log.warn(
267
+ `Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`,
268
+ { firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(), slotAtNextL1Block },
269
+ );
270
+
271
+ const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
272
+
273
+ if (prunedBlocks.length > 0) {
274
+ this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
275
+ type: L2BlockSourceEvents.L2PruneUncheckpointed,
276
+ slotNumber: firstUncheckpointedBlockSlot,
277
+ blocks: prunedBlocks,
278
+ });
279
+ }
280
+ }
281
+
282
+ /** Queries the rollup contract on whether a prune can be executed on the immediate next L1 block. */
283
+ private async canPrune(currentL1BlockNumber: bigint, currentL1Timestamp: bigint): Promise<boolean> {
284
+ const time = (currentL1Timestamp ?? 0n) + BigInt(this.l1Constants.ethereumSlotDuration);
285
+ const result = await this.rollup.canPruneAtTime(time, { blockNumber: currentL1BlockNumber });
286
+ if (result) {
287
+ this.log.debug(`Rollup contract allows pruning at L1 block ${currentL1BlockNumber} time ${time}`, {
288
+ currentL1Timestamp,
289
+ pruneTime: time,
290
+ currentL1BlockNumber,
291
+ });
292
+ }
293
+ return result;
294
+ }
295
+
296
+ /** Checks if there'd be a reorg for the next checkpoint submission and start pruning now. */
297
+ @trackSpan('Archiver.handleEpochPrune')
298
+ private async handleEpochPrune(
299
+ provenCheckpointNumber: CheckpointNumber,
300
+ currentL1BlockNumber: bigint,
301
+ currentL1Timestamp: bigint,
302
+ ): Promise<{ rollupCanPrune: boolean }> {
303
+ const rollupCanPrune = await this.canPrune(currentL1BlockNumber, currentL1Timestamp);
304
+ const localPendingCheckpointNumber = await this.store.getSynchedCheckpointNumber();
305
+ const canPrune = localPendingCheckpointNumber > provenCheckpointNumber && rollupCanPrune;
306
+
307
+ if (canPrune) {
308
+ const timer = new Timer();
309
+ const pruneFrom = CheckpointNumber(provenCheckpointNumber + 1);
310
+
311
+ const header = await this.getCheckpointHeader(pruneFrom);
312
+ if (header === undefined) {
313
+ throw new Error(`Missing checkpoint header ${pruneFrom}`);
314
+ }
315
+
316
+ const pruneFromSlotNumber = header.slotNumber;
317
+ const pruneFromEpochNumber: EpochNumber = getEpochAtSlot(pruneFromSlotNumber, this.l1Constants);
318
+
319
+ const checkpointsToUnwind = localPendingCheckpointNumber - provenCheckpointNumber;
320
+
321
+ // Fetch checkpoints and blocks in bounded batches to avoid unbounded concurrent
322
+ // promises when the gap between local pending and proven checkpoint numbers is large.
323
+ const BATCH_SIZE = 10;
324
+ const indices = Array.from({ length: checkpointsToUnwind }, (_, i) => CheckpointNumber(i + pruneFrom));
325
+ const checkpoints = (await asyncPool(BATCH_SIZE, indices, idx => this.store.getCheckpointData(idx))).filter(
326
+ isDefined,
327
+ );
328
+ const newBlocks = (
329
+ await asyncPool(BATCH_SIZE, checkpoints, cp =>
330
+ this.store.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber)),
331
+ )
332
+ )
333
+ .filter(isDefined)
334
+ .flat();
335
+
336
+ // Emit an event for listening services to react to the chain prune
337
+ this.events.emit(L2BlockSourceEvents.L2PruneUnproven, {
338
+ type: L2BlockSourceEvents.L2PruneUnproven,
339
+ epochNumber: pruneFromEpochNumber,
340
+ blocks: newBlocks,
341
+ });
342
+
343
+ this.log.debug(
344
+ `L2 prune from ${provenCheckpointNumber + 1} to ${localPendingCheckpointNumber} will occur on next checkpoint submission.`,
345
+ );
346
+ await this.updater.removeCheckpointsAfter(provenCheckpointNumber);
347
+ this.log.warn(
348
+ `Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` +
349
+ `due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
350
+ `Updated latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
351
+ );
352
+ this.instrumentation.processPrune(timer.ms());
353
+ // TODO(palla/reorg): Do we need to set the block synched L1 block number here?
354
+ // Seems like the next iteration should handle this.
355
+ // await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
356
+ }
357
+
358
+ return { rollupCanPrune };
359
+ }
360
+
361
+ private nextRange(end: bigint, limit: bigint): [bigint, bigint] {
362
+ const batchSize = (this.config.batchSize * this.l1Constants.slotDuration) / this.l1Constants.ethereumSlotDuration;
363
+ const nextStart = end + 1n;
364
+ const nextEnd = nextStart + BigInt(batchSize);
365
+ if (nextEnd > limit) {
366
+ return [nextStart, limit];
367
+ }
368
+ return [nextStart, nextEnd];
369
+ }
370
+
371
+ @trackSpan('Archiver.handleL1ToL2Messages')
372
+ private async handleL1ToL2Messages(currentL1Block: L1BlockId): Promise<boolean> {
373
+ // Load the syncpoint, which may have been updated in a previous iteration
374
+ const {
375
+ messagesSynchedTo = {
376
+ l1BlockNumber: this.l1Constants.l1StartBlock,
377
+ l1BlockHash: this.l1Constants.l1StartBlockHash,
378
+ },
379
+ } = await this.store.getSynchPoint();
380
+
381
+ // Nothing to do if L1 block number has not moved forward
382
+ const currentL1BlockNumber = currentL1Block.l1BlockNumber;
383
+ if (currentL1BlockNumber <= messagesSynchedTo.l1BlockNumber) {
384
+ return true;
385
+ }
386
+
387
+ // Compare local message store state with the remote. If they match, we just advance the match pointer.
388
+ const remoteMessagesState = await this.inbox.getState({ blockNumber: currentL1BlockNumber });
389
+ const localLastMessage = await this.store.getLastL1ToL2Message();
390
+ if (await this.localStateMatches(localLastMessage, remoteMessagesState)) {
391
+ this.log.trace(`Local L1 to L2 messages are already in sync with remote at L1 block ${currentL1BlockNumber}`);
392
+ await this.store.setMessageSyncState(currentL1Block, remoteMessagesState.treeInProgress);
393
+ return true;
394
+ }
395
+
396
+ // If not, then we are out of sync. Most likely there are new messages on the inbox, so we try retrieving them.
397
+ // However, it could also be the case that there was an L1 reorg and our syncpoint is no longer valid.
398
+ // If that's the case, we'd get an exception out of the message store since the rolling hash of the first message
399
+ // we try to insert would not match the one in the db, in which case we rollback to the last common message with L1.
400
+ try {
401
+ await this.retrieveAndStoreMessages(messagesSynchedTo.l1BlockNumber, currentL1BlockNumber);
402
+ } catch (error) {
403
+ if (isErrorClass(error, MessageStoreError)) {
404
+ this.log.warn(
405
+ `Failed to store L1 to L2 messages retrieved from L1: ${error.message}. Rolling back syncpoint to retry.`,
406
+ { inboxMessage: error.inboxMessage },
407
+ );
408
+ await this.rollbackL1ToL2Messages(remoteMessagesState.treeInProgress);
409
+ return false;
410
+ }
411
+ throw error;
412
+ }
413
+
414
+ // Note that, if there are no new messages to insert, but there was an L1 reorg that pruned out last messages,
415
+ // we'd notice by comparing our local state with the remote one again, and seeing they don't match even after
416
+ // our sync attempt. In this case, we also rollback our syncpoint, and trigger a retry.
417
+ const localLastMessageAfterSync = await this.store.getLastL1ToL2Message();
418
+ if (!(await this.localStateMatches(localLastMessageAfterSync, remoteMessagesState))) {
419
+ this.log.warn(
420
+ `Local L1 to L2 messages state does not match remote after sync attempt. Rolling back syncpoint to retry.`,
421
+ { localLastMessageAfterSync, remoteMessagesState },
422
+ );
423
+ await this.rollbackL1ToL2Messages(remoteMessagesState.treeInProgress);
424
+ return false;
425
+ }
426
+
427
+ // Advance the syncpoint after a successful sync
428
+ await this.store.setMessageSyncState(currentL1Block, remoteMessagesState.treeInProgress);
429
+ return true;
430
+ }
431
+
432
+ /** Checks if the local rolling hash and message count matches the remote state */
433
+ private async localStateMatches(localLastMessage: InboxMessage | undefined, remoteState: InboxContractState) {
434
+ const localMessageCount = await this.store.getTotalL1ToL2MessageCount();
435
+ this.log.trace(`Comparing local and remote inbox state`, { localMessageCount, localLastMessage, remoteState });
436
+
437
+ return (
438
+ remoteState.totalMessagesInserted === localMessageCount &&
439
+ remoteState.messagesRollingHash.equals(localLastMessage?.rollingHash ?? Buffer16.ZERO)
440
+ );
441
+ }
442
+
443
+ /** Retrieves L1 to L2 messages from L1 in batches and stores them. */
444
+ private async retrieveAndStoreMessages(fromL1Block: bigint, toL1Block: bigint): Promise<void> {
445
+ let searchStartBlock: bigint = 0n;
446
+ let searchEndBlock: bigint = fromL1Block;
447
+
448
+ let lastMessage: InboxMessage | undefined;
449
+ let messageCount = 0;
450
+
451
+ do {
452
+ [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, toL1Block);
453
+ this.log.trace(`Retrieving L1 to L2 messages in L1 blocks ${searchStartBlock}-${searchEndBlock}`);
454
+ const messages = await retrieveL1ToL2Messages(this.inbox, searchStartBlock, searchEndBlock);
455
+ const timer = new Timer();
456
+ await this.store.addL1ToL2Messages(messages);
457
+ const perMsg = timer.ms() / messages.length;
458
+ this.instrumentation.processNewMessages(messages.length, perMsg);
459
+ for (const msg of messages) {
460
+ this.log.debug(`Downloaded L1 to L2 message`, { ...msg, leaf: msg.leaf.toString() });
461
+ lastMessage = msg;
462
+ messageCount++;
463
+ }
464
+ } while (searchEndBlock < toL1Block);
465
+
466
+ if (messageCount > 0) {
467
+ this.log.info(
468
+ `Retrieved ${messageCount} new L1 to L2 messages up to message with index ${lastMessage?.index} for checkpoint ${lastMessage?.checkpointNumber}`,
469
+ { lastMessage, messageCount },
470
+ );
471
+ }
472
+ }
473
+
474
+ /**
475
+ * Rolls back local L1 to L2 messages to the last common message with L1, and updates the syncpoint to the L1 block of that message.
476
+ * If no common message is found, rolls back all messages and sets the syncpoint to the start block.
477
+ */
478
+ private async rollbackL1ToL2Messages(remoteTreeInProgress: bigint): Promise<L1BlockId> {
479
+ // Slowly go back through our messages until we find the last common message.
480
+ // We could query the logs in batch as an optimization, but the depth of the reorg should not be deep, and this
481
+ // is a very rare case, so it's fine to query one log at a time.
482
+ let commonMsg: undefined | InboxMessage;
483
+ let messagesToDelete = 0;
484
+ this.log.verbose(`Searching most recent common L1 to L2 message`);
485
+ for await (const localMsg of this.store.iterateL1ToL2Messages({ reverse: true })) {
486
+ const remoteMsg = await retrieveL1ToL2Message(this.inbox, localMsg);
487
+ const logCtx = { remoteMsg, localMsg: localMsg };
488
+ if (remoteMsg && remoteMsg.rollingHash.equals(localMsg.rollingHash)) {
489
+ this.log.verbose(
490
+ `Found most recent common L1 to L2 message at index ${localMsg.index} on L1 block ${localMsg.l1BlockNumber}`,
491
+ logCtx,
492
+ );
493
+ commonMsg = remoteMsg;
494
+ break;
495
+ } else if (remoteMsg) {
496
+ this.log.debug(`Local L1 to L2 message with index ${localMsg.index} has different rolling hash`, logCtx);
497
+ messagesToDelete++;
498
+ } else {
499
+ this.log.debug(`Local L1 to L2 message with index ${localMsg.index} not found on L1`, logCtx);
500
+ messagesToDelete++;
501
+ }
502
+ }
503
+
504
+ // Delete everything after the common message we found, if anything needs to be deleted.
505
+ // Do not exit early if there are no messages to delete, we still want to update the syncpoint.
506
+ if (messagesToDelete > 0) {
507
+ const lastGoodIndex = commonMsg?.index;
508
+ this.log.warn(`Rolling back all local L1 to L2 messages after index ${lastGoodIndex ?? 'initial'}`);
509
+ await this.store.removeL1ToL2Messages(lastGoodIndex !== undefined ? lastGoodIndex + 1n : 0n);
510
+ }
511
+
512
+ // Update the syncpoint so the loop below reprocesses the changed messages. We go to the block before
513
+ // the last common one, so we force reprocessing it, in case new messages were added on that same L1 block
514
+ // after the last common message.
515
+ const syncPointL1BlockNumber = commonMsg ? commonMsg.l1BlockNumber - 1n : this.l1Constants.l1StartBlock;
516
+ const syncPointL1BlockHash = await this.getL1BlockHash(syncPointL1BlockNumber);
517
+ const messagesSyncPoint = { l1BlockNumber: syncPointL1BlockNumber, l1BlockHash: syncPointL1BlockHash };
518
+ await this.store.setMessageSyncState(messagesSyncPoint, remoteTreeInProgress);
519
+ this.log.verbose(`Updated messages syncpoint to L1 block ${syncPointL1BlockNumber}`, {
520
+ ...messagesSyncPoint,
521
+ remoteTreeInProgress,
522
+ });
523
+ return messagesSyncPoint;
524
+ }
525
+
526
+ private async getL1BlockHash(l1BlockNumber: bigint): Promise<Buffer32> {
527
+ const block = await this.publicClient.getBlock({ blockNumber: l1BlockNumber, includeTransactions: false });
528
+ if (!block) {
529
+ throw new Error(`Missing L1 block ${l1BlockNumber}`);
530
+ }
531
+ return Buffer32.fromString(block.hash);
532
+ }
533
+
534
+ @trackSpan('Archiver.handleCheckpoints')
535
+ private async handleCheckpoints(
536
+ blocksSynchedTo: bigint,
537
+ currentL1BlockNumber: bigint,
538
+ initialSyncComplete: boolean,
539
+ ): Promise<RollupStatus> {
540
+ const localPendingCheckpointNumber = await this.store.getSynchedCheckpointNumber();
541
+ const initialValidationResult: ValidateCheckpointResult | undefined =
542
+ await this.store.getPendingChainValidationStatus();
543
+ const {
544
+ provenCheckpointNumber,
545
+ provenArchive,
546
+ pendingCheckpointNumber,
547
+ pendingArchive,
548
+ archiveOfMyCheckpoint: archiveForLocalPendingCheckpointNumber,
549
+ } = await execInSpan(this.tracer, 'Archiver.getRollupStatus', () =>
550
+ this.rollup.status(localPendingCheckpointNumber, { blockNumber: currentL1BlockNumber }),
551
+ );
552
+ const rollupStatus: RollupStatus = {
553
+ provenCheckpointNumber,
554
+ provenArchive: provenArchive.toString(),
555
+ pendingCheckpointNumber,
556
+ pendingArchive: pendingArchive.toString(),
557
+ validationResult: initialValidationResult,
558
+ };
559
+ this.log.trace(`Retrieved rollup status at current L1 block ${currentL1BlockNumber}.`, {
560
+ localPendingCheckpointNumber,
561
+ blocksSynchedTo,
562
+ currentL1BlockNumber,
563
+ archiveForLocalPendingCheckpointNumber,
564
+ ...rollupStatus,
565
+ });
566
+
567
+ const updateProvenCheckpoint = async () => {
568
+ // Annoying edge case: if proven checkpoint is moved back to 0 due to a reorg at the beginning of the chain,
569
+ // we need to set it to zero. This is an edge case because we dont have a checkpoint zero (initial checkpoint is one),
570
+ // so localCheckpointForDestinationProvenCheckpointNumber would not be found below.
571
+ if (provenCheckpointNumber === 0) {
572
+ const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
573
+ if (localProvenCheckpointNumber !== provenCheckpointNumber) {
574
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
575
+ this.log.info(`Rolled back proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
576
+ }
577
+ }
578
+
579
+ const localCheckpointForDestinationProvenCheckpointNumber =
580
+ await this.store.getCheckpointData(provenCheckpointNumber);
581
+
582
+ // Sanity check. I've hit what seems to be a state where the proven checkpoint is set to a value greater than the latest
583
+ // synched checkpoint when requesting L2Tips from the archiver. This is the only place where the proven checkpoint is set.
584
+ const synched = await this.store.getSynchedCheckpointNumber();
585
+ if (
586
+ localCheckpointForDestinationProvenCheckpointNumber &&
587
+ synched < localCheckpointForDestinationProvenCheckpointNumber.checkpointNumber
588
+ ) {
589
+ this.log.error(
590
+ `Hit local checkpoint greater than last synched checkpoint: ${localCheckpointForDestinationProvenCheckpointNumber.checkpointNumber} > ${synched}`,
591
+ );
592
+ }
593
+
594
+ this.log.trace(
595
+ `Local checkpoint for remote proven checkpoint ${provenCheckpointNumber} is ${
596
+ localCheckpointForDestinationProvenCheckpointNumber?.archive.root.toString() ?? 'undefined'
597
+ }`,
598
+ );
599
+
600
+ if (
601
+ localCheckpointForDestinationProvenCheckpointNumber &&
602
+ provenArchive.equals(localCheckpointForDestinationProvenCheckpointNumber.archive.root)
603
+ ) {
604
+ const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
605
+ if (localProvenCheckpointNumber !== provenCheckpointNumber) {
606
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
607
+ this.log.info(`Updated proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
608
+ const provenSlotNumber = localCheckpointForDestinationProvenCheckpointNumber.header.slotNumber;
609
+ const provenEpochNumber: EpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants);
610
+ const lastBlockNumberInCheckpoint =
611
+ localCheckpointForDestinationProvenCheckpointNumber.startBlock +
612
+ localCheckpointForDestinationProvenCheckpointNumber.blockCount -
613
+ 1;
614
+
615
+ this.events.emit(L2BlockSourceEvents.L2BlockProven, {
616
+ type: L2BlockSourceEvents.L2BlockProven,
617
+ blockNumber: BlockNumber(lastBlockNumberInCheckpoint),
618
+ slotNumber: provenSlotNumber,
619
+ epochNumber: provenEpochNumber,
620
+ });
621
+ this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber);
622
+ } else {
623
+ this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`);
624
+ }
625
+ }
626
+ };
627
+
628
+ // This is an edge case that we only hit if there are no proposed checkpoints.
629
+ // If we have 0 checkpoints locally and there are no checkpoints onchain there is nothing to do.
630
+ const noCheckpoints = localPendingCheckpointNumber === 0 && pendingCheckpointNumber === 0;
631
+ if (noCheckpoints) {
632
+ await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
633
+ this.log.debug(
634
+ `No checkpoints to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}, no checkpoints on chain`,
635
+ );
636
+ return rollupStatus;
637
+ }
638
+
639
+ await updateProvenCheckpoint();
640
+
641
+ // Related to the L2 reorgs of the pending chain. We are only interested in actually addressing a reorg if there
642
+ // are any state that could be impacted by it. If we have no checkpoints, there is no impact.
643
+ if (localPendingCheckpointNumber > 0) {
644
+ const localPendingCheckpoint = await this.store.getCheckpointData(localPendingCheckpointNumber);
645
+ if (localPendingCheckpoint === undefined) {
646
+ throw new Error(`Missing checkpoint ${localPendingCheckpointNumber}`);
647
+ }
648
+
649
+ const localPendingArchiveRoot = localPendingCheckpoint.archive.root.toString();
650
+ const noCheckpointSinceLast = localPendingCheckpoint && pendingArchive.toString() === localPendingArchiveRoot;
651
+ if (noCheckpointSinceLast) {
652
+ // We believe the following line causes a problem when we encounter L1 re-orgs.
653
+ // Basically, by setting the synched L1 block number here, we are saying that we have
654
+ // processed all checkpoints up to the current L1 block number and we will not attempt to retrieve logs from
655
+ // this block again (or any blocks before).
656
+ // However, in the re-org scenario, our L1 node is temporarily lying to us and we end up potentially missing checkpoints.
657
+ // We must only set this block number based on actually retrieved logs.
658
+ // TODO(#8621): Tackle this properly when we handle L1 Re-orgs.
659
+ // await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
660
+ this.log.debug(`No checkpoints to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}`);
661
+ return rollupStatus;
662
+ }
663
+
664
+ const localPendingCheckpointInChain = archiveForLocalPendingCheckpointNumber.equals(
665
+ localPendingCheckpoint.archive.root,
666
+ );
667
+ if (!localPendingCheckpointInChain) {
668
+ // If our local pending checkpoint tip is not in the chain on L1 a "prune" must have happened
669
+ // or the L1 have reorged.
670
+ // In any case, we have to figure out how far into the past the action will take us.
671
+ // For simplicity here, we will simply rewind until we end in a checkpoint that is also on the chain on L1.
672
+ this.log.debug(
673
+ `L2 prune has been detected due to local pending checkpoint ${localPendingCheckpointNumber} not in chain`,
674
+ { localPendingCheckpointNumber, localPendingArchiveRoot, archiveForLocalPendingCheckpointNumber },
675
+ );
676
+
677
+ let tipAfterUnwind = localPendingCheckpointNumber;
678
+ while (true) {
679
+ const candidateCheckpoint = await this.store.getCheckpointData(tipAfterUnwind);
680
+ if (candidateCheckpoint === undefined) {
681
+ break;
682
+ }
683
+
684
+ const archiveAtContract = await this.rollup.archiveAt(candidateCheckpoint.checkpointNumber);
685
+ this.log.trace(
686
+ `Checking local checkpoint ${candidateCheckpoint.checkpointNumber} with archive ${candidateCheckpoint.archive.root}`,
687
+ {
688
+ archiveAtContract,
689
+ archiveLocal: candidateCheckpoint.archive.root.toString(),
690
+ },
691
+ );
692
+ if (archiveAtContract.equals(candidateCheckpoint.archive.root)) {
693
+ break;
694
+ }
695
+ tipAfterUnwind--;
696
+ }
697
+
698
+ const checkpointsToRemove = localPendingCheckpointNumber - tipAfterUnwind;
699
+ await this.updater.removeCheckpointsAfter(CheckpointNumber(tipAfterUnwind));
700
+
701
+ this.log.warn(
702
+ `Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` +
703
+ `due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` +
704
+ `Updated L2 latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
705
+ );
706
+ }
707
+ }
708
+
709
+ // Retrieve checkpoints in batches. Each batch is estimated to accommodate up to 'blockBatchSize' L1 blocks,
710
+ // computed using the L2 block time vs the L1 block time.
711
+ let searchStartBlock: bigint = blocksSynchedTo;
712
+ let searchEndBlock: bigint = blocksSynchedTo;
713
+ let lastRetrievedCheckpoint: PublishedCheckpoint | undefined;
714
+ let lastL1BlockWithCheckpoint: bigint | undefined = undefined;
715
+
716
+ do {
717
+ [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
718
+
719
+ this.log.trace(`Retrieving checkpoints from L1 block ${searchStartBlock} to ${searchEndBlock}`);
720
+
721
+ // TODO(md): Retrieve from blob client then from consensus client, then from peers
722
+ const retrievedCheckpoints = await execInSpan(this.tracer, 'Archiver.retrieveCheckpointsFromRollup', () =>
723
+ retrieveCheckpointsFromRollup(
724
+ this.rollup,
725
+ this.publicClient,
726
+ this.debugClient,
727
+ this.blobClient,
728
+ searchStartBlock, // TODO(palla/reorg): If the L2 reorg was due to an L1 reorg, we need to start search earlier
729
+ searchEndBlock,
730
+ this.instrumentation,
731
+ this.log,
732
+ !initialSyncComplete, // isHistoricalSync
733
+ ),
734
+ );
735
+
736
+ if (retrievedCheckpoints.length === 0) {
737
+ // We are not calling `setBlockSynchedL1BlockNumber` because it may cause sync issues if based off infura.
738
+ // See further details in earlier comments.
739
+ this.log.trace(`Retrieved no new checkpoints from L1 block ${searchStartBlock} to ${searchEndBlock}`);
740
+ continue;
741
+ }
742
+
743
+ this.log.debug(
744
+ `Retrieved ${retrievedCheckpoints.length} new checkpoints between L1 blocks ${searchStartBlock} and ${searchEndBlock}`,
745
+ {
746
+ lastProcessedCheckpoint: retrievedCheckpoints[retrievedCheckpoints.length - 1].l1,
747
+ searchStartBlock,
748
+ searchEndBlock,
749
+ },
750
+ );
751
+
752
+ const publishedCheckpoints = await Promise.all(retrievedCheckpoints.map(b => retrievedToPublishedCheckpoint(b)));
753
+ const validCheckpoints: PublishedCheckpoint[] = [];
754
+
755
+ for (const published of publishedCheckpoints) {
756
+ const validationResult = this.config.skipValidateCheckpointAttestations
757
+ ? { valid: true as const }
758
+ : await validateCheckpointAttestations(published, this.epochCache, this.l1Constants, this.log);
759
+
760
+ // Only update the validation result if it has changed, so we can keep track of the first invalid checkpoint
761
+ // in case there is a sequence of more than one invalid checkpoint, as we need to invalidate the first one.
762
+ // There is an exception though: if a checkpoint is invalidated and replaced with another invalid checkpoint,
763
+ // we need to update the validation result, since we need to be able to invalidate the new one.
764
+ // See test 'chain progresses if an invalid checkpoint is invalidated with an invalid one' for more info.
765
+ if (
766
+ rollupStatus.validationResult?.valid !== validationResult.valid ||
767
+ (!rollupStatus.validationResult.valid &&
768
+ !validationResult.valid &&
769
+ rollupStatus.validationResult.checkpoint.checkpointNumber === validationResult.checkpoint.checkpointNumber)
770
+ ) {
771
+ rollupStatus.validationResult = validationResult;
772
+ }
773
+
774
+ if (!validationResult.valid) {
775
+ this.log.warn(`Skipping checkpoint ${published.checkpoint.number} due to invalid attestations`, {
776
+ checkpointHash: published.checkpoint.hash(),
777
+ l1BlockNumber: published.l1.blockNumber,
778
+ ...pick(validationResult, 'reason'),
779
+ });
780
+
781
+ // Emit event for invalid checkpoint detection
782
+ this.events.emit(L2BlockSourceEvents.InvalidAttestationsCheckpointDetected, {
783
+ type: L2BlockSourceEvents.InvalidAttestationsCheckpointDetected,
784
+ validationResult,
785
+ });
786
+
787
+ // We keep consuming checkpoints if we find an invalid one, since we do not listen for CheckpointInvalidated events
788
+ // We just pretend the invalid ones are not there and keep consuming the next checkpoints
789
+ // Note that this breaks if the committee ever attests to a descendant of an invalid checkpoint
790
+ continue;
791
+ }
792
+
793
+ // Check the inHash of the checkpoint against the l1->l2 messages.
794
+ // The messages should've been synced up to the currentL1BlockNumber and must be available for the published
795
+ // checkpoints we just retrieved.
796
+ const l1ToL2Messages = await this.store.getL1ToL2Messages(published.checkpoint.number);
797
+ const computedInHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
798
+ const publishedInHash = published.checkpoint.header.inHash;
799
+ if (!computedInHash.equals(publishedInHash)) {
800
+ this.log.fatal(`Mismatch inHash for checkpoint ${published.checkpoint.number}`, {
801
+ checkpointHash: published.checkpoint.hash(),
802
+ l1BlockNumber: published.l1.blockNumber,
803
+ computedInHash,
804
+ publishedInHash,
805
+ });
806
+ // Throwing an error since this is most likely caused by a bug.
807
+ throw new Error(
808
+ `Mismatch inHash for checkpoint ${published.checkpoint.number}. Expected ${computedInHash} but got ${publishedInHash}`,
809
+ );
810
+ }
811
+
812
+ validCheckpoints.push(published);
813
+ this.log.debug(
814
+ `Ingesting new checkpoint ${published.checkpoint.number} with ${published.checkpoint.blocks.length} blocks`,
815
+ {
816
+ checkpointHash: published.checkpoint.hash(),
817
+ l1BlockNumber: published.l1.blockNumber,
818
+ ...published.checkpoint.header.toInspect(),
819
+ blocks: published.checkpoint.blocks.map(b => b.getStats()),
820
+ },
821
+ );
822
+ }
823
+
824
+ for (const published of validCheckpoints) {
825
+ this.instrumentation.processCheckpointL1Timing({
826
+ slotNumber: published.checkpoint.header.slotNumber,
827
+ l1Timestamp: published.l1.timestamp,
828
+ l1Constants: this.l1Constants,
829
+ });
830
+ }
831
+
832
+ try {
833
+ const updatedValidationResult =
834
+ rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
835
+ const [processDuration, result] = await elapsed(() =>
836
+ execInSpan(this.tracer, 'Archiver.addCheckpoints', () =>
837
+ this.updater.addCheckpoints(validCheckpoints, updatedValidationResult),
838
+ ),
839
+ );
840
+
841
+ if (validCheckpoints.length > 0) {
842
+ this.instrumentation.processNewCheckpointedBlocks(
843
+ processDuration / validCheckpoints.length,
844
+ validCheckpoints.flatMap(c => c.checkpoint.blocks),
845
+ );
846
+ }
847
+
848
+ // If blocks were pruned due to conflict with L1 checkpoints, emit event
849
+ if (result.prunedBlocks && result.prunedBlocks.length > 0) {
850
+ const prunedCheckpointNumber = result.prunedBlocks[0].checkpointNumber;
851
+ const prunedSlotNumber = result.prunedBlocks[0].header.globalVariables.slotNumber;
852
+
853
+ this.log.info(
854
+ `Pruned ${result.prunedBlocks.length} mismatching blocks for checkpoint ${prunedCheckpointNumber}`,
855
+ { prunedBlocks: result.prunedBlocks.map(b => b.toBlockInfo()), prunedSlotNumber, prunedCheckpointNumber },
856
+ );
857
+
858
+ // Emit event for listening services to react to the prune.
859
+ // Note: slotNumber comes from the first pruned block. If pruned blocks theoretically spanned multiple slots,
860
+ // only one slot number would be reported (though in practice all blocks in a checkpoint span a single slot).
861
+ this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
862
+ type: L2BlockSourceEvents.L2PruneUncheckpointed,
863
+ slotNumber: prunedSlotNumber,
864
+ blocks: result.prunedBlocks,
865
+ });
866
+ }
867
+ } catch (err) {
868
+ if (err instanceof InitialCheckpointNumberNotSequentialError) {
869
+ const { previousCheckpointNumber, newCheckpointNumber } = err;
870
+ const previousCheckpoint = previousCheckpointNumber
871
+ ? await this.store.getCheckpointData(CheckpointNumber(previousCheckpointNumber))
872
+ : undefined;
873
+ const updatedL1SyncPoint = previousCheckpoint?.l1.blockNumber ?? this.l1Constants.l1StartBlock;
874
+ await this.store.setCheckpointSynchedL1BlockNumber(updatedL1SyncPoint);
875
+ this.log.warn(
876
+ `Attempting to insert checkpoint ${newCheckpointNumber} with previous block ${previousCheckpointNumber}. Rolling back L1 sync point to ${updatedL1SyncPoint} to try and fetch the missing blocks.`,
877
+ {
878
+ previousCheckpointNumber,
879
+ newCheckpointNumber,
880
+ updatedL1SyncPoint,
881
+ },
882
+ );
883
+ }
884
+ throw err;
885
+ }
886
+
887
+ for (const checkpoint of validCheckpoints) {
888
+ this.log.info(`Downloaded checkpoint ${checkpoint.checkpoint.number}`, {
889
+ checkpointHash: checkpoint.checkpoint.hash(),
890
+ checkpointNumber: checkpoint.checkpoint.number,
891
+ blockCount: checkpoint.checkpoint.blocks.length,
892
+ txCount: checkpoint.checkpoint.blocks.reduce((acc, b) => acc + b.body.txEffects.length, 0),
893
+ header: checkpoint.checkpoint.header.toInspect(),
894
+ archiveRoot: checkpoint.checkpoint.archive.root.toString(),
895
+ archiveNextLeafIndex: checkpoint.checkpoint.archive.nextAvailableLeafIndex,
896
+ });
897
+ }
898
+ lastRetrievedCheckpoint = validCheckpoints.at(-1) ?? lastRetrievedCheckpoint;
899
+ lastL1BlockWithCheckpoint = retrievedCheckpoints.at(-1)?.l1.blockNumber ?? lastL1BlockWithCheckpoint;
900
+ } while (searchEndBlock < currentL1BlockNumber);
901
+
902
+ // Important that we update AFTER inserting the blocks.
903
+ await updateProvenCheckpoint();
904
+
905
+ return { ...rollupStatus, lastRetrievedCheckpoint, lastL1BlockWithCheckpoint };
906
+ }
907
+
908
+ private async checkForNewCheckpointsBeforeL1SyncPoint(
909
+ status: RollupStatus,
910
+ blocksSynchedTo: bigint,
911
+ currentL1BlockNumber: bigint,
912
+ ): Promise<void> {
913
+ const { lastRetrievedCheckpoint, pendingCheckpointNumber } = status;
914
+ // Compare the last checkpoint we have (either retrieved in this round or loaded from store) with what the
915
+ // rollup contract told us was the latest one (pinned at the currentL1BlockNumber).
916
+ const latestLocalCheckpointNumber =
917
+ lastRetrievedCheckpoint?.checkpoint.number ?? (await this.store.getSynchedCheckpointNumber());
918
+ if (latestLocalCheckpointNumber < pendingCheckpointNumber) {
919
+ // Here we have consumed all logs until the `currentL1Block` we pinned at the beginning of the archiver loop,
920
+ // but still haven't reached the pending checkpoint according to the call to the rollup contract.
921
+ // We suspect an L1 reorg that added checkpoints *behind* us. If that is the case, it must have happened between
922
+ // the last checkpoint we saw and the current one, so we reset the last synched L1 block number. In the edge case
923
+ // we don't have one, we go back 2 L1 epochs, which is the deepest possible reorg (assuming Casper is working).
924
+ let latestLocalCheckpointArchive: string | undefined = undefined;
925
+ let targetL1BlockNumber = maxBigint(currentL1BlockNumber - 64n, 0n);
926
+ if (lastRetrievedCheckpoint) {
927
+ latestLocalCheckpointArchive = lastRetrievedCheckpoint.checkpoint.archive.root.toString();
928
+ targetL1BlockNumber = lastRetrievedCheckpoint.l1.blockNumber;
929
+ } else if (latestLocalCheckpointNumber > 0) {
930
+ const checkpoint = await this.store.getRangeOfCheckpoints(latestLocalCheckpointNumber, 1).then(([c]) => c);
931
+ latestLocalCheckpointArchive = checkpoint.archive.root.toString();
932
+ targetL1BlockNumber = checkpoint.l1.blockNumber;
933
+ }
934
+ this.log.warn(
935
+ `Failed to reach checkpoint ${pendingCheckpointNumber} at ${currentL1BlockNumber} (latest is ${latestLocalCheckpointNumber}). ` +
936
+ `Rolling back last synched L1 block number to ${targetL1BlockNumber}.`,
937
+ {
938
+ latestLocalCheckpointNumber,
939
+ latestLocalCheckpointArchive,
940
+ blocksSynchedTo,
941
+ currentL1BlockNumber,
942
+ ...status,
943
+ },
944
+ );
945
+ await this.store.setCheckpointSynchedL1BlockNumber(targetL1BlockNumber);
946
+ } else {
947
+ this.log.trace(`No new checkpoints behind L1 sync point to retrieve.`, {
948
+ latestLocalCheckpointNumber,
949
+ pendingCheckpointNumber,
950
+ });
951
+ }
952
+ }
953
+
954
+ private async getCheckpointHeader(number: CheckpointNumber) {
955
+ const checkpoint = await this.store.getCheckpointData(number);
956
+ if (!checkpoint) {
957
+ return undefined;
958
+ }
959
+ return checkpoint.header;
960
+ }
961
+ }