@aztec/archiver 0.0.1-commit.e558bd1c → 0.0.1-commit.e5a3663dd

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/README.md +12 -6
  2. package/dest/archiver.d.ts +26 -15
  3. package/dest/archiver.d.ts.map +1 -1
  4. package/dest/archiver.js +161 -153
  5. package/dest/config.d.ts +5 -3
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +16 -4
  8. package/dest/errors.d.ts +61 -10
  9. package/dest/errors.d.ts.map +1 -1
  10. package/dest/errors.js +88 -14
  11. package/dest/factory.d.ts +6 -7
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +40 -32
  14. package/dest/index.d.ts +11 -3
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +10 -2
  17. package/dest/l1/bin/retrieve-calldata.js +32 -28
  18. package/dest/l1/calldata_retriever.d.ts +74 -50
  19. package/dest/l1/calldata_retriever.d.ts.map +1 -1
  20. package/dest/l1/calldata_retriever.js +197 -260
  21. package/dest/l1/data_retrieval.d.ts +26 -17
  22. package/dest/l1/data_retrieval.d.ts.map +1 -1
  23. package/dest/l1/data_retrieval.js +42 -47
  24. package/dest/l1/spire_proposer.d.ts +5 -5
  25. package/dest/l1/spire_proposer.d.ts.map +1 -1
  26. package/dest/l1/spire_proposer.js +9 -17
  27. package/dest/l1/validate_historical_logs.d.ts +23 -0
  28. package/dest/l1/validate_historical_logs.d.ts.map +1 -0
  29. package/dest/l1/validate_historical_logs.js +108 -0
  30. package/dest/modules/contract_data_source_adapter.d.ts +25 -0
  31. package/dest/modules/contract_data_source_adapter.d.ts.map +1 -0
  32. package/dest/modules/contract_data_source_adapter.js +42 -0
  33. package/dest/modules/data_source_base.d.ts +27 -14
  34. package/dest/modules/data_source_base.d.ts.map +1 -1
  35. package/dest/modules/data_source_base.js +98 -125
  36. package/dest/modules/data_store_updater.d.ts +37 -17
  37. package/dest/modules/data_store_updater.d.ts.map +1 -1
  38. package/dest/modules/data_store_updater.js +155 -112
  39. package/dest/modules/instrumentation.d.ts +21 -3
  40. package/dest/modules/instrumentation.d.ts.map +1 -1
  41. package/dest/modules/instrumentation.js +41 -8
  42. package/dest/modules/l1_synchronizer.d.ts +13 -11
  43. package/dest/modules/l1_synchronizer.d.ts.map +1 -1
  44. package/dest/modules/l1_synchronizer.js +355 -182
  45. package/dest/modules/validation.d.ts +4 -3
  46. package/dest/modules/validation.d.ts.map +1 -1
  47. package/dest/modules/validation.js +6 -6
  48. package/dest/store/block_store.d.ts +107 -31
  49. package/dest/store/block_store.d.ts.map +1 -1
  50. package/dest/store/block_store.js +477 -141
  51. package/dest/store/contract_class_store.d.ts +17 -4
  52. package/dest/store/contract_class_store.d.ts.map +1 -1
  53. package/dest/store/contract_class_store.js +24 -68
  54. package/dest/store/contract_instance_store.d.ts +28 -1
  55. package/dest/store/contract_instance_store.d.ts.map +1 -1
  56. package/dest/store/contract_instance_store.js +37 -2
  57. package/dest/store/data_stores.d.ts +68 -0
  58. package/dest/store/data_stores.d.ts.map +1 -0
  59. package/dest/store/data_stores.js +50 -0
  60. package/dest/store/function_names_cache.d.ts +17 -0
  61. package/dest/store/function_names_cache.d.ts.map +1 -0
  62. package/dest/store/function_names_cache.js +30 -0
  63. package/dest/store/l2_tips_cache.d.ts +20 -0
  64. package/dest/store/l2_tips_cache.d.ts.map +1 -0
  65. package/dest/store/l2_tips_cache.js +109 -0
  66. package/dest/store/log_store.d.ts +6 -3
  67. package/dest/store/log_store.d.ts.map +1 -1
  68. package/dest/store/log_store.js +95 -20
  69. package/dest/store/message_store.d.ts +11 -1
  70. package/dest/store/message_store.d.ts.map +1 -1
  71. package/dest/store/message_store.js +51 -9
  72. package/dest/test/fake_l1_state.d.ts +25 -1
  73. package/dest/test/fake_l1_state.d.ts.map +1 -1
  74. package/dest/test/fake_l1_state.js +166 -32
  75. package/dest/test/mock_archiver.d.ts +1 -1
  76. package/dest/test/mock_archiver.d.ts.map +1 -1
  77. package/dest/test/mock_archiver.js +3 -2
  78. package/dest/test/mock_l1_to_l2_message_source.d.ts +1 -1
  79. package/dest/test/mock_l1_to_l2_message_source.d.ts.map +1 -1
  80. package/dest/test/mock_l1_to_l2_message_source.js +2 -1
  81. package/dest/test/mock_l2_block_source.d.ts +35 -5
  82. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  83. package/dest/test/mock_l2_block_source.js +182 -89
  84. package/dest/test/mock_structs.d.ts +4 -1
  85. package/dest/test/mock_structs.d.ts.map +1 -1
  86. package/dest/test/mock_structs.js +13 -1
  87. package/dest/test/noop_l1_archiver.d.ts +7 -4
  88. package/dest/test/noop_l1_archiver.d.ts.map +1 -1
  89. package/dest/test/noop_l1_archiver.js +14 -8
  90. package/package.json +13 -13
  91. package/src/archiver.ts +199 -174
  92. package/src/config.ts +23 -2
  93. package/src/errors.ts +133 -22
  94. package/src/factory.ts +53 -30
  95. package/src/index.ts +18 -2
  96. package/src/l1/README.md +25 -68
  97. package/src/l1/bin/retrieve-calldata.ts +40 -27
  98. package/src/l1/calldata_retriever.ts +261 -379
  99. package/src/l1/data_retrieval.ts +58 -69
  100. package/src/l1/spire_proposer.ts +7 -15
  101. package/src/l1/validate_historical_logs.ts +140 -0
  102. package/src/modules/contract_data_source_adapter.ts +59 -0
  103. package/src/modules/data_source_base.ts +142 -144
  104. package/src/modules/data_store_updater.ts +187 -141
  105. package/src/modules/instrumentation.ts +56 -9
  106. package/src/modules/l1_synchronizer.ts +463 -218
  107. package/src/modules/validation.ts +10 -9
  108. package/src/store/block_store.ts +587 -177
  109. package/src/store/contract_class_store.ts +31 -103
  110. package/src/store/contract_instance_store.ts +51 -5
  111. package/src/store/data_stores.ts +108 -0
  112. package/src/store/function_names_cache.ts +37 -0
  113. package/src/store/l2_tips_cache.ts +134 -0
  114. package/src/store/log_store.ts +128 -32
  115. package/src/store/message_store.ts +60 -10
  116. package/src/structs/inbox_message.ts +1 -1
  117. package/src/test/fake_l1_state.ts +213 -42
  118. package/src/test/mock_archiver.ts +3 -2
  119. package/src/test/mock_l1_to_l2_message_source.ts +1 -0
  120. package/src/test/mock_l2_block_source.ts +230 -82
  121. package/src/test/mock_structs.ts +20 -6
  122. package/src/test/noop_l1_archiver.ts +16 -8
  123. package/dest/store/kv_archiver_store.d.ts +0 -340
  124. package/dest/store/kv_archiver_store.d.ts.map +0 -1
  125. package/dest/store/kv_archiver_store.js +0 -446
  126. package/src/store/kv_archiver_store.ts +0 -639
@@ -1,33 +1,40 @@
1
1
  import type { BlobClientInterface } from '@aztec/blob-client/client';
2
2
  import { EpochCache } from '@aztec/epoch-cache';
3
- import { InboxContract, RollupContract } from '@aztec/ethereum/contracts';
4
- import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
3
+ import { InboxContract, type InboxContractState, RollupContract } from '@aztec/ethereum/contracts';
5
4
  import type { L1BlockId } from '@aztec/ethereum/l1-types';
5
+ import { getFinalizedL1Block } from '@aztec/ethereum/queries';
6
6
  import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
7
+ import { asyncPool } from '@aztec/foundation/async-pool';
7
8
  import { maxBigint } from '@aztec/foundation/bigint';
8
9
  import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
9
- import { Buffer32 } from '@aztec/foundation/buffer';
10
- import { pick } from '@aztec/foundation/collection';
10
+ import { Buffer16, Buffer32 } from '@aztec/foundation/buffer';
11
+ import { compactArray, partition, pick } from '@aztec/foundation/collection';
11
12
  import { Fr } from '@aztec/foundation/curves/bn254';
12
13
  import { EthAddress } from '@aztec/foundation/eth-address';
13
14
  import { type Logger, createLogger } from '@aztec/foundation/log';
15
+ import { retryTimes } from '@aztec/foundation/retry';
14
16
  import { count } from '@aztec/foundation/string';
15
17
  import { DateProvider, Timer, elapsed } from '@aztec/foundation/timer';
16
- import { isDefined } from '@aztec/foundation/types';
18
+ import { isDefined, isErrorClass } from '@aztec/foundation/types';
17
19
  import { type ArchiverEmitter, L2BlockSourceEvents, type ValidateCheckpointResult } from '@aztec/stdlib/block';
18
- import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
20
+ import { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
19
21
  import { type L1RollupConstants, getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers';
20
22
  import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
23
+ import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
21
24
  import { type Traceable, type Tracer, execInSpan, trackSpan } from '@aztec/telemetry-client';
22
25
 
23
26
  import { InitialCheckpointNumberNotSequentialError } from '../errors.js';
24
27
  import {
25
- retrieveCheckpointsFromRollup,
28
+ type RetrievedCheckpointFromCalldata,
29
+ getCheckpointBlobDataFromBlobs,
30
+ retrieveCheckpointCalldataFromRollup,
26
31
  retrieveL1ToL2Message,
27
32
  retrieveL1ToL2Messages,
28
33
  retrievedToPublishedCheckpoint,
29
34
  } from '../l1/data_retrieval.js';
30
- import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
35
+ import { type ArchiverDataStores, getArchiverSynchPoint } from '../store/data_stores.js';
36
+ import type { L2TipsCache } from '../store/l2_tips_cache.js';
37
+ import { MessageStoreError } from '../store/message_store.js';
31
38
  import type { InboxMessage } from '../structs/inbox_message.js';
32
39
  import { ArchiverDataStoreUpdater } from './data_store_updater.js';
33
40
  import type { ArchiverInstrumentation } from './instrumentation.js';
@@ -60,26 +67,29 @@ export class ArchiverL1Synchronizer implements Traceable {
60
67
  private readonly debugClient: ViemPublicDebugClient,
61
68
  private readonly rollup: RollupContract,
62
69
  private readonly inbox: InboxContract,
63
- private readonly l1Addresses: Pick<
64
- L1ContractAddresses,
65
- 'registryAddress' | 'governanceProposerAddress' | 'slashFactoryAddress'
66
- > & { slashingProposerAddress: EthAddress },
67
- private readonly store: KVArchiverDataStore,
70
+ private readonly stores: ArchiverDataStores,
68
71
  private config: {
69
72
  batchSize: number;
70
73
  skipValidateCheckpointAttestations?: boolean;
74
+ skipPromoteProposedCheckpointDuringL1Sync?: boolean;
71
75
  maxAllowedEthClientDriftSeconds: number;
72
76
  },
73
77
  private readonly blobClient: BlobClientInterface,
74
78
  private readonly epochCache: EpochCache,
75
79
  private readonly dateProvider: DateProvider,
76
80
  private readonly instrumentation: ArchiverInstrumentation,
77
- private readonly l1Constants: L1RollupConstants & { l1StartBlockHash: Buffer32; genesisArchiveRoot: Fr },
81
+ private readonly l1Constants: L1RollupConstants & {
82
+ l1StartBlockHash: Buffer32;
83
+ genesisArchiveRoot: Fr;
84
+ },
78
85
  private readonly events: ArchiverEmitter,
79
86
  tracer: Tracer,
87
+ l2TipsCache?: L2TipsCache,
80
88
  private readonly log: Logger = createLogger('archiver:l1-sync'),
81
89
  ) {
82
- this.updater = new ArchiverDataStoreUpdater(this.store);
90
+ this.updater = new ArchiverDataStoreUpdater(this.stores, l2TipsCache, {
91
+ rollupManaLimit: l1Constants.rollupManaLimit,
92
+ });
83
93
  this.tracer = tracer;
84
94
  }
85
95
 
@@ -87,6 +97,7 @@ export class ArchiverL1Synchronizer implements Traceable {
87
97
  public setConfig(newConfig: {
88
98
  batchSize: number;
89
99
  skipValidateCheckpointAttestations?: boolean;
100
+ skipPromoteProposedCheckpointDuringL1Sync?: boolean;
90
101
  maxAllowedEthClientDriftSeconds: number;
91
102
  }) {
92
103
  this.config = newConfig;
@@ -102,6 +113,13 @@ export class ArchiverL1Synchronizer implements Traceable {
102
113
  return this.l1Timestamp;
103
114
  }
104
115
 
116
+ private getSignatureContext(): CoordinationSignatureContext {
117
+ return {
118
+ chainId: this.publicClient.chain.id,
119
+ rollupAddress: EthAddress.fromString(this.rollup.address),
120
+ };
121
+ }
122
+
105
123
  /** Checks that the ethereum node we are connected to has a latest timestamp no more than the allowed drift. Throw if not. */
106
124
  public async testEthereumNodeSynced(): Promise<void> {
107
125
  const maxAllowedDelay = this.config.maxAllowedEthClientDriftSeconds;
@@ -119,10 +137,15 @@ export class ArchiverL1Synchronizer implements Traceable {
119
137
 
120
138
  @trackSpan('Archiver.syncFromL1')
121
139
  public async syncFromL1(initialSyncComplete: boolean): Promise<void> {
140
+ // In between the various calls to L1, the block number can move meaning some of the following
141
+ // calls will return data for blocks that were not present during earlier calls. To combat this
142
+ // we ensure that all data retrieval methods only retrieve data up to the currentBlockNumber
143
+ // captured at the top of this function.
122
144
  const currentL1Block = await this.publicClient.getBlock({ includeTransactions: false });
123
145
  const currentL1BlockNumber = currentL1Block.number;
124
146
  const currentL1BlockHash = Buffer32.fromString(currentL1Block.hash);
125
147
  const currentL1Timestamp = currentL1Block.timestamp;
148
+ const currentL1BlockData = { l1BlockNumber: currentL1BlockNumber, l1BlockHash: currentL1BlockHash };
126
149
 
127
150
  if (this.l1BlockHash && currentL1BlockHash.equals(this.l1BlockHash)) {
128
151
  this.log.trace(`No new L1 blocks since last sync at L1 block ${this.l1BlockNumber}`);
@@ -139,45 +162,27 @@ export class ArchiverL1Synchronizer implements Traceable {
139
162
  );
140
163
  }
141
164
 
142
- // Load sync point for blocks and messages defaulting to start block
143
- const {
144
- blocksSynchedTo = this.l1Constants.l1StartBlock,
145
- messagesSynchedTo = {
146
- l1BlockNumber: this.l1Constants.l1StartBlock,
147
- l1BlockHash: this.l1Constants.l1StartBlockHash,
148
- },
149
- } = await this.store.getSynchPoint();
165
+ // Query finalized block on L1
166
+ const rawFinalizedL1Block = await getFinalizedL1Block(this.publicClient);
167
+ const finalizedL1Block: L1BlockId | undefined = rawFinalizedL1Block && {
168
+ l1BlockNumber: rawFinalizedL1Block.number,
169
+ l1BlockHash: Buffer32.fromString(rawFinalizedL1Block.hash),
170
+ };
150
171
 
151
- this.log.debug(`Starting new archiver sync iteration`, {
152
- blocksSynchedTo,
153
- messagesSynchedTo,
154
- currentL1BlockNumber,
155
- currentL1BlockHash,
156
- });
172
+ // Load sync point for blocks defaulting to start block
173
+ const { blocksSynchedTo = this.l1Constants.l1StartBlock } = await getArchiverSynchPoint(this.stores);
174
+ this.log.debug(`Starting new archiver sync iteration`, { blocksSynchedTo, currentL1BlockData, finalizedL1Block });
175
+
176
+ // Sync L1 to L2 messages. We retry this a few times since there are error conditions that reset the sync point, requiring a new iteration.
177
+ // Note that we cannot just wait for the l1 synchronizer to loop again, since the synchronizer would report as synced up to the current L1
178
+ // block, when that wouldn't be the case, since L1 to L2 messages would need another iteration.
179
+ await retryTimes(
180
+ () => this.handleL1ToL2Messages(currentL1BlockData, finalizedL1Block),
181
+ 'Handling L1 to L2 messages',
182
+ 3,
183
+ 0.1,
184
+ );
157
185
 
158
- // ********** Ensuring Consistency of data pulled from L1 **********
159
-
160
- /**
161
- * There are a number of calls in this sync operation to L1 for retrieving
162
- * events and transaction data. There are a couple of things we need to bear in mind
163
- * to ensure that data is read exactly once.
164
- *
165
- * The first is the problem of eventually consistent ETH service providers like Infura.
166
- * Each L1 read operation will query data from the last L1 block that it saw emit its kind of data.
167
- * (so pending L1 to L2 messages will read from the last L1 block that emitted a message and so on)
168
- * This will mean the archiver will lag behind L1 and will only advance when there's L2-relevant activity on the chain.
169
- *
170
- * The second is that in between the various calls to L1, the block number can move meaning some
171
- * of the following calls will return data for blocks that were not present during earlier calls.
172
- * To combat this for the time being we simply ensure that all data retrieval methods only retrieve
173
- * data up to the currentBlockNumber captured at the top of this function. We might want to improve on this
174
- * in future but for the time being it should give us the guarantees that we need
175
- */
176
-
177
- // ********** Events that are processed per L1 block **********
178
- await this.handleL1ToL2Messages(messagesSynchedTo, currentL1BlockNumber);
179
-
180
- // ********** Events that are processed per checkpoint **********
181
186
  if (currentL1BlockNumber > blocksSynchedTo) {
182
187
  // First we retrieve new checkpoints and L2 blocks and store them in the DB. This will also update the
183
188
  // pending chain validation status, proven checkpoint number, and synched L1 block number.
@@ -201,7 +206,7 @@ export class ArchiverL1Synchronizer implements Traceable {
201
206
  // past it, since otherwise we'll keep downloading it and reprocessing it on every iteration until
202
207
  // we get a valid checkpoint to advance the syncpoint.
203
208
  if (!rollupStatus.validationResult?.valid && rollupStatus.lastL1BlockWithCheckpoint !== undefined) {
204
- await this.store.setCheckpointSynchedL1BlockNumber(rollupStatus.lastL1BlockWithCheckpoint);
209
+ await this.stores.blocks.setSynchedL1BlockNumber(rollupStatus.lastL1BlockWithCheckpoint);
205
210
  }
206
211
 
207
212
  // And lastly we check if we are missing any checkpoints behind us due to a possible L1 reorg.
@@ -215,6 +220,9 @@ export class ArchiverL1Synchronizer implements Traceable {
215
220
  this.instrumentation.updateL1BlockHeight(currentL1BlockNumber);
216
221
  }
217
222
 
223
+ // Update the finalized L2 checkpoint based on L1 finality.
224
+ await this.updateFinalizedCheckpoint(finalizedL1Block);
225
+
218
226
  // After syncing has completed, update the current l1 block number and timestamp,
219
227
  // otherwise we risk announcing to the world that we've synced to a given point,
220
228
  // but the corresponding blocks have not been processed (see #12631).
@@ -230,11 +238,38 @@ export class ArchiverL1Synchronizer implements Traceable {
230
238
  });
231
239
  }
232
240
 
241
+ /** Updates the finalized checkpoint using the pre-fetched finalized L1 block from the current sync iteration. */
242
+ private async updateFinalizedCheckpoint(finalizedL1Block: L1BlockId | undefined): Promise<void> {
243
+ try {
244
+ if (!finalizedL1Block) {
245
+ this.log.trace(`Skipping finalized checkpoint update: L1 has no finalized block yet.`);
246
+ return;
247
+ }
248
+ const finalizedL1BlockNumber = finalizedL1Block.l1BlockNumber;
249
+ const finalizedCheckpointNumber = await this.rollup.getProvenCheckpointNumber({
250
+ blockNumber: finalizedL1BlockNumber,
251
+ });
252
+ const localFinalizedCheckpointNumber = await this.stores.blocks.getFinalizedCheckpointNumber();
253
+ if (localFinalizedCheckpointNumber !== finalizedCheckpointNumber) {
254
+ await this.updater.setFinalizedCheckpointNumber(finalizedCheckpointNumber);
255
+ this.log.info(`Updated finalized chain to checkpoint ${finalizedCheckpointNumber}`, {
256
+ finalizedCheckpointNumber,
257
+ finalizedL1BlockNumber,
258
+ });
259
+ }
260
+ } catch (err: any) {
261
+ // The rollup contract may not exist at the finalized L1 block right after deployment.
262
+ if (!err?.message?.includes('returned no data')) {
263
+ this.log.warn(`Failed to update finalized checkpoint: ${err}`);
264
+ }
265
+ }
266
+ }
267
+
233
268
  /** Prune all proposed local blocks that should have been checkpointed by now. */
234
269
  private async pruneUncheckpointedBlocks(currentL1Timestamp: bigint) {
235
270
  const [lastCheckpointedBlockNumber, lastProposedBlockNumber] = await Promise.all([
236
- this.store.getCheckpointedL2BlockNumber(),
237
- this.store.getLatestBlockNumber(),
271
+ this.stores.blocks.getCheckpointedL2BlockNumber(),
272
+ this.stores.blocks.getLatestL2BlockNumber(),
238
273
  ]);
239
274
 
240
275
  // If there are no uncheckpointed blocks, we got nothing to do
@@ -243,29 +278,36 @@ export class ArchiverL1Synchronizer implements Traceable {
243
278
  return;
244
279
  }
245
280
 
246
- // What's the slot of the first uncheckpointed block?
281
+ // What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
282
+ const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
247
283
  const firstUncheckpointedBlockNumber = BlockNumber(lastCheckpointedBlockNumber + 1);
248
- const [firstUncheckpointedBlockHeader] = await this.store.getBlockHeaders(firstUncheckpointedBlockNumber, 1);
284
+
285
+ // What's the slot of the first uncheckpointed block?
286
+ const [firstUncheckpointedBlockHeader] = await this.stores.blocks.getBlockHeaders(
287
+ firstUncheckpointedBlockNumber,
288
+ 1,
289
+ );
249
290
  const firstUncheckpointedBlockSlot = firstUncheckpointedBlockHeader?.getSlot();
250
291
 
251
- // What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
252
- const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
292
+ if (firstUncheckpointedBlockSlot === undefined || firstUncheckpointedBlockSlot >= slotAtNextL1Block) {
293
+ return;
294
+ }
253
295
 
254
- // Prune provisional blocks from slots that have ended without being checkpointed
255
- if (firstUncheckpointedBlockSlot !== undefined && firstUncheckpointedBlockSlot < slotAtNextL1Block) {
256
- this.log.warn(
257
- `Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`,
258
- { firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(), slotAtNextL1Block },
259
- );
260
- const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
296
+ // Prune provisional blocks from slots that have ended without being checkpointed.
297
+ // This also clears any proposed checkpoint whose blocks are being pruned.
298
+ this.log.warn(
299
+ `Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`,
300
+ { firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(), slotAtNextL1Block },
301
+ );
261
302
 
262
- if (prunedBlocks.length > 0) {
263
- this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
264
- type: L2BlockSourceEvents.L2PruneUncheckpointed,
265
- slotNumber: firstUncheckpointedBlockSlot,
266
- blocks: prunedBlocks,
267
- });
268
- }
303
+ const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
304
+
305
+ if (prunedBlocks.length > 0) {
306
+ this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
307
+ type: L2BlockSourceEvents.L2PruneUncheckpointed,
308
+ slotNumber: firstUncheckpointedBlockSlot,
309
+ blocks: prunedBlocks,
310
+ });
269
311
  }
270
312
  }
271
313
 
@@ -291,7 +333,7 @@ export class ArchiverL1Synchronizer implements Traceable {
291
333
  currentL1Timestamp: bigint,
292
334
  ): Promise<{ rollupCanPrune: boolean }> {
293
335
  const rollupCanPrune = await this.canPrune(currentL1BlockNumber, currentL1Timestamp);
294
- const localPendingCheckpointNumber = await this.store.getSynchedCheckpointNumber();
336
+ const localPendingCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
295
337
  const canPrune = localPendingCheckpointNumber > provenCheckpointNumber && rollupCanPrune;
296
338
 
297
339
  if (canPrune) {
@@ -308,17 +350,20 @@ export class ArchiverL1Synchronizer implements Traceable {
308
350
 
309
351
  const checkpointsToUnwind = localPendingCheckpointNumber - provenCheckpointNumber;
310
352
 
311
- const checkpointPromises = Array.from({ length: checkpointsToUnwind })
312
- .fill(0)
313
- .map((_, i) => this.store.getCheckpointData(CheckpointNumber(i + pruneFrom)));
314
- const checkpoints = await Promise.all(checkpointPromises);
315
-
316
- const blockPromises = await Promise.all(
317
- checkpoints
318
- .filter(isDefined)
319
- .map(cp => this.store.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber))),
320
- );
321
- const newBlocks = blockPromises.filter(isDefined).flat();
353
+ // Fetch checkpoints and blocks in bounded batches to avoid unbounded concurrent
354
+ // promises when the gap between local pending and proven checkpoint numbers is large.
355
+ const BATCH_SIZE = 10;
356
+ const indices = Array.from({ length: checkpointsToUnwind }, (_, i) => CheckpointNumber(i + pruneFrom));
357
+ const checkpoints = (
358
+ await asyncPool(BATCH_SIZE, indices, idx => this.stores.blocks.getCheckpointData(idx))
359
+ ).filter(isDefined);
360
+ const newBlocks = (
361
+ await asyncPool(BATCH_SIZE, checkpoints, cp =>
362
+ this.stores.blocks.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber)),
363
+ )
364
+ )
365
+ .filter(isDefined)
366
+ .flat();
322
367
 
323
368
  // Emit an event for listening services to react to the chain prune
324
369
  this.events.emit(L2BlockSourceEvents.L2PruneUnproven, {
@@ -334,12 +379,12 @@ export class ArchiverL1Synchronizer implements Traceable {
334
379
  this.log.warn(
335
380
  `Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` +
336
381
  `due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
337
- `Updated latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
382
+ `Updated latest checkpoint is ${await this.stores.blocks.getLatestCheckpointNumber()}.`,
338
383
  );
339
384
  this.instrumentation.processPrune(timer.ms());
340
385
  // TODO(palla/reorg): Do we need to set the block synched L1 block number here?
341
386
  // Seems like the next iteration should handle this.
342
- // await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
387
+ // await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
343
388
  }
344
389
 
345
390
  return { rollupCanPrune };
@@ -356,67 +401,102 @@ export class ArchiverL1Synchronizer implements Traceable {
356
401
  }
357
402
 
358
403
  @trackSpan('Archiver.handleL1ToL2Messages')
359
- private async handleL1ToL2Messages(messagesSyncPoint: L1BlockId, currentL1BlockNumber: bigint): Promise<void> {
360
- this.log.trace(`Handling L1 to L2 messages from ${messagesSyncPoint.l1BlockNumber} to ${currentL1BlockNumber}.`);
361
- if (currentL1BlockNumber <= messagesSyncPoint.l1BlockNumber) {
362
- return;
404
+ private async handleL1ToL2Messages(
405
+ currentL1Block: L1BlockId,
406
+ finalizedL1Block: L1BlockId | undefined,
407
+ ): Promise<boolean> {
408
+ // Load the syncpoint, which may have been updated in a previous iteration
409
+ const {
410
+ messagesSynchedTo = {
411
+ l1BlockNumber: this.l1Constants.l1StartBlock,
412
+ l1BlockHash: this.l1Constants.l1StartBlockHash,
413
+ },
414
+ } = await getArchiverSynchPoint(this.stores);
415
+
416
+ // Nothing to do if L1 block number has not moved forward
417
+ const currentL1BlockNumber = currentL1Block.l1BlockNumber;
418
+ if (currentL1BlockNumber <= messagesSynchedTo.l1BlockNumber) {
419
+ return true;
363
420
  }
364
421
 
365
- // Load remote and local inbox states.
366
- const localMessagesInserted = await this.store.getTotalL1ToL2MessageCount();
367
- const localLastMessage = await this.store.getLastL1ToL2Message();
422
+ // Compare local message store state with the remote. If they match, we just advance the match pointer.
368
423
  const remoteMessagesState = await this.inbox.getState({ blockNumber: currentL1BlockNumber });
424
+ const localLastMessage = await this.stores.messages.getLastMessage();
425
+ if (await this.localStateMatches(localLastMessage, remoteMessagesState)) {
426
+ this.log.trace(`Local L1 to L2 messages are already in sync with remote at L1 block ${currentL1BlockNumber}`);
427
+ await this.stores.messages.setMessageSyncState(
428
+ currentL1Block,
429
+ remoteMessagesState.treeInProgress,
430
+ finalizedL1Block,
431
+ );
432
+ return true;
433
+ }
369
434
 
370
- this.log.trace(`Retrieved remote inbox state at L1 block ${currentL1BlockNumber}.`, {
371
- localMessagesInserted,
372
- localLastMessage,
373
- remoteMessagesState,
374
- });
435
+ // If not, then we are out of sync. Most likely there are new messages on the inbox, so we try retrieving them.
436
+ // However, it could also be the case that there was an L1 reorg and our syncpoint is no longer valid.
437
+ // If that's the case, we'd get an exception out of the message store since the rolling hash of the first message
438
+ // we try to insert would not match the one in the db, in which case we rollback to the last common message with L1.
439
+ try {
440
+ await this.retrieveAndStoreMessages(messagesSynchedTo.l1BlockNumber, currentL1BlockNumber);
441
+ } catch (error) {
442
+ if (isErrorClass(error, MessageStoreError)) {
443
+ this.log.warn(
444
+ `Failed to store L1 to L2 messages retrieved from L1: ${error.message}. Rolling back syncpoint to retry.`,
445
+ { inboxMessage: error.inboxMessage },
446
+ );
447
+ await this.rollbackL1ToL2Messages(remoteMessagesState);
448
+ return false;
449
+ }
450
+ throw error;
451
+ }
375
452
 
376
- // Compare message count and rolling hash. If they match, no need to retrieve anything.
377
- if (
378
- remoteMessagesState.totalMessagesInserted === localMessagesInserted &&
379
- remoteMessagesState.messagesRollingHash.equals(localLastMessage?.rollingHash ?? Buffer32.ZERO)
380
- ) {
381
- this.log.trace(
382
- `No L1 to L2 messages to query between L1 blocks ${messagesSyncPoint.l1BlockNumber} and ${currentL1BlockNumber}.`,
453
+ // Note that, if there are no new messages to insert, but there was an L1 reorg that pruned out last messages,
454
+ // we'd notice by comparing our local state with the remote one again, and seeing they don't match even after
455
+ // our sync attempt. In this case, we also rollback our syncpoint, and trigger a retry.
456
+ const localLastMessageAfterSync = await this.stores.messages.getLastMessage();
457
+ if (!(await this.localStateMatches(localLastMessageAfterSync, remoteMessagesState))) {
458
+ this.log.warn(
459
+ `Local L1 to L2 messages state does not match remote after sync attempt. Rolling back syncpoint to retry.`,
460
+ { localLastMessageAfterSync, remoteMessagesState },
383
461
  );
384
- return;
462
+ await this.rollbackL1ToL2Messages(remoteMessagesState);
463
+ return false;
385
464
  }
386
465
 
387
- // Check if our syncpoint is still valid. If not, there was an L1 reorg and we need to re-retrieve messages.
388
- // Note that we need to fetch it from logs and not from inbox state at the syncpoint l1 block number, since it
389
- // could be older than 128 blocks and non-archive nodes cannot resolve it.
390
- if (localLastMessage) {
391
- const remoteLastMessage = await this.retrieveL1ToL2Message(localLastMessage.leaf);
392
- this.log.trace(`Retrieved remote message for local last`, { remoteLastMessage, localLastMessage });
393
- if (!remoteLastMessage || !remoteLastMessage.rollingHash.equals(localLastMessage.rollingHash)) {
394
- this.log.warn(`Rolling back L1 to L2 messages due to hash mismatch or msg not found.`, {
395
- remoteLastMessage,
396
- messagesSyncPoint,
397
- localLastMessage,
398
- });
466
+ // Advance the syncpoint after a successful sync
467
+ await this.stores.messages.setMessageSyncState(
468
+ currentL1Block,
469
+ remoteMessagesState.treeInProgress,
470
+ finalizedL1Block,
471
+ );
472
+ return true;
473
+ }
399
474
 
400
- messagesSyncPoint = await this.rollbackL1ToL2Messages(localLastMessage, messagesSyncPoint);
401
- this.log.debug(`Rolled back L1 to L2 messages to L1 block ${messagesSyncPoint.l1BlockNumber}.`, {
402
- messagesSyncPoint,
403
- });
404
- }
405
- }
475
+ /** Checks if the local rolling hash and message count matches the remote state */
476
+ private async localStateMatches(localLastMessage: InboxMessage | undefined, remoteState: InboxContractState) {
477
+ const localMessageCount = await this.stores.messages.getTotalL1ToL2MessageCount();
478
+ this.log.trace(`Comparing local and remote inbox state`, { localMessageCount, localLastMessage, remoteState });
406
479
 
407
- // Retrieve and save messages in batches. Each batch is estimated to acommodate up to L2 'blockBatchSize' blocks,
480
+ return (
481
+ remoteState.totalMessagesInserted === localMessageCount &&
482
+ remoteState.messagesRollingHash.equals(localLastMessage?.rollingHash ?? Buffer16.ZERO)
483
+ );
484
+ }
485
+
486
+ /** Retrieves L1 to L2 messages from L1 in batches and stores them. */
487
+ private async retrieveAndStoreMessages(fromL1Block: bigint, toL1Block: bigint): Promise<void> {
408
488
  let searchStartBlock: bigint = 0n;
409
- let searchEndBlock: bigint = messagesSyncPoint.l1BlockNumber;
489
+ let searchEndBlock: bigint = fromL1Block;
410
490
 
411
491
  let lastMessage: InboxMessage | undefined;
412
492
  let messageCount = 0;
413
493
 
414
494
  do {
415
- [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
495
+ [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, toL1Block);
416
496
  this.log.trace(`Retrieving L1 to L2 messages in L1 blocks ${searchStartBlock}-${searchEndBlock}`);
417
497
  const messages = await retrieveL1ToL2Messages(this.inbox, searchStartBlock, searchEndBlock);
418
498
  const timer = new Timer();
419
- await this.store.addL1ToL2Messages(messages);
499
+ await this.stores.messages.addL1ToL2Messages(messages);
420
500
  const perMsg = timer.ms() / messages.length;
421
501
  this.instrumentation.processNewMessages(messages.length, perMsg);
422
502
  for (const msg of messages) {
@@ -424,81 +504,108 @@ export class ArchiverL1Synchronizer implements Traceable {
424
504
  lastMessage = msg;
425
505
  messageCount++;
426
506
  }
427
- } while (searchEndBlock < currentL1BlockNumber);
507
+ } while (searchEndBlock < toL1Block);
428
508
 
429
- // Log stats for messages retrieved (if any).
430
509
  if (messageCount > 0) {
431
510
  this.log.info(
432
511
  `Retrieved ${messageCount} new L1 to L2 messages up to message with index ${lastMessage?.index} for checkpoint ${lastMessage?.checkpointNumber}`,
433
512
  { lastMessage, messageCount },
434
513
  );
435
514
  }
436
-
437
- // Warn if the resulting rolling hash does not match the remote state we had retrieved.
438
- if (lastMessage && !lastMessage.rollingHash.equals(remoteMessagesState.messagesRollingHash)) {
439
- this.log.warn(`Last message retrieved rolling hash does not match remote state.`, {
440
- lastMessage,
441
- remoteMessagesState,
442
- });
443
- }
444
515
  }
445
516
 
446
- private async retrieveL1ToL2Message(leaf: Fr): Promise<InboxMessage | undefined> {
447
- const currentL1BlockNumber = await this.publicClient.getBlockNumber();
448
- let searchStartBlock: bigint = 0n;
449
- let searchEndBlock: bigint = this.l1Constants.l1StartBlock - 1n;
517
+ /**
518
+ * Rolls back local L1 to L2 messages to the last common message with L1, and updates the syncpoint to the L1 block of that message.
519
+ * If no common message is found, rolls back all messages and sets the syncpoint to the start block.
520
+ */
521
+ private async rollbackL1ToL2Messages(remoteMessagesState: InboxContractState): Promise<L1BlockId> {
522
+ const { treeInProgress: remoteTreeInProgress, messagesRollingHash: remoteRollingHash } = remoteMessagesState;
450
523
 
451
- do {
452
- [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
524
+ const messagesFinalizedL1Block = await this.stores.messages.getMessagesFinalizedL1Block();
525
+ const finalizedL1BlockNumber = messagesFinalizedL1Block?.l1BlockNumber;
453
526
 
454
- const message = await retrieveL1ToL2Message(this.inbox, leaf, searchStartBlock, searchEndBlock);
455
-
456
- if (message) {
457
- return message;
527
+ // Slowly go back through our messages until we find the last common message. We could query the logs in
528
+ // batch as an optimization, but the depth of the reorg should not be deep, and this is a very rare case,
529
+ // so it's fine to query one log at a time.
530
+ let commonMsg: undefined | InboxMessage;
531
+ let messagesToDelete = 0;
532
+ this.log.verbose(`Searching most recent common L1 to L2 message`);
533
+ for await (const localMsg of this.stores.messages.iterateL1ToL2Messages({ reverse: true })) {
534
+ const logCtx = { remoteMsg: undefined as InboxMessage | undefined, localMsg, remoteMessagesState };
535
+
536
+ // First check if the local message rolling hash matches the current rolling hash of the inbox contract,
537
+ // which means we just need to rollback some local messages and we should be back in sync. This means there
538
+ // was an L1 reorg that removed some of the messages we had, but no new messages were added compared.
539
+ if (localMsg.rollingHash.equals(remoteRollingHash)) {
540
+ this.log.info(
541
+ `Found common L1 to L2 message at index ${localMsg.index} on L1 block ${localMsg.l1BlockNumber} matching current remote state`,
542
+ logCtx,
543
+ );
544
+ commonMsg = localMsg;
545
+ break;
458
546
  }
459
- } while (searchEndBlock < currentL1BlockNumber);
460
547
 
461
- return undefined;
462
- }
548
+ // Messages at or below the finalized L1 block cannot have been reorged — accept as common without querying L1.
549
+ if (finalizedL1BlockNumber !== undefined && localMsg.l1BlockNumber <= finalizedL1BlockNumber) {
550
+ this.log.info(`Found common L1 to L2 message at finalized L1 block ${localMsg.l1BlockNumber}`, logCtx);
551
+ commonMsg = localMsg;
552
+ break;
553
+ }
463
554
 
464
- private async rollbackL1ToL2Messages(
465
- localLastMessage: InboxMessage,
466
- messagesSyncPoint: L1BlockId,
467
- ): Promise<L1BlockId> {
468
- // Slowly go back through our messages until we find the last common message.
469
- // We could query the logs in batch as an optimization, but the depth of the reorg should not be deep, and this
470
- // is a very rare case, so it's fine to query one log at a time.
471
- let commonMsg: undefined | InboxMessage;
472
- this.log.verbose(`Searching most recent common L1 to L2 message at or before index ${localLastMessage.index}`);
473
- for await (const msg of this.store.iterateL1ToL2Messages({ reverse: true, end: localLastMessage.index })) {
474
- const remoteMsg = await this.retrieveL1ToL2Message(msg.leaf);
475
- const logCtx = { remoteMsg, localMsg: msg };
476
- if (remoteMsg && remoteMsg.rollingHash.equals(msg.rollingHash)) {
477
- this.log.verbose(
478
- `Found most recent common L1 to L2 message at index ${msg.index} on L1 block ${msg.l1BlockNumber}`,
555
+ // If there's no match with the current remote state, check if the message exists on the inbox contract at all
556
+ // by looking at the inbox events. If the L1 reorg *added* new messages in addition to deleting existing ones,
557
+ // then the current remote state's rolling hash will not match anything we have locally, so we need to check existence
558
+ // of individual messages via logs. Note we use logs and not historical queries so we don't have to depend on
559
+ // an archival rpc node, since the message could be from a long time ago if we're catching up with syncing.
560
+ const remoteMsg = await retrieveL1ToL2Message(this.inbox, localMsg);
561
+ logCtx.remoteMsg = remoteMsg;
562
+ if (remoteMsg && remoteMsg.rollingHash.equals(localMsg.rollingHash)) {
563
+ this.log.info(
564
+ `Found most recent common L1 to L2 message at index ${localMsg.index} on L1 block ${localMsg.l1BlockNumber}`,
479
565
  logCtx,
480
566
  );
481
567
  commonMsg = remoteMsg;
482
568
  break;
483
569
  } else if (remoteMsg) {
484
- this.log.debug(`Local L1 to L2 message with index ${msg.index} has different rolling hash`, logCtx);
570
+ this.log.debug(`Local L1 to L2 message with index ${localMsg.index} has different rolling hash`, logCtx);
571
+ messagesToDelete++;
485
572
  } else {
486
- this.log.debug(`Local L1 to L2 message with index ${msg.index} not found on L1`, logCtx);
573
+ this.log.debug(`Local L1 to L2 message with index ${localMsg.index} not found on L1`, logCtx);
574
+ messagesToDelete++;
487
575
  }
488
576
  }
489
577
 
490
- // Delete everything after the common message we found.
491
- const lastGoodIndex = commonMsg?.index;
492
- this.log.warn(`Deleting all local L1 to L2 messages after index ${lastGoodIndex ?? 'undefined'}`);
493
- await this.store.removeL1ToL2Messages(lastGoodIndex !== undefined ? lastGoodIndex + 1n : 0n);
578
+ // Delete everything after the common message we found, if anything needs to be deleted.
579
+ // Do not exit early if there are no messages to delete, we still want to update the syncpoint.
580
+ if (messagesToDelete > 0) {
581
+ const lastGoodIndex = commonMsg?.index;
582
+ this.log.warn(`Rolling back all local L1 to L2 messages after index ${lastGoodIndex ?? 'initial'}`);
583
+ await this.stores.messages.removeL1ToL2Messages(lastGoodIndex !== undefined ? lastGoodIndex + 1n : 0n);
584
+ }
494
585
 
495
586
  // Update the syncpoint so the loop below reprocesses the changed messages. We go to the block before
496
587
  // the last common one, so we force reprocessing it, in case new messages were added on that same L1 block
497
- // after the last common message.
498
- const syncPointL1BlockNumber = commonMsg ? commonMsg.l1BlockNumber - 1n : this.l1Constants.l1StartBlock;
499
- const syncPointL1BlockHash = await this.getL1BlockHash(syncPointL1BlockNumber);
500
- messagesSyncPoint = { l1BlockNumber: syncPointL1BlockNumber, l1BlockHash: syncPointL1BlockHash };
501
- await this.store.setMessageSynchedL1Block(messagesSyncPoint);
588
+ // after the last common message. Cap at the finalized L1 block: messages at or below finalized cannot
589
+ // have been reorged, so there is no need to walk back any further than that.
590
+ const syncPointL1BlockNumber = maxBigint(
591
+ ...compactArray([
592
+ commonMsg ? commonMsg.l1BlockNumber - 1n : undefined,
593
+ finalizedL1BlockNumber,
594
+ this.l1Constants.l1StartBlock,
595
+ ]),
596
+ );
597
+
598
+ const syncPointL1BlockHash =
599
+ syncPointL1BlockNumber === finalizedL1BlockNumber
600
+ ? messagesFinalizedL1Block!.l1BlockHash
601
+ : await this.getL1BlockHash(syncPointL1BlockNumber);
602
+
603
+ const messagesSyncPoint = { l1BlockNumber: syncPointL1BlockNumber, l1BlockHash: syncPointL1BlockHash };
604
+ await this.stores.messages.setMessageSyncState(messagesSyncPoint, remoteTreeInProgress);
605
+ this.log.verbose(`Updated messages syncpoint to L1 block ${messagesSyncPoint.l1BlockNumber}`, {
606
+ ...messagesSyncPoint,
607
+ remoteTreeInProgress,
608
+ });
502
609
  return messagesSyncPoint;
503
610
  }
504
611
 
@@ -516,9 +623,9 @@ export class ArchiverL1Synchronizer implements Traceable {
516
623
  currentL1BlockNumber: bigint,
517
624
  initialSyncComplete: boolean,
518
625
  ): Promise<RollupStatus> {
519
- const localPendingCheckpointNumber = await this.store.getSynchedCheckpointNumber();
626
+ const localPendingCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
520
627
  const initialValidationResult: ValidateCheckpointResult | undefined =
521
- await this.store.getPendingChainValidationStatus();
628
+ await this.stores.blocks.getPendingChainValidationStatus();
522
629
  const {
523
630
  provenCheckpointNumber,
524
631
  provenArchive,
@@ -548,19 +655,19 @@ export class ArchiverL1Synchronizer implements Traceable {
548
655
  // we need to set it to zero. This is an edge case because we dont have a checkpoint zero (initial checkpoint is one),
549
656
  // so localCheckpointForDestinationProvenCheckpointNumber would not be found below.
550
657
  if (provenCheckpointNumber === 0) {
551
- const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
658
+ const localProvenCheckpointNumber = await this.stores.blocks.getProvenCheckpointNumber();
552
659
  if (localProvenCheckpointNumber !== provenCheckpointNumber) {
553
- await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
660
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
554
661
  this.log.info(`Rolled back proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
555
662
  }
556
663
  }
557
664
 
558
665
  const localCheckpointForDestinationProvenCheckpointNumber =
559
- await this.store.getCheckpointData(provenCheckpointNumber);
666
+ await this.stores.blocks.getCheckpointData(provenCheckpointNumber);
560
667
 
561
668
  // Sanity check. I've hit what seems to be a state where the proven checkpoint is set to a value greater than the latest
562
669
  // synched checkpoint when requesting L2Tips from the archiver. This is the only place where the proven checkpoint is set.
563
- const synched = await this.store.getSynchedCheckpointNumber();
670
+ const synched = await this.stores.blocks.getLatestCheckpointNumber();
564
671
  if (
565
672
  localCheckpointForDestinationProvenCheckpointNumber &&
566
673
  synched < localCheckpointForDestinationProvenCheckpointNumber.checkpointNumber
@@ -580,15 +687,15 @@ export class ArchiverL1Synchronizer implements Traceable {
580
687
  localCheckpointForDestinationProvenCheckpointNumber &&
581
688
  provenArchive.equals(localCheckpointForDestinationProvenCheckpointNumber.archive.root)
582
689
  ) {
583
- const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
690
+ const localProvenCheckpointNumber = await this.stores.blocks.getProvenCheckpointNumber();
584
691
  if (localProvenCheckpointNumber !== provenCheckpointNumber) {
585
- await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
692
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
586
693
  this.log.info(`Updated proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
587
694
  const provenSlotNumber = localCheckpointForDestinationProvenCheckpointNumber.header.slotNumber;
588
695
  const provenEpochNumber: EpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants);
589
696
  const lastBlockNumberInCheckpoint =
590
697
  localCheckpointForDestinationProvenCheckpointNumber.startBlock +
591
- localCheckpointForDestinationProvenCheckpointNumber.numBlocks -
698
+ localCheckpointForDestinationProvenCheckpointNumber.blockCount -
592
699
  1;
593
700
 
594
701
  this.events.emit(L2BlockSourceEvents.L2BlockProven, {
@@ -597,7 +704,7 @@ export class ArchiverL1Synchronizer implements Traceable {
597
704
  slotNumber: provenSlotNumber,
598
705
  epochNumber: provenEpochNumber,
599
706
  });
600
- this.instrumentation.updateLastProvenBlock(lastBlockNumberInCheckpoint);
707
+ this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber);
601
708
  } else {
602
709
  this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`);
603
710
  }
@@ -608,7 +715,7 @@ export class ArchiverL1Synchronizer implements Traceable {
608
715
  // If we have 0 checkpoints locally and there are no checkpoints onchain there is nothing to do.
609
716
  const noCheckpoints = localPendingCheckpointNumber === 0 && pendingCheckpointNumber === 0;
610
717
  if (noCheckpoints) {
611
- await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
718
+ await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
612
719
  this.log.debug(
613
720
  `No checkpoints to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}, no checkpoints on chain`,
614
721
  );
@@ -620,7 +727,7 @@ export class ArchiverL1Synchronizer implements Traceable {
620
727
  // Related to the L2 reorgs of the pending chain. We are only interested in actually addressing a reorg if there
621
728
  // are any state that could be impacted by it. If we have no checkpoints, there is no impact.
622
729
  if (localPendingCheckpointNumber > 0) {
623
- const localPendingCheckpoint = await this.store.getCheckpointData(localPendingCheckpointNumber);
730
+ const localPendingCheckpoint = await this.stores.blocks.getCheckpointData(localPendingCheckpointNumber);
624
731
  if (localPendingCheckpoint === undefined) {
625
732
  throw new Error(`Missing checkpoint ${localPendingCheckpointNumber}`);
626
733
  }
@@ -635,7 +742,7 @@ export class ArchiverL1Synchronizer implements Traceable {
635
742
  // However, in the re-org scenario, our L1 node is temporarily lying to us and we end up potentially missing checkpoints.
636
743
  // We must only set this block number based on actually retrieved logs.
637
744
  // TODO(#8621): Tackle this properly when we handle L1 Re-orgs.
638
- // await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
745
+ // await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
639
746
  this.log.debug(`No checkpoints to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}`);
640
747
  return rollupStatus;
641
748
  }
@@ -655,7 +762,7 @@ export class ArchiverL1Synchronizer implements Traceable {
655
762
 
656
763
  let tipAfterUnwind = localPendingCheckpointNumber;
657
764
  while (true) {
658
- const candidateCheckpoint = await this.store.getCheckpointData(tipAfterUnwind);
765
+ const candidateCheckpoint = await this.stores.blocks.getCheckpointData(tipAfterUnwind);
659
766
  if (candidateCheckpoint === undefined) {
660
767
  break;
661
768
  }
@@ -680,7 +787,7 @@ export class ArchiverL1Synchronizer implements Traceable {
680
787
  this.log.warn(
681
788
  `Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` +
682
789
  `due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` +
683
- `Updated L2 latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
790
+ `Updated L2 latest checkpoint is ${await this.stores.blocks.getLatestCheckpointNumber()}.`,
684
791
  );
685
792
  }
686
793
  }
@@ -697,23 +804,20 @@ export class ArchiverL1Synchronizer implements Traceable {
697
804
 
698
805
  this.log.trace(`Retrieving checkpoints from L1 block ${searchStartBlock} to ${searchEndBlock}`);
699
806
 
700
- // TODO(md): Retrieve from blob client then from consensus client, then from peers
701
- const retrievedCheckpoints = await execInSpan(this.tracer, 'Archiver.retrieveCheckpointsFromRollup', () =>
702
- retrieveCheckpointsFromRollup(
807
+ // First fetch calldata only, no blobs yet, since we may be able to just get that data out of the proposed chain
808
+ const calldataCheckpoints = await execInSpan(this.tracer, 'Archiver.retrieveCheckpointCalldataFromRollup', () =>
809
+ retrieveCheckpointCalldataFromRollup(
703
810
  this.rollup,
704
811
  this.publicClient,
705
812
  this.debugClient,
706
- this.blobClient,
707
813
  searchStartBlock, // TODO(palla/reorg): If the L2 reorg was due to an L1 reorg, we need to start search earlier
708
814
  searchEndBlock,
709
- this.l1Addresses,
710
815
  this.instrumentation,
711
816
  this.log,
712
- !initialSyncComplete, // isHistoricalSync
713
817
  ),
714
818
  );
715
819
 
716
- if (retrievedCheckpoints.length === 0) {
820
+ if (calldataCheckpoints.length === 0) {
717
821
  // We are not calling `setBlockSynchedL1BlockNumber` because it may cause sync issues if based off infura.
718
822
  // See further details in earlier comments.
719
823
  this.log.trace(`Retrieved no new checkpoints from L1 block ${searchStartBlock} to ${searchEndBlock}`);
@@ -721,21 +825,56 @@ export class ArchiverL1Synchronizer implements Traceable {
721
825
  }
722
826
 
723
827
  this.log.debug(
724
- `Retrieved ${retrievedCheckpoints.length} new checkpoints between L1 blocks ${searchStartBlock} and ${searchEndBlock}`,
828
+ `Retrieved ${calldataCheckpoints.length} new checkpoint calldata between L1 blocks ${searchStartBlock} and ${searchEndBlock}`,
725
829
  {
726
- lastProcessedCheckpoint: retrievedCheckpoints[retrievedCheckpoints.length - 1].l1,
830
+ lastProcessedCheckpoint: calldataCheckpoints[calldataCheckpoints.length - 1].l1,
727
831
  searchStartBlock,
728
832
  searchEndBlock,
729
833
  },
730
834
  );
731
835
 
732
- const publishedCheckpoints = await Promise.all(retrievedCheckpoints.map(b => retrievedToPublishedCheckpoint(b)));
836
+ // Check if the last checkpoint matches a local pending entry (so we can skip blob fetch).
837
+ // We only check the last one; if it matches, the blob fetch is skipped for that entry.
838
+ // TODO(palla/pipelining): We may have more than a single checkpoint to promote
839
+ const lastCalldataCheckpoint = calldataCheckpoints[calldataCheckpoints.length - 1];
840
+ const promoteResult = await this.tryBuildPublishedCheckpointFromProposed(lastCalldataCheckpoint);
841
+ const checkpointToPromote = promoteResult && !('diverged' in promoteResult) ? promoteResult : undefined;
842
+ const evictProposedFrom =
843
+ promoteResult && 'diverged' in promoteResult ? promoteResult.fromCheckpointNumber : undefined;
844
+
845
+ // Then fetch blobs in parallel and build the full published checkpoints
846
+ const toFetchBlobs = checkpointToPromote ? calldataCheckpoints.slice(0, -1) : calldataCheckpoints;
847
+ const blobFetched = await asyncPool(10, toFetchBlobs, async checkpoint =>
848
+ retrievedToPublishedCheckpoint({
849
+ ...checkpoint,
850
+ checkpointBlobData: await getCheckpointBlobDataFromBlobs(
851
+ this.blobClient,
852
+ checkpoint.l1.blockHash,
853
+ checkpoint.blobHashes,
854
+ checkpoint.checkpointNumber,
855
+ this.log,
856
+ !initialSyncComplete,
857
+ checkpoint.parentBeaconBlockRoot,
858
+ checkpoint.l1.timestamp,
859
+ ),
860
+ }),
861
+ );
862
+
863
+ // And add the promoted checkpoint to the list of all checkpoints
864
+ const publishedCheckpoints = checkpointToPromote ? [...blobFetched, checkpointToPromote] : blobFetched;
733
865
  const validCheckpoints: PublishedCheckpoint[] = [];
734
866
 
867
+ // Now loop through all checkpoints and validate their attestations
735
868
  for (const published of publishedCheckpoints) {
736
869
  const validationResult = this.config.skipValidateCheckpointAttestations
737
870
  ? { valid: true as const }
738
- : await validateCheckpointAttestations(published, this.epochCache, this.l1Constants, this.log);
871
+ : await validateCheckpointAttestations(
872
+ published,
873
+ this.epochCache,
874
+ this.l1Constants,
875
+ this.getSignatureContext(),
876
+ this.log,
877
+ );
739
878
 
740
879
  // Only update the validation result if it has changed, so we can keep track of the first invalid checkpoint
741
880
  // in case there is a sequence of more than one invalid checkpoint, as we need to invalidate the first one.
@@ -773,7 +912,7 @@ export class ArchiverL1Synchronizer implements Traceable {
773
912
  // Check the inHash of the checkpoint against the l1->l2 messages.
774
913
  // The messages should've been synced up to the currentL1BlockNumber and must be available for the published
775
914
  // checkpoints we just retrieved.
776
- const l1ToL2Messages = await this.store.getL1ToL2Messages(published.checkpoint.number);
915
+ const l1ToL2Messages = await this.stores.messages.getL1ToL2Messages(published.checkpoint.number);
777
916
  const computedInHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
778
917
  const publishedInHash = published.checkpoint.header.inHash;
779
918
  if (!computedInHash.equals(publishedInHash)) {
@@ -801,25 +940,53 @@ export class ArchiverL1Synchronizer implements Traceable {
801
940
  );
802
941
  }
803
942
 
943
+ for (const published of validCheckpoints) {
944
+ this.instrumentation.processCheckpointL1Timing({
945
+ slotNumber: published.checkpoint.header.slotNumber,
946
+ l1Timestamp: published.l1.timestamp,
947
+ l1Constants: this.l1Constants,
948
+ });
949
+ }
950
+
804
951
  try {
805
952
  const updatedValidationResult =
806
953
  rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
954
+
955
+ // Split valid checkpoints: the promoted one (if any) is persisted via the proposed-promotion path,
956
+ // the rest via addCheckpoints. Both paths run within the same store transaction for atomicity.
957
+ const [[maybeValidCheckpointToPromote], checkpointsToAdd] = partition(
958
+ validCheckpoints,
959
+ c => c.checkpoint.number === checkpointToPromote?.checkpoint.number,
960
+ );
961
+
807
962
  const [processDuration, result] = await elapsed(() =>
808
963
  execInSpan(this.tracer, 'Archiver.addCheckpoints', () =>
809
- this.updater.addCheckpoints(validCheckpoints, updatedValidationResult),
964
+ this.updater.addCheckpoints(
965
+ checkpointsToAdd,
966
+ updatedValidationResult,
967
+ maybeValidCheckpointToPromote && {
968
+ l1: lastCalldataCheckpoint.l1,
969
+ attestations: lastCalldataCheckpoint.attestations,
970
+ checkpoint: maybeValidCheckpointToPromote,
971
+ },
972
+ evictProposedFrom,
973
+ ),
810
974
  ),
811
975
  );
812
- this.instrumentation.processNewBlocks(
813
- processDuration / validCheckpoints.length,
814
- validCheckpoints.flatMap(c => c.checkpoint.blocks),
815
- );
976
+
977
+ if (checkpointsToAdd.length > 0) {
978
+ this.instrumentation.processNewCheckpointedBlocks(
979
+ processDuration / checkpointsToAdd.length,
980
+ checkpointsToAdd.flatMap(c => c.checkpoint.blocks),
981
+ );
982
+ }
816
983
 
817
984
  // If blocks were pruned due to conflict with L1 checkpoints, emit event
818
985
  if (result.prunedBlocks && result.prunedBlocks.length > 0) {
819
986
  const prunedCheckpointNumber = result.prunedBlocks[0].checkpointNumber;
820
987
  const prunedSlotNumber = result.prunedBlocks[0].header.globalVariables.slotNumber;
821
988
 
822
- this.log.warn(
989
+ this.log.info(
823
990
  `Pruned ${result.prunedBlocks.length} mismatching blocks for checkpoint ${prunedCheckpointNumber}`,
824
991
  { prunedBlocks: result.prunedBlocks.map(b => b.toBlockInfo()), prunedSlotNumber, prunedCheckpointNumber },
825
992
  );
@@ -837,10 +1004,10 @@ export class ArchiverL1Synchronizer implements Traceable {
837
1004
  if (err instanceof InitialCheckpointNumberNotSequentialError) {
838
1005
  const { previousCheckpointNumber, newCheckpointNumber } = err;
839
1006
  const previousCheckpoint = previousCheckpointNumber
840
- ? await this.store.getCheckpointData(CheckpointNumber(previousCheckpointNumber))
1007
+ ? await this.stores.blocks.getCheckpointData(CheckpointNumber(previousCheckpointNumber))
841
1008
  : undefined;
842
1009
  const updatedL1SyncPoint = previousCheckpoint?.l1.blockNumber ?? this.l1Constants.l1StartBlock;
843
- await this.store.setCheckpointSynchedL1BlockNumber(updatedL1SyncPoint);
1010
+ await this.stores.blocks.setSynchedL1BlockNumber(updatedL1SyncPoint);
844
1011
  this.log.warn(
845
1012
  `Attempting to insert checkpoint ${newCheckpointNumber} with previous block ${previousCheckpointNumber}. Rolling back L1 sync point to ${updatedL1SyncPoint} to try and fetch the missing blocks.`,
846
1013
  {
@@ -865,7 +1032,7 @@ export class ArchiverL1Synchronizer implements Traceable {
865
1032
  });
866
1033
  }
867
1034
  lastRetrievedCheckpoint = validCheckpoints.at(-1) ?? lastRetrievedCheckpoint;
868
- lastL1BlockWithCheckpoint = retrievedCheckpoints.at(-1)?.l1.blockNumber ?? lastL1BlockWithCheckpoint;
1035
+ lastL1BlockWithCheckpoint = calldataCheckpoints.at(-1)?.l1.blockNumber ?? lastL1BlockWithCheckpoint;
869
1036
  } while (searchEndBlock < currentL1BlockNumber);
870
1037
 
871
1038
  // Important that we update AFTER inserting the blocks.
@@ -874,6 +1041,82 @@ export class ArchiverL1Synchronizer implements Traceable {
874
1041
  return { ...rollupStatus, lastRetrievedCheckpoint, lastL1BlockWithCheckpoint };
875
1042
  }
876
1043
 
1044
+ /**
1045
+ * Checks if a specific checkpoint matches a local pending entry, and if so, loads local data to build
1046
+ * a synthetic published checkpoint (skipping blob fetch).
1047
+ *
1048
+ * Returns { diverged: true, fromCheckpointNumber } when the L1 checkpoint does NOT match local pending
1049
+ * data for that number, so the caller can evict the entire pending suffix >= fromCheckpointNumber
1050
+ * (those entries chain off the now-invalid local state) within the same addCheckpoints transaction.
1051
+ */
1052
+ private async tryBuildPublishedCheckpointFromProposed(
1053
+ calldataCheckpoint: RetrievedCheckpointFromCalldata | undefined,
1054
+ ): Promise<PublishedCheckpoint | { diverged: true; fromCheckpointNumber: CheckpointNumber } | undefined> {
1055
+ if (this.config.skipPromoteProposedCheckpointDuringL1Sync || !calldataCheckpoint) {
1056
+ return undefined;
1057
+ }
1058
+
1059
+ // Look up the specific pending entry for the checkpoint being mined, not just the tip
1060
+ const proposed = await this.stores.blocks.getProposedCheckpointByNumber(calldataCheckpoint.checkpointNumber);
1061
+ if (!proposed) {
1062
+ return undefined;
1063
+ }
1064
+
1065
+ if (
1066
+ !proposed.header.equals(calldataCheckpoint.header) ||
1067
+ !proposed.archive.root.equals(calldataCheckpoint.archiveRoot)
1068
+ ) {
1069
+ this.log.warn(
1070
+ `Local proposed checkpoint ${proposed.checkpointNumber} does not match checkpoint retrieved from L1, overriding with L1 data`,
1071
+ {
1072
+ proposedCheckpointNumber: proposed.checkpointNumber,
1073
+ proposedHeader: proposed.header.toInspect(),
1074
+ proposedArchiveRoot: proposed.archive.root.toString(),
1075
+ calldataCheckpointNumber: calldataCheckpoint.checkpointNumber,
1076
+ calldataHeader: calldataCheckpoint.header.toInspect(),
1077
+ calldataArchiveRoot: calldataCheckpoint.archiveRoot.toString(),
1078
+ },
1079
+ );
1080
+ // Return a divergence signal so the caller can evict pending >= this number
1081
+ return { diverged: true, fromCheckpointNumber: proposed.checkpointNumber };
1082
+ }
1083
+
1084
+ this.log.debug(
1085
+ `Building published checkpoint from proposed ${calldataCheckpoint.checkpointNumber} (skipping blob fetch)`,
1086
+ { proposedHeader: proposed.header.toInspect(), proposedArchiveRoot: proposed.archive.root.toString() },
1087
+ );
1088
+
1089
+ const blocks = await this.stores.blocks.getBlocks(BlockNumber(proposed.startBlock), proposed.blockCount);
1090
+ if (blocks.length !== proposed.blockCount) {
1091
+ this.log.warn(
1092
+ `Local proposed checkpoint ${proposed.checkpointNumber} has wrong block count (expected ${proposed.blockCount} blocks starting at ${proposed.startBlock} but got ${blocks.length})`,
1093
+ {
1094
+ proposedCheckpointNumber: proposed.checkpointNumber,
1095
+ proposedStartBlock: proposed.startBlock,
1096
+ proposedBlockCount: proposed.blockCount,
1097
+ retrievedBlocks: blocks.map(b => b.number),
1098
+ },
1099
+ );
1100
+ return undefined;
1101
+ }
1102
+
1103
+ const checkpoint = Checkpoint.from({
1104
+ archive: proposed.archive,
1105
+ header: proposed.header,
1106
+ blocks,
1107
+ number: proposed.checkpointNumber,
1108
+ feeAssetPriceModifier: proposed.feeAssetPriceModifier,
1109
+ });
1110
+ const promotedCheckpoint = PublishedCheckpoint.from({
1111
+ checkpoint,
1112
+ l1: calldataCheckpoint.l1,
1113
+ attestations: calldataCheckpoint.attestations,
1114
+ });
1115
+ this.instrumentation.processCheckpointPromoted();
1116
+
1117
+ return promotedCheckpoint;
1118
+ }
1119
+
877
1120
  private async checkForNewCheckpointsBeforeL1SyncPoint(
878
1121
  status: RollupStatus,
879
1122
  blocksSynchedTo: bigint,
@@ -883,7 +1126,7 @@ export class ArchiverL1Synchronizer implements Traceable {
883
1126
  // Compare the last checkpoint we have (either retrieved in this round or loaded from store) with what the
884
1127
  // rollup contract told us was the latest one (pinned at the currentL1BlockNumber).
885
1128
  const latestLocalCheckpointNumber =
886
- lastRetrievedCheckpoint?.checkpoint.number ?? (await this.store.getSynchedCheckpointNumber());
1129
+ lastRetrievedCheckpoint?.checkpoint.number ?? (await this.stores.blocks.getLatestCheckpointNumber());
887
1130
  if (latestLocalCheckpointNumber < pendingCheckpointNumber) {
888
1131
  // Here we have consumed all logs until the `currentL1Block` we pinned at the beginning of the archiver loop,
889
1132
  // but still haven't reached the pending checkpoint according to the call to the rollup contract.
@@ -896,7 +1139,9 @@ export class ArchiverL1Synchronizer implements Traceable {
896
1139
  latestLocalCheckpointArchive = lastRetrievedCheckpoint.checkpoint.archive.root.toString();
897
1140
  targetL1BlockNumber = lastRetrievedCheckpoint.l1.blockNumber;
898
1141
  } else if (latestLocalCheckpointNumber > 0) {
899
- const checkpoint = await this.store.getRangeOfCheckpoints(latestLocalCheckpointNumber, 1).then(([c]) => c);
1142
+ const checkpoint = await this.stores.blocks
1143
+ .getRangeOfCheckpoints(latestLocalCheckpointNumber, 1)
1144
+ .then(([c]) => c);
900
1145
  latestLocalCheckpointArchive = checkpoint.archive.root.toString();
901
1146
  targetL1BlockNumber = checkpoint.l1.blockNumber;
902
1147
  }
@@ -911,7 +1156,7 @@ export class ArchiverL1Synchronizer implements Traceable {
911
1156
  ...status,
912
1157
  },
913
1158
  );
914
- await this.store.setCheckpointSynchedL1BlockNumber(targetL1BlockNumber);
1159
+ await this.stores.blocks.setSynchedL1BlockNumber(targetL1BlockNumber);
915
1160
  } else {
916
1161
  this.log.trace(`No new checkpoints behind L1 sync point to retrieve.`, {
917
1162
  latestLocalCheckpointNumber,
@@ -921,7 +1166,7 @@ export class ArchiverL1Synchronizer implements Traceable {
921
1166
  }
922
1167
 
923
1168
  private async getCheckpointHeader(number: CheckpointNumber) {
924
- const checkpoint = await this.store.getCheckpointData(number);
1169
+ const checkpoint = await this.stores.blocks.getCheckpointData(number);
925
1170
  if (!checkpoint) {
926
1171
  return undefined;
927
1172
  }