@aztec/archiver 0.0.1-commit.f2ce05ee → 0.0.1-commit.f5a9928

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +19 -13
  2. package/dest/archiver.d.ts +76 -18
  3. package/dest/archiver.d.ts.map +1 -1
  4. package/dest/archiver.js +329 -162
  5. package/dest/config.d.ts +7 -3
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +43 -14
  8. package/dest/errors.d.ts +55 -9
  9. package/dest/errors.d.ts.map +1 -1
  10. package/dest/errors.js +81 -14
  11. package/dest/factory.d.ts +20 -9
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +87 -39
  14. package/dest/index.d.ts +12 -3
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +11 -2
  17. package/dest/l1/bin/retrieve-calldata.js +32 -28
  18. package/dest/l1/calldata_retriever.d.ts +81 -50
  19. package/dest/l1/calldata_retriever.d.ts.map +1 -1
  20. package/dest/l1/calldata_retriever.js +202 -260
  21. package/dest/l1/data_retrieval.d.ts +32 -18
  22. package/dest/l1/data_retrieval.d.ts.map +1 -1
  23. package/dest/l1/data_retrieval.js +42 -47
  24. package/dest/l1/spire_proposer.d.ts +5 -5
  25. package/dest/l1/spire_proposer.d.ts.map +1 -1
  26. package/dest/l1/spire_proposer.js +9 -17
  27. package/dest/l1/trace_tx.d.ts +12 -66
  28. package/dest/l1/trace_tx.d.ts.map +1 -1
  29. package/dest/l1/validate_historical_logs.d.ts +23 -0
  30. package/dest/l1/validate_historical_logs.d.ts.map +1 -0
  31. package/dest/l1/validate_historical_logs.js +108 -0
  32. package/dest/modules/contract_data_source_adapter.d.ts +25 -0
  33. package/dest/modules/contract_data_source_adapter.d.ts.map +1 -0
  34. package/dest/modules/contract_data_source_adapter.js +32 -0
  35. package/dest/modules/data_source_base.d.ts +74 -46
  36. package/dest/modules/data_source_base.d.ts.map +1 -1
  37. package/dest/modules/data_source_base.js +252 -190
  38. package/dest/modules/data_store_updater.d.ts +49 -17
  39. package/dest/modules/data_store_updater.d.ts.map +1 -1
  40. package/dest/modules/data_store_updater.js +218 -122
  41. package/dest/modules/instrumentation.d.ts +29 -3
  42. package/dest/modules/instrumentation.d.ts.map +1 -1
  43. package/dest/modules/instrumentation.js +67 -11
  44. package/dest/modules/l1_synchronizer.d.ts +16 -14
  45. package/dest/modules/l1_synchronizer.d.ts.map +1 -1
  46. package/dest/modules/l1_synchronizer.js +488 -219
  47. package/dest/modules/outbox_trees_resolver.d.ts +64 -0
  48. package/dest/modules/outbox_trees_resolver.d.ts.map +1 -0
  49. package/dest/modules/outbox_trees_resolver.js +184 -0
  50. package/dest/modules/validation.d.ts +30 -7
  51. package/dest/modules/validation.d.ts.map +1 -1
  52. package/dest/modules/validation.js +35 -17
  53. package/dest/store/block_store.d.ts +186 -74
  54. package/dest/store/block_store.d.ts.map +1 -1
  55. package/dest/store/block_store.js +810 -285
  56. package/dest/store/contract_class_store.d.ts +17 -4
  57. package/dest/store/contract_class_store.d.ts.map +1 -1
  58. package/dest/store/contract_class_store.js +42 -68
  59. package/dest/store/contract_instance_store.d.ts +28 -1
  60. package/dest/store/contract_instance_store.d.ts.map +1 -1
  61. package/dest/store/contract_instance_store.js +58 -6
  62. package/dest/store/data_stores.d.ts +68 -0
  63. package/dest/store/data_stores.d.ts.map +1 -0
  64. package/dest/store/data_stores.js +54 -0
  65. package/dest/store/function_names_cache.d.ts +17 -0
  66. package/dest/store/function_names_cache.d.ts.map +1 -0
  67. package/dest/store/function_names_cache.js +30 -0
  68. package/dest/store/l2_tips_cache.d.ts +25 -0
  69. package/dest/store/l2_tips_cache.d.ts.map +1 -0
  70. package/dest/store/l2_tips_cache.js +26 -0
  71. package/dest/store/log_store.d.ts +42 -37
  72. package/dest/store/log_store.d.ts.map +1 -1
  73. package/dest/store/log_store.js +262 -408
  74. package/dest/store/log_store_codec.d.ts +78 -0
  75. package/dest/store/log_store_codec.d.ts.map +1 -0
  76. package/dest/store/log_store_codec.js +110 -0
  77. package/dest/store/message_store.d.ts +11 -1
  78. package/dest/store/message_store.d.ts.map +1 -1
  79. package/dest/store/message_store.js +51 -9
  80. package/dest/test/fake_l1_state.d.ts +25 -1
  81. package/dest/test/fake_l1_state.d.ts.map +1 -1
  82. package/dest/test/fake_l1_state.js +166 -32
  83. package/dest/test/mock_archiver.d.ts +1 -1
  84. package/dest/test/mock_archiver.d.ts.map +1 -1
  85. package/dest/test/mock_archiver.js +3 -2
  86. package/dest/test/mock_l2_block_source.d.ts +61 -47
  87. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  88. package/dest/test/mock_l2_block_source.js +306 -211
  89. package/dest/test/mock_structs.d.ts +4 -1
  90. package/dest/test/mock_structs.d.ts.map +1 -1
  91. package/dest/test/mock_structs.js +15 -3
  92. package/dest/test/noop_l1_archiver.d.ts +17 -6
  93. package/dest/test/noop_l1_archiver.d.ts.map +1 -1
  94. package/dest/test/noop_l1_archiver.js +31 -11
  95. package/package.json +15 -14
  96. package/src/archiver.ts +427 -185
  97. package/src/config.ts +51 -13
  98. package/src/errors.ts +122 -21
  99. package/src/factory.ts +104 -29
  100. package/src/index.ts +19 -2
  101. package/src/l1/README.md +25 -68
  102. package/src/l1/bin/retrieve-calldata.ts +40 -27
  103. package/src/l1/calldata_retriever.ts +267 -379
  104. package/src/l1/data_retrieval.ts +66 -70
  105. package/src/l1/spire_proposer.ts +7 -15
  106. package/src/l1/validate_historical_logs.ts +140 -0
  107. package/src/modules/contract_data_source_adapter.ts +46 -0
  108. package/src/modules/data_source_base.ts +337 -238
  109. package/src/modules/data_store_updater.ts +259 -154
  110. package/src/modules/instrumentation.ts +71 -11
  111. package/src/modules/l1_synchronizer.ts +659 -269
  112. package/src/modules/outbox_trees_resolver.ts +220 -0
  113. package/src/modules/validation.ts +80 -23
  114. package/src/store/block_store.ts +989 -358
  115. package/src/store/contract_class_store.ts +49 -103
  116. package/src/store/contract_instance_store.ts +74 -11
  117. package/src/store/data_stores.ts +104 -0
  118. package/src/store/function_names_cache.ts +37 -0
  119. package/src/store/l2_tips_cache.ts +35 -0
  120. package/src/store/log_store.ts +302 -559
  121. package/src/store/log_store_codec.ts +143 -0
  122. package/src/store/message_store.ts +60 -10
  123. package/src/structs/inbox_message.ts +1 -1
  124. package/src/test/fake_l1_state.ts +213 -42
  125. package/src/test/mock_archiver.ts +3 -2
  126. package/src/test/mock_l2_block_source.ts +378 -233
  127. package/src/test/mock_structs.ts +25 -8
  128. package/src/test/noop_l1_archiver.ts +61 -12
  129. package/dest/store/kv_archiver_store.d.ts +0 -340
  130. package/dest/store/kv_archiver_store.d.ts.map +0 -1
  131. package/dest/store/kv_archiver_store.js +0 -446
  132. package/src/store/kv_archiver_store.ts +0 -639
@@ -1,37 +1,56 @@
1
1
  import type { BlobClientInterface } from '@aztec/blob-client/client';
2
2
  import { EpochCache } from '@aztec/epoch-cache';
3
- import { InboxContract, RollupContract } from '@aztec/ethereum/contracts';
4
- import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
3
+ import { InboxContract, type InboxContractState, RollupContract } from '@aztec/ethereum/contracts';
5
4
  import type { L1BlockId } from '@aztec/ethereum/l1-types';
5
+ import { getFinalizedL1Block } from '@aztec/ethereum/queries';
6
6
  import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
7
+ import { asyncPool } from '@aztec/foundation/async-pool';
7
8
  import { maxBigint } from '@aztec/foundation/bigint';
8
9
  import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
9
- import { Buffer32 } from '@aztec/foundation/buffer';
10
- import { pick } from '@aztec/foundation/collection';
10
+ import { Buffer16, Buffer32 } from '@aztec/foundation/buffer';
11
+ import { compactArray, partition, pick } from '@aztec/foundation/collection';
11
12
  import { Fr } from '@aztec/foundation/curves/bn254';
12
13
  import { EthAddress } from '@aztec/foundation/eth-address';
13
14
  import { type Logger, createLogger } from '@aztec/foundation/log';
15
+ import { retryTimes } from '@aztec/foundation/retry';
14
16
  import { count } from '@aztec/foundation/string';
15
17
  import { DateProvider, Timer, elapsed } from '@aztec/foundation/timer';
16
- import { isDefined } from '@aztec/foundation/types';
17
- import { type ArchiverEmitter, L2BlockSourceEvents, type ValidateCheckpointResult } from '@aztec/stdlib/block';
18
- import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
18
+ import { isDefined, isErrorClass } from '@aztec/foundation/types';
19
+ import {
20
+ type ArchiverEmitter,
21
+ type L2Block,
22
+ L2BlockSourceEvents,
23
+ type ValidateCheckpointResult,
24
+ } from '@aztec/stdlib/block';
25
+ import {
26
+ Checkpoint,
27
+ type CheckpointData,
28
+ type CheckpointInfo,
29
+ type L1PublishedData,
30
+ PublishedCheckpoint,
31
+ } from '@aztec/stdlib/checkpoint';
19
32
  import { type L1RollupConstants, getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers';
20
33
  import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
34
+ import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
21
35
  import { type Traceable, type Tracer, execInSpan, trackSpan } from '@aztec/telemetry-client';
22
36
 
23
37
  import { InitialCheckpointNumberNotSequentialError } from '../errors.js';
24
38
  import {
25
- retrieveCheckpointsFromRollup,
39
+ type RetrievedCheckpointFromCalldata,
40
+ getCheckpointBlobDataFromBlobs,
41
+ retrieveCheckpointCalldataFromRollup,
26
42
  retrieveL1ToL2Message,
27
43
  retrieveL1ToL2Messages,
28
44
  retrievedToPublishedCheckpoint,
29
45
  } from '../l1/data_retrieval.js';
30
- import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
46
+ import type { RejectedCheckpoint } from '../store/block_store.js';
47
+ import { type ArchiverDataStores, getArchiverSynchPoint } from '../store/data_stores.js';
48
+ import type { L2TipsCache } from '../store/l2_tips_cache.js';
49
+ import { MessageStoreError } from '../store/message_store.js';
31
50
  import type { InboxMessage } from '../structs/inbox_message.js';
32
51
  import { ArchiverDataStoreUpdater } from './data_store_updater.js';
33
52
  import type { ArchiverInstrumentation } from './instrumentation.js';
34
- import { validateCheckpointAttestations } from './validation.js';
53
+ import { validateCheckpointAttestationsFromCalldata } from './validation.js';
35
54
 
36
55
  type RollupStatus = {
37
56
  provenCheckpointNumber: CheckpointNumber;
@@ -39,13 +58,17 @@ type RollupStatus = {
39
58
  pendingCheckpointNumber: CheckpointNumber;
40
59
  pendingArchive: string;
41
60
  validationResult: ValidateCheckpointResult | undefined;
61
+ /** Last valid checkpoint observed on L1 and synced on this iteration */
42
62
  lastRetrievedCheckpoint?: PublishedCheckpoint;
43
- lastL1BlockWithCheckpoint?: bigint;
63
+ /** Last checkpoint observed on L1 across both valid and rejected entries on this iteration */
64
+ lastSeenCheckpoint?: { checkpointNumber: CheckpointNumber; l1: L1PublishedData };
65
+ /** Blocks added while handling checkpoints this iteration, for the aggregate block source update event. */
66
+ blocksAdded: L2Block[];
44
67
  };
45
68
 
46
69
  /**
47
70
  * Handles L1 synchronization for the archiver.
48
- * Responsible for fetching checkpoints, L1L2 messages, and handling L1 reorgs.
71
+ * Responsible for fetching checkpoints, L1 to L2 messages, and handling L1 reorgs.
49
72
  */
50
73
  export class ArchiverL1Synchronizer implements Traceable {
51
74
  private l1BlockNumber: bigint | undefined;
@@ -60,26 +83,29 @@ export class ArchiverL1Synchronizer implements Traceable {
60
83
  private readonly debugClient: ViemPublicDebugClient,
61
84
  private readonly rollup: RollupContract,
62
85
  private readonly inbox: InboxContract,
63
- private readonly l1Addresses: Pick<
64
- L1ContractAddresses,
65
- 'registryAddress' | 'governanceProposerAddress' | 'slashFactoryAddress'
66
- > & { slashingProposerAddress: EthAddress },
67
- private readonly store: KVArchiverDataStore,
86
+ private readonly stores: ArchiverDataStores,
68
87
  private config: {
69
88
  batchSize: number;
70
89
  skipValidateCheckpointAttestations?: boolean;
90
+ skipPromoteProposedCheckpointDuringL1Sync?: boolean;
71
91
  maxAllowedEthClientDriftSeconds: number;
72
92
  },
73
93
  private readonly blobClient: BlobClientInterface,
74
94
  private readonly epochCache: EpochCache,
75
95
  private readonly dateProvider: DateProvider,
76
96
  private readonly instrumentation: ArchiverInstrumentation,
77
- private readonly l1Constants: L1RollupConstants & { l1StartBlockHash: Buffer32; genesisArchiveRoot: Fr },
97
+ private readonly l1Constants: L1RollupConstants & {
98
+ l1StartBlockHash: Buffer32;
99
+ genesisArchiveRoot: Fr;
100
+ },
78
101
  private readonly events: ArchiverEmitter,
79
102
  tracer: Tracer,
103
+ l2TipsCache?: L2TipsCache,
80
104
  private readonly log: Logger = createLogger('archiver:l1-sync'),
81
105
  ) {
82
- this.updater = new ArchiverDataStoreUpdater(this.store);
106
+ this.updater = new ArchiverDataStoreUpdater(this.stores, l2TipsCache, {
107
+ rollupManaLimit: l1Constants.rollupManaLimit,
108
+ });
83
109
  this.tracer = tracer;
84
110
  }
85
111
 
@@ -87,6 +113,7 @@ export class ArchiverL1Synchronizer implements Traceable {
87
113
  public setConfig(newConfig: {
88
114
  batchSize: number;
89
115
  skipValidateCheckpointAttestations?: boolean;
116
+ skipPromoteProposedCheckpointDuringL1Sync?: boolean;
90
117
  maxAllowedEthClientDriftSeconds: number;
91
118
  }) {
92
119
  this.config = newConfig;
@@ -102,6 +129,13 @@ export class ArchiverL1Synchronizer implements Traceable {
102
129
  return this.l1Timestamp;
103
130
  }
104
131
 
132
+ private getSignatureContext(): CoordinationSignatureContext {
133
+ return {
134
+ chainId: this.publicClient.chain.id,
135
+ rollupAddress: EthAddress.fromString(this.rollup.address),
136
+ };
137
+ }
138
+
105
139
  /** Checks that the ethereum node we are connected to has a latest timestamp no more than the allowed drift. Throw if not. */
106
140
  public async testEthereumNodeSynced(): Promise<void> {
107
141
  const maxAllowedDelay = this.config.maxAllowedEthClientDriftSeconds;
@@ -118,70 +152,60 @@ export class ArchiverL1Synchronizer implements Traceable {
118
152
  }
119
153
 
120
154
  @trackSpan('Archiver.syncFromL1')
121
- public async syncFromL1(initialSyncComplete: boolean): Promise<void> {
155
+ public async syncFromL1(initialSyncComplete: boolean): Promise<L2Block[]> {
156
+ const blocksAdded: L2Block[] = [];
157
+
158
+ // In between the various calls to L1, the block number can move meaning some of the following
159
+ // calls will return data for blocks that were not present during earlier calls. To combat this
160
+ // we ensure that all data retrieval methods only retrieve data up to the currentBlockNumber
161
+ // captured at the top of this function.
122
162
  const currentL1Block = await this.publicClient.getBlock({ includeTransactions: false });
123
163
  const currentL1BlockNumber = currentL1Block.number;
124
164
  const currentL1BlockHash = Buffer32.fromString(currentL1Block.hash);
125
165
  const currentL1Timestamp = currentL1Block.timestamp;
166
+ const currentL1BlockData = { l1BlockNumber: currentL1BlockNumber, l1BlockHash: currentL1BlockHash };
126
167
 
127
168
  if (this.l1BlockHash && currentL1BlockHash.equals(this.l1BlockHash)) {
128
169
  this.log.trace(`No new L1 blocks since last sync at L1 block ${this.l1BlockNumber}`);
129
- return;
170
+ return blocksAdded;
130
171
  }
131
172
 
132
- // Warn if the latest L1 block timestamp is too old
173
+ // Log at error if the latest L1 block timestamp is too old
133
174
  const maxAllowedDelay = this.config.maxAllowedEthClientDriftSeconds;
134
175
  const now = this.dateProvider.nowInSeconds();
135
176
  if (maxAllowedDelay > 0 && Number(currentL1Timestamp) <= now - maxAllowedDelay) {
136
- this.log.warn(
177
+ this.log.error(
137
178
  `Latest L1 block ${currentL1BlockNumber} timestamp ${currentL1Timestamp} is too old. Make sure your Ethereum node is synced.`,
138
179
  { currentL1BlockNumber, currentL1Timestamp, now, maxAllowedDelay },
139
180
  );
140
181
  }
141
182
 
142
- // Load sync point for blocks and messages defaulting to start block
143
- const {
144
- blocksSynchedTo = this.l1Constants.l1StartBlock,
145
- messagesSynchedTo = {
146
- l1BlockNumber: this.l1Constants.l1StartBlock,
147
- l1BlockHash: this.l1Constants.l1StartBlockHash,
148
- },
149
- } = await this.store.getSynchPoint();
183
+ // Query finalized block on L1
184
+ const rawFinalizedL1Block = await getFinalizedL1Block(this.publicClient);
185
+ const finalizedL1Block: L1BlockId | undefined = rawFinalizedL1Block && {
186
+ l1BlockNumber: rawFinalizedL1Block.number,
187
+ l1BlockHash: Buffer32.fromString(rawFinalizedL1Block.hash),
188
+ };
150
189
 
151
- this.log.debug(`Starting new archiver sync iteration`, {
152
- blocksSynchedTo,
153
- messagesSynchedTo,
154
- currentL1BlockNumber,
155
- currentL1BlockHash,
156
- });
190
+ // Load sync point for blocks defaulting to start block
191
+ const { blocksSynchedTo = this.l1Constants.l1StartBlock } = await getArchiverSynchPoint(this.stores);
192
+ this.log.debug(`Starting new archiver sync iteration`, { blocksSynchedTo, currentL1BlockData, finalizedL1Block });
193
+
194
+ // Sync L1 to L2 messages. We retry this a few times since there are error conditions that reset the sync point, requiring a new iteration.
195
+ // Note that we cannot just wait for the l1 synchronizer to loop again, since the synchronizer would report as synced up to the current L1
196
+ // block, when that wouldn't be the case, since L1 to L2 messages would need another iteration.
197
+ await retryTimes(
198
+ () => this.handleL1ToL2Messages(currentL1BlockData, finalizedL1Block),
199
+ 'Handling L1 to L2 messages',
200
+ 3,
201
+ 0.1,
202
+ );
157
203
 
158
- // ********** Ensuring Consistency of data pulled from L1 **********
159
-
160
- /**
161
- * There are a number of calls in this sync operation to L1 for retrieving
162
- * events and transaction data. There are a couple of things we need to bear in mind
163
- * to ensure that data is read exactly once.
164
- *
165
- * The first is the problem of eventually consistent ETH service providers like Infura.
166
- * Each L1 read operation will query data from the last L1 block that it saw emit its kind of data.
167
- * (so pending L1 to L2 messages will read from the last L1 block that emitted a message and so on)
168
- * This will mean the archiver will lag behind L1 and will only advance when there's L2-relevant activity on the chain.
169
- *
170
- * The second is that in between the various calls to L1, the block number can move meaning some
171
- * of the following calls will return data for blocks that were not present during earlier calls.
172
- * To combat this for the time being we simply ensure that all data retrieval methods only retrieve
173
- * data up to the currentBlockNumber captured at the top of this function. We might want to improve on this
174
- * in future but for the time being it should give us the guarantees that we need
175
- */
176
-
177
- // ********** Events that are processed per L1 block **********
178
- await this.handleL1ToL2Messages(messagesSynchedTo, currentL1BlockNumber);
179
-
180
- // ********** Events that are processed per checkpoint **********
181
204
  if (currentL1BlockNumber > blocksSynchedTo) {
182
205
  // First we retrieve new checkpoints and L2 blocks and store them in the DB. This will also update the
183
206
  // pending chain validation status, proven checkpoint number, and synched L1 block number.
184
207
  const rollupStatus = await this.handleCheckpoints(blocksSynchedTo, currentL1BlockNumber, initialSyncComplete);
208
+ blocksAdded.push(...rollupStatus.blocksAdded);
185
209
 
186
210
  // Then we try pruning uncheckpointed blocks if a new slot was mined without checkpoints
187
211
  await this.pruneUncheckpointedBlocks(currentL1Timestamp);
@@ -197,24 +221,19 @@ export class ArchiverL1Synchronizer implements Traceable {
197
221
  currentL1Timestamp,
198
222
  );
199
223
 
200
- // If the last checkpoint we processed had an invalid attestation, we manually advance the L1 syncpoint
201
- // past it, since otherwise we'll keep downloading it and reprocessing it on every iteration until
202
- // we get a valid checkpoint to advance the syncpoint.
203
- if (!rollupStatus.validationResult?.valid && rollupStatus.lastL1BlockWithCheckpoint !== undefined) {
204
- await this.store.setCheckpointSynchedL1BlockNumber(rollupStatus.lastL1BlockWithCheckpoint);
205
- }
206
-
207
224
  // And lastly we check if we are missing any checkpoints behind us due to a possible L1 reorg.
208
225
  // We only do this if rollup cant prune on the next submission. Otherwise we will end up
209
- // re-syncing the checkpoints we have just unwound above. We also dont do this if the last checkpoint is invalid,
210
- // since the archiver will rightfully refuse to sync up to it.
211
- if (!rollupCanPrune && rollupStatus.validationResult?.valid) {
226
+ // re-syncing the checkpoints we have just unwound above.
227
+ if (!rollupCanPrune) {
212
228
  await this.checkForNewCheckpointsBeforeL1SyncPoint(rollupStatus, blocksSynchedTo, currentL1BlockNumber);
213
229
  }
214
230
 
215
231
  this.instrumentation.updateL1BlockHeight(currentL1BlockNumber);
216
232
  }
217
233
 
234
+ // Update the finalized L2 checkpoint based on L1 finality.
235
+ await this.updateFinalizedCheckpoint(finalizedL1Block);
236
+
218
237
  // After syncing has completed, update the current l1 block number and timestamp,
219
238
  // otherwise we risk announcing to the world that we've synced to a given point,
220
239
  // but the corresponding blocks have not been processed (see #12631).
@@ -228,13 +247,42 @@ export class ArchiverL1Synchronizer implements Traceable {
228
247
  l1TimestampAtStart: currentL1Timestamp,
229
248
  l1BlockNumberAtEnd,
230
249
  });
250
+
251
+ return blocksAdded;
252
+ }
253
+
254
+ /** Updates the finalized checkpoint using the pre-fetched finalized L1 block from the current sync iteration. */
255
+ private async updateFinalizedCheckpoint(finalizedL1Block: L1BlockId | undefined): Promise<void> {
256
+ try {
257
+ if (!finalizedL1Block) {
258
+ this.log.trace(`Skipping finalized checkpoint update: L1 has no finalized block yet.`);
259
+ return;
260
+ }
261
+ const finalizedL1BlockNumber = finalizedL1Block.l1BlockNumber;
262
+ const finalizedCheckpointNumber = await this.rollup.getProvenCheckpointNumber({
263
+ blockNumber: finalizedL1BlockNumber,
264
+ });
265
+ const localFinalizedCheckpointNumber = await this.stores.blocks.getFinalizedCheckpointNumber();
266
+ if (localFinalizedCheckpointNumber !== finalizedCheckpointNumber) {
267
+ await this.updater.setFinalizedCheckpointNumber(finalizedCheckpointNumber);
268
+ this.log.info(`Updated finalized chain to checkpoint ${finalizedCheckpointNumber}`, {
269
+ finalizedCheckpointNumber,
270
+ finalizedL1BlockNumber,
271
+ });
272
+ }
273
+ } catch (err: any) {
274
+ // The rollup contract may not exist at the finalized L1 block right after deployment.
275
+ if (!err?.message?.includes('returned no data')) {
276
+ this.log.warn(`Failed to update finalized checkpoint: ${err}`);
277
+ }
278
+ }
231
279
  }
232
280
 
233
281
  /** Prune all proposed local blocks that should have been checkpointed by now. */
234
- private async pruneUncheckpointedBlocks(currentL1Timestamp: bigint) {
282
+ private async pruneUncheckpointedBlocks(currentL1Timestamp: bigint): Promise<void> {
235
283
  const [lastCheckpointedBlockNumber, lastProposedBlockNumber] = await Promise.all([
236
- this.store.getCheckpointedL2BlockNumber(),
237
- this.store.getLatestBlockNumber(),
284
+ this.stores.blocks.getCheckpointedL2BlockNumber(),
285
+ this.stores.blocks.getLatestL2BlockNumber(),
238
286
  ]);
239
287
 
240
288
  // If there are no uncheckpointed blocks, we got nothing to do
@@ -243,29 +291,35 @@ export class ArchiverL1Synchronizer implements Traceable {
243
291
  return;
244
292
  }
245
293
 
246
- // What's the slot of the first uncheckpointed block?
247
- const firstUncheckpointedBlockNumber = BlockNumber(lastCheckpointedBlockNumber + 1);
248
- const [firstUncheckpointedBlockHeader] = await this.store.getBlockHeaders(firstUncheckpointedBlockNumber, 1);
249
- const firstUncheckpointedBlockSlot = firstUncheckpointedBlockHeader?.getSlot();
250
-
251
294
  // What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
252
295
  const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
296
+ const firstUncheckpointedBlockNumber = BlockNumber(lastCheckpointedBlockNumber + 1);
253
297
 
254
- // Prune provisional blocks from slots that have ended without being checkpointed
255
- if (firstUncheckpointedBlockSlot !== undefined && firstUncheckpointedBlockSlot < slotAtNextL1Block) {
256
- this.log.warn(
257
- `Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`,
258
- { firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(), slotAtNextL1Block },
259
- );
260
- const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
298
+ // What's the slot of the first uncheckpointed block?
299
+ const firstUncheckpointedBlockData = await this.stores.blocks.getBlockData({
300
+ number: firstUncheckpointedBlockNumber,
301
+ });
302
+ const firstUncheckpointedBlockSlot = firstUncheckpointedBlockData?.header.getSlot();
261
303
 
262
- if (prunedBlocks.length > 0) {
263
- this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
264
- type: L2BlockSourceEvents.L2PruneUncheckpointed,
265
- slotNumber: firstUncheckpointedBlockSlot,
266
- blocks: prunedBlocks,
267
- });
268
- }
304
+ if (firstUncheckpointedBlockSlot === undefined || firstUncheckpointedBlockSlot >= slotAtNextL1Block) {
305
+ return;
306
+ }
307
+
308
+ // Prune provisional blocks from slots that have ended without being checkpointed.
309
+ // This also clears any proposed checkpoint whose blocks are being pruned.
310
+ this.log.warn(
311
+ `Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`,
312
+ { firstUncheckpointedBlockHeader: firstUncheckpointedBlockData?.header.toInspect(), slotAtNextL1Block },
313
+ );
314
+
315
+ const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
316
+ if (prunedBlocks.length > 0) {
317
+ this.instrumentation.recordPrune('uncheckpointed');
318
+ this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
319
+ type: L2BlockSourceEvents.L2PruneUncheckpointed,
320
+ slotNumber: firstUncheckpointedBlockSlot,
321
+ blocks: prunedBlocks,
322
+ });
269
323
  }
270
324
  }
271
325
 
@@ -291,7 +345,7 @@ export class ArchiverL1Synchronizer implements Traceable {
291
345
  currentL1Timestamp: bigint,
292
346
  ): Promise<{ rollupCanPrune: boolean }> {
293
347
  const rollupCanPrune = await this.canPrune(currentL1BlockNumber, currentL1Timestamp);
294
- const localPendingCheckpointNumber = await this.store.getSynchedCheckpointNumber();
348
+ const localPendingCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
295
349
  const canPrune = localPendingCheckpointNumber > provenCheckpointNumber && rollupCanPrune;
296
350
 
297
351
  if (canPrune) {
@@ -308,17 +362,20 @@ export class ArchiverL1Synchronizer implements Traceable {
308
362
 
309
363
  const checkpointsToUnwind = localPendingCheckpointNumber - provenCheckpointNumber;
310
364
 
311
- const checkpointPromises = Array.from({ length: checkpointsToUnwind })
312
- .fill(0)
313
- .map((_, i) => this.store.getCheckpointData(CheckpointNumber(i + pruneFrom)));
314
- const checkpoints = await Promise.all(checkpointPromises);
315
-
316
- const blockPromises = await Promise.all(
317
- checkpoints
318
- .filter(isDefined)
319
- .map(cp => this.store.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber))),
320
- );
321
- const newBlocks = blockPromises.filter(isDefined).flat();
365
+ // Fetch checkpoints and blocks in bounded batches to avoid unbounded concurrent
366
+ // promises when the gap between local pending and proven checkpoint numbers is large.
367
+ const BATCH_SIZE = 10;
368
+ const indices = Array.from({ length: checkpointsToUnwind }, (_, i) => CheckpointNumber(i + pruneFrom));
369
+ const checkpoints = (
370
+ await asyncPool(BATCH_SIZE, indices, idx => this.stores.blocks.getCheckpointData(idx))
371
+ ).filter(isDefined);
372
+ const newBlocks = (
373
+ await asyncPool(BATCH_SIZE, checkpoints, cp =>
374
+ this.stores.blocks.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber)),
375
+ )
376
+ )
377
+ .filter(isDefined)
378
+ .flat();
322
379
 
323
380
  // Emit an event for listening services to react to the chain prune
324
381
  this.events.emit(L2BlockSourceEvents.L2PruneUnproven, {
@@ -334,12 +391,12 @@ export class ArchiverL1Synchronizer implements Traceable {
334
391
  this.log.warn(
335
392
  `Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` +
336
393
  `due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
337
- `Updated latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
394
+ `Updated latest checkpoint is ${await this.stores.blocks.getLatestCheckpointNumber()}.`,
338
395
  );
339
396
  this.instrumentation.processPrune(timer.ms());
340
397
  // TODO(palla/reorg): Do we need to set the block synched L1 block number here?
341
398
  // Seems like the next iteration should handle this.
342
- // await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
399
+ // await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
343
400
  }
344
401
 
345
402
  return { rollupCanPrune };
@@ -356,67 +413,102 @@ export class ArchiverL1Synchronizer implements Traceable {
356
413
  }
357
414
 
358
415
  @trackSpan('Archiver.handleL1ToL2Messages')
359
- private async handleL1ToL2Messages(messagesSyncPoint: L1BlockId, currentL1BlockNumber: bigint): Promise<void> {
360
- this.log.trace(`Handling L1 to L2 messages from ${messagesSyncPoint.l1BlockNumber} to ${currentL1BlockNumber}.`);
361
- if (currentL1BlockNumber <= messagesSyncPoint.l1BlockNumber) {
362
- return;
416
+ private async handleL1ToL2Messages(
417
+ currentL1Block: L1BlockId,
418
+ finalizedL1Block: L1BlockId | undefined,
419
+ ): Promise<boolean> {
420
+ // Load the syncpoint, which may have been updated in a previous iteration
421
+ const {
422
+ messagesSynchedTo = {
423
+ l1BlockNumber: this.l1Constants.l1StartBlock,
424
+ l1BlockHash: this.l1Constants.l1StartBlockHash,
425
+ },
426
+ } = await getArchiverSynchPoint(this.stores);
427
+
428
+ // Nothing to do if L1 block number has not moved forward
429
+ const currentL1BlockNumber = currentL1Block.l1BlockNumber;
430
+ if (currentL1BlockNumber <= messagesSynchedTo.l1BlockNumber) {
431
+ return true;
363
432
  }
364
433
 
365
- // Load remote and local inbox states.
366
- const localMessagesInserted = await this.store.getTotalL1ToL2MessageCount();
367
- const localLastMessage = await this.store.getLastL1ToL2Message();
434
+ // Compare local message store state with the remote. If they match, we just advance the match pointer.
368
435
  const remoteMessagesState = await this.inbox.getState({ blockNumber: currentL1BlockNumber });
436
+ const localLastMessage = await this.stores.messages.getLastMessage();
437
+ if (await this.localStateMatches(localLastMessage, remoteMessagesState)) {
438
+ this.log.trace(`Local L1 to L2 messages are already in sync with remote at L1 block ${currentL1BlockNumber}`);
439
+ await this.stores.messages.setMessageSyncState(
440
+ currentL1Block,
441
+ remoteMessagesState.treeInProgress,
442
+ finalizedL1Block,
443
+ );
444
+ return true;
445
+ }
369
446
 
370
- this.log.trace(`Retrieved remote inbox state at L1 block ${currentL1BlockNumber}.`, {
371
- localMessagesInserted,
372
- localLastMessage,
373
- remoteMessagesState,
374
- });
447
+ // If not, then we are out of sync. Most likely there are new messages on the inbox, so we try retrieving them.
448
+ // However, it could also be the case that there was an L1 reorg and our syncpoint is no longer valid.
449
+ // If that's the case, we'd get an exception out of the message store since the rolling hash of the first message
450
+ // we try to insert would not match the one in the db, in which case we rollback to the last common message with L1.
451
+ try {
452
+ await this.retrieveAndStoreMessages(messagesSynchedTo.l1BlockNumber, currentL1BlockNumber);
453
+ } catch (error) {
454
+ if (isErrorClass(error, MessageStoreError)) {
455
+ this.log.warn(
456
+ `Failed to store L1 to L2 messages retrieved from L1: ${error.message}. Rolling back syncpoint to retry.`,
457
+ { inboxMessage: error.inboxMessage },
458
+ );
459
+ await this.rollbackL1ToL2Messages(remoteMessagesState);
460
+ return false;
461
+ }
462
+ throw error;
463
+ }
375
464
 
376
- // Compare message count and rolling hash. If they match, no need to retrieve anything.
377
- if (
378
- remoteMessagesState.totalMessagesInserted === localMessagesInserted &&
379
- remoteMessagesState.messagesRollingHash.equals(localLastMessage?.rollingHash ?? Buffer32.ZERO)
380
- ) {
381
- this.log.trace(
382
- `No L1 to L2 messages to query between L1 blocks ${messagesSyncPoint.l1BlockNumber} and ${currentL1BlockNumber}.`,
465
+ // Note that, if there are no new messages to insert, but there was an L1 reorg that pruned out last messages,
466
+ // we'd notice by comparing our local state with the remote one again, and seeing they don't match even after
467
+ // our sync attempt. In this case, we also rollback our syncpoint, and trigger a retry.
468
+ const localLastMessageAfterSync = await this.stores.messages.getLastMessage();
469
+ if (!(await this.localStateMatches(localLastMessageAfterSync, remoteMessagesState))) {
470
+ this.log.warn(
471
+ `Local L1 to L2 messages state does not match remote after sync attempt. Rolling back syncpoint to retry.`,
472
+ { localLastMessageAfterSync, remoteMessagesState },
383
473
  );
384
- return;
474
+ await this.rollbackL1ToL2Messages(remoteMessagesState);
475
+ return false;
385
476
  }
386
477
 
387
- // Check if our syncpoint is still valid. If not, there was an L1 reorg and we need to re-retrieve messages.
388
- // Note that we need to fetch it from logs and not from inbox state at the syncpoint l1 block number, since it
389
- // could be older than 128 blocks and non-archive nodes cannot resolve it.
390
- if (localLastMessage) {
391
- const remoteLastMessage = await this.retrieveL1ToL2Message(localLastMessage.leaf);
392
- this.log.trace(`Retrieved remote message for local last`, { remoteLastMessage, localLastMessage });
393
- if (!remoteLastMessage || !remoteLastMessage.rollingHash.equals(localLastMessage.rollingHash)) {
394
- this.log.warn(`Rolling back L1 to L2 messages due to hash mismatch or msg not found.`, {
395
- remoteLastMessage,
396
- messagesSyncPoint,
397
- localLastMessage,
398
- });
478
+ // Advance the syncpoint after a successful sync
479
+ await this.stores.messages.setMessageSyncState(
480
+ currentL1Block,
481
+ remoteMessagesState.treeInProgress,
482
+ finalizedL1Block,
483
+ );
484
+ return true;
485
+ }
399
486
 
400
- messagesSyncPoint = await this.rollbackL1ToL2Messages(localLastMessage, messagesSyncPoint);
401
- this.log.debug(`Rolled back L1 to L2 messages to L1 block ${messagesSyncPoint.l1BlockNumber}.`, {
402
- messagesSyncPoint,
403
- });
404
- }
405
- }
487
+ /** Checks if the local rolling hash and message count matches the remote state */
488
+ private async localStateMatches(localLastMessage: InboxMessage | undefined, remoteState: InboxContractState) {
489
+ const localMessageCount = await this.stores.messages.getTotalL1ToL2MessageCount();
490
+ this.log.trace(`Comparing local and remote inbox state`, { localMessageCount, localLastMessage, remoteState });
491
+
492
+ return (
493
+ remoteState.totalMessagesInserted === localMessageCount &&
494
+ remoteState.messagesRollingHash.equals(localLastMessage?.rollingHash ?? Buffer16.ZERO)
495
+ );
496
+ }
406
497
 
407
- // Retrieve and save messages in batches. Each batch is estimated to acommodate up to L2 'blockBatchSize' blocks,
498
+ /** Retrieves L1 to L2 messages from L1 in batches and stores them. */
499
+ private async retrieveAndStoreMessages(fromL1Block: bigint, toL1Block: bigint): Promise<void> {
408
500
  let searchStartBlock: bigint = 0n;
409
- let searchEndBlock: bigint = messagesSyncPoint.l1BlockNumber;
501
+ let searchEndBlock: bigint = fromL1Block;
410
502
 
411
503
  let lastMessage: InboxMessage | undefined;
412
504
  let messageCount = 0;
413
505
 
414
506
  do {
415
- [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
507
+ [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, toL1Block);
416
508
  this.log.trace(`Retrieving L1 to L2 messages in L1 blocks ${searchStartBlock}-${searchEndBlock}`);
417
509
  const messages = await retrieveL1ToL2Messages(this.inbox, searchStartBlock, searchEndBlock);
418
510
  const timer = new Timer();
419
- await this.store.addL1ToL2Messages(messages);
511
+ await this.stores.messages.addL1ToL2Messages(messages);
420
512
  const perMsg = timer.ms() / messages.length;
421
513
  this.instrumentation.processNewMessages(messages.length, perMsg);
422
514
  for (const msg of messages) {
@@ -424,81 +516,108 @@ export class ArchiverL1Synchronizer implements Traceable {
424
516
  lastMessage = msg;
425
517
  messageCount++;
426
518
  }
427
- } while (searchEndBlock < currentL1BlockNumber);
519
+ } while (searchEndBlock < toL1Block);
428
520
 
429
- // Log stats for messages retrieved (if any).
430
521
  if (messageCount > 0) {
431
522
  this.log.info(
432
523
  `Retrieved ${messageCount} new L1 to L2 messages up to message with index ${lastMessage?.index} for checkpoint ${lastMessage?.checkpointNumber}`,
433
524
  { lastMessage, messageCount },
434
525
  );
435
526
  }
436
-
437
- // Warn if the resulting rolling hash does not match the remote state we had retrieved.
438
- if (lastMessage && !lastMessage.rollingHash.equals(remoteMessagesState.messagesRollingHash)) {
439
- this.log.warn(`Last message retrieved rolling hash does not match remote state.`, {
440
- lastMessage,
441
- remoteMessagesState,
442
- });
443
- }
444
527
  }
445
528
 
446
- private async retrieveL1ToL2Message(leaf: Fr): Promise<InboxMessage | undefined> {
447
- const currentL1BlockNumber = await this.publicClient.getBlockNumber();
448
- let searchStartBlock: bigint = 0n;
449
- let searchEndBlock: bigint = this.l1Constants.l1StartBlock - 1n;
529
+ /**
530
+ * Rolls back local L1 to L2 messages to the last common message with L1, and updates the syncpoint to the L1 block of that message.
531
+ * If no common message is found, rolls back all messages and sets the syncpoint to the start block.
532
+ */
533
+ private async rollbackL1ToL2Messages(remoteMessagesState: InboxContractState): Promise<L1BlockId> {
534
+ const { treeInProgress: remoteTreeInProgress, messagesRollingHash: remoteRollingHash } = remoteMessagesState;
450
535
 
451
- do {
452
- [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
453
-
454
- const message = await retrieveL1ToL2Message(this.inbox, leaf, searchStartBlock, searchEndBlock);
536
+ const messagesFinalizedL1Block = await this.stores.messages.getMessagesFinalizedL1Block();
537
+ const finalizedL1BlockNumber = messagesFinalizedL1Block?.l1BlockNumber;
455
538
 
456
- if (message) {
457
- return message;
539
+ // Slowly go back through our messages until we find the last common message. We could query the logs in
540
+ // batch as an optimization, but the depth of the reorg should not be deep, and this is a very rare case,
541
+ // so it's fine to query one log at a time.
542
+ let commonMsg: undefined | InboxMessage;
543
+ let messagesToDelete = 0;
544
+ this.log.verbose(`Searching most recent common L1 to L2 message`);
545
+ for await (const localMsg of this.stores.messages.iterateL1ToL2Messages({ reverse: true })) {
546
+ const logCtx = { remoteMsg: undefined as InboxMessage | undefined, localMsg, remoteMessagesState };
547
+
548
+ // First check if the local message rolling hash matches the current rolling hash of the inbox contract,
549
+ // which means we just need to rollback some local messages and we should be back in sync. This means there
550
+ // was an L1 reorg that removed some of the messages we had, but no new messages were added compared.
551
+ if (localMsg.rollingHash.equals(remoteRollingHash)) {
552
+ this.log.info(
553
+ `Found common L1 to L2 message at index ${localMsg.index} on L1 block ${localMsg.l1BlockNumber} matching current remote state`,
554
+ logCtx,
555
+ );
556
+ commonMsg = localMsg;
557
+ break;
458
558
  }
459
- } while (searchEndBlock < currentL1BlockNumber);
460
559
 
461
- return undefined;
462
- }
560
+ // Messages at or below the finalized L1 block cannot have been reorged — accept as common without querying L1.
561
+ if (finalizedL1BlockNumber !== undefined && localMsg.l1BlockNumber <= finalizedL1BlockNumber) {
562
+ this.log.info(`Found common L1 to L2 message at finalized L1 block ${localMsg.l1BlockNumber}`, logCtx);
563
+ commonMsg = localMsg;
564
+ break;
565
+ }
463
566
 
464
- private async rollbackL1ToL2Messages(
465
- localLastMessage: InboxMessage,
466
- messagesSyncPoint: L1BlockId,
467
- ): Promise<L1BlockId> {
468
- // Slowly go back through our messages until we find the last common message.
469
- // We could query the logs in batch as an optimization, but the depth of the reorg should not be deep, and this
470
- // is a very rare case, so it's fine to query one log at a time.
471
- let commonMsg: undefined | InboxMessage;
472
- this.log.verbose(`Searching most recent common L1 to L2 message at or before index ${localLastMessage.index}`);
473
- for await (const msg of this.store.iterateL1ToL2Messages({ reverse: true, end: localLastMessage.index })) {
474
- const remoteMsg = await this.retrieveL1ToL2Message(msg.leaf);
475
- const logCtx = { remoteMsg, localMsg: msg };
476
- if (remoteMsg && remoteMsg.rollingHash.equals(msg.rollingHash)) {
477
- this.log.verbose(
478
- `Found most recent common L1 to L2 message at index ${msg.index} on L1 block ${msg.l1BlockNumber}`,
567
+ // If there's no match with the current remote state, check if the message exists on the inbox contract at all
568
+ // by looking at the inbox events. If the L1 reorg *added* new messages in addition to deleting existing ones,
569
+ // then the current remote state's rolling hash will not match anything we have locally, so we need to check existence
570
+ // of individual messages via logs. Note we use logs and not historical queries so we don't have to depend on
571
+ // an archival rpc node, since the message could be from a long time ago if we're catching up with syncing.
572
+ const remoteMsg = await retrieveL1ToL2Message(this.inbox, localMsg);
573
+ logCtx.remoteMsg = remoteMsg;
574
+ if (remoteMsg && remoteMsg.rollingHash.equals(localMsg.rollingHash)) {
575
+ this.log.info(
576
+ `Found most recent common L1 to L2 message at index ${localMsg.index} on L1 block ${localMsg.l1BlockNumber}`,
479
577
  logCtx,
480
578
  );
481
579
  commonMsg = remoteMsg;
482
580
  break;
483
581
  } else if (remoteMsg) {
484
- this.log.debug(`Local L1 to L2 message with index ${msg.index} has different rolling hash`, logCtx);
582
+ this.log.debug(`Local L1 to L2 message with index ${localMsg.index} has different rolling hash`, logCtx);
583
+ messagesToDelete++;
485
584
  } else {
486
- this.log.debug(`Local L1 to L2 message with index ${msg.index} not found on L1`, logCtx);
585
+ this.log.debug(`Local L1 to L2 message with index ${localMsg.index} not found on L1`, logCtx);
586
+ messagesToDelete++;
487
587
  }
488
588
  }
489
589
 
490
- // Delete everything after the common message we found.
491
- const lastGoodIndex = commonMsg?.index;
492
- this.log.warn(`Deleting all local L1 to L2 messages after index ${lastGoodIndex ?? 'undefined'}`);
493
- await this.store.removeL1ToL2Messages(lastGoodIndex !== undefined ? lastGoodIndex + 1n : 0n);
590
+ // Delete everything after the common message we found, if anything needs to be deleted.
591
+ // Do not exit early if there are no messages to delete, we still want to update the syncpoint.
592
+ if (messagesToDelete > 0) {
593
+ const lastGoodIndex = commonMsg?.index;
594
+ this.log.warn(`Rolling back all local L1 to L2 messages after index ${lastGoodIndex ?? 'initial'}`);
595
+ await this.stores.messages.removeL1ToL2Messages(lastGoodIndex !== undefined ? lastGoodIndex + 1n : 0n);
596
+ }
494
597
 
495
598
  // Update the syncpoint so the loop below reprocesses the changed messages. We go to the block before
496
599
  // the last common one, so we force reprocessing it, in case new messages were added on that same L1 block
497
- // after the last common message.
498
- const syncPointL1BlockNumber = commonMsg ? commonMsg.l1BlockNumber - 1n : this.l1Constants.l1StartBlock;
499
- const syncPointL1BlockHash = await this.getL1BlockHash(syncPointL1BlockNumber);
500
- messagesSyncPoint = { l1BlockNumber: syncPointL1BlockNumber, l1BlockHash: syncPointL1BlockHash };
501
- await this.store.setMessageSynchedL1Block(messagesSyncPoint);
600
+ // after the last common message. Cap at the finalized L1 block: messages at or below finalized cannot
601
+ // have been reorged, so there is no need to walk back any further than that.
602
+ const syncPointL1BlockNumber = maxBigint(
603
+ ...compactArray([
604
+ commonMsg ? commonMsg.l1BlockNumber - 1n : undefined,
605
+ finalizedL1BlockNumber,
606
+ this.l1Constants.l1StartBlock,
607
+ ]),
608
+ );
609
+
610
+ const syncPointL1BlockHash =
611
+ syncPointL1BlockNumber === finalizedL1BlockNumber
612
+ ? messagesFinalizedL1Block!.l1BlockHash
613
+ : await this.getL1BlockHash(syncPointL1BlockNumber);
614
+
615
+ const messagesSyncPoint = { l1BlockNumber: syncPointL1BlockNumber, l1BlockHash: syncPointL1BlockHash };
616
+ await this.stores.messages.setMessageSyncState(messagesSyncPoint, remoteTreeInProgress);
617
+ this.log.verbose(`Updated messages syncpoint to L1 block ${messagesSyncPoint.l1BlockNumber}`, {
618
+ ...messagesSyncPoint,
619
+ remoteTreeInProgress,
620
+ });
502
621
  return messagesSyncPoint;
503
622
  }
504
623
 
@@ -516,9 +635,10 @@ export class ArchiverL1Synchronizer implements Traceable {
516
635
  currentL1BlockNumber: bigint,
517
636
  initialSyncComplete: boolean,
518
637
  ): Promise<RollupStatus> {
519
- const localPendingCheckpointNumber = await this.store.getSynchedCheckpointNumber();
638
+ const blocksAdded: L2Block[] = [];
639
+ const localPendingCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
520
640
  const initialValidationResult: ValidateCheckpointResult | undefined =
521
- await this.store.getPendingChainValidationStatus();
641
+ await this.stores.blocks.getPendingChainValidationStatus();
522
642
  const {
523
643
  provenCheckpointNumber,
524
644
  provenArchive,
@@ -534,6 +654,7 @@ export class ArchiverL1Synchronizer implements Traceable {
534
654
  pendingCheckpointNumber,
535
655
  pendingArchive: pendingArchive.toString(),
536
656
  validationResult: initialValidationResult,
657
+ blocksAdded,
537
658
  };
538
659
  this.log.trace(`Retrieved rollup status at current L1 block ${currentL1BlockNumber}.`, {
539
660
  localPendingCheckpointNumber,
@@ -548,19 +669,19 @@ export class ArchiverL1Synchronizer implements Traceable {
548
669
  // we need to set it to zero. This is an edge case because we dont have a checkpoint zero (initial checkpoint is one),
549
670
  // so localCheckpointForDestinationProvenCheckpointNumber would not be found below.
550
671
  if (provenCheckpointNumber === 0) {
551
- const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
672
+ const localProvenCheckpointNumber = await this.stores.blocks.getProvenCheckpointNumber();
552
673
  if (localProvenCheckpointNumber !== provenCheckpointNumber) {
553
- await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
674
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
554
675
  this.log.info(`Rolled back proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
555
676
  }
556
677
  }
557
678
 
558
679
  const localCheckpointForDestinationProvenCheckpointNumber =
559
- await this.store.getCheckpointData(provenCheckpointNumber);
680
+ await this.stores.blocks.getCheckpointData(provenCheckpointNumber);
560
681
 
561
682
  // Sanity check. I've hit what seems to be a state where the proven checkpoint is set to a value greater than the latest
562
683
  // synched checkpoint when requesting L2Tips from the archiver. This is the only place where the proven checkpoint is set.
563
- const synched = await this.store.getSynchedCheckpointNumber();
684
+ const synched = await this.stores.blocks.getLatestCheckpointNumber();
564
685
  if (
565
686
  localCheckpointForDestinationProvenCheckpointNumber &&
566
687
  synched < localCheckpointForDestinationProvenCheckpointNumber.checkpointNumber
@@ -580,15 +701,15 @@ export class ArchiverL1Synchronizer implements Traceable {
580
701
  localCheckpointForDestinationProvenCheckpointNumber &&
581
702
  provenArchive.equals(localCheckpointForDestinationProvenCheckpointNumber.archive.root)
582
703
  ) {
583
- const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
704
+ const localProvenCheckpointNumber = await this.stores.blocks.getProvenCheckpointNumber();
584
705
  if (localProvenCheckpointNumber !== provenCheckpointNumber) {
585
- await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
706
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
586
707
  this.log.info(`Updated proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
587
708
  const provenSlotNumber = localCheckpointForDestinationProvenCheckpointNumber.header.slotNumber;
588
709
  const provenEpochNumber: EpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants);
589
710
  const lastBlockNumberInCheckpoint =
590
711
  localCheckpointForDestinationProvenCheckpointNumber.startBlock +
591
- localCheckpointForDestinationProvenCheckpointNumber.numBlocks -
712
+ localCheckpointForDestinationProvenCheckpointNumber.blockCount -
592
713
  1;
593
714
 
594
715
  this.events.emit(L2BlockSourceEvents.L2BlockProven, {
@@ -597,7 +718,7 @@ export class ArchiverL1Synchronizer implements Traceable {
597
718
  slotNumber: provenSlotNumber,
598
719
  epochNumber: provenEpochNumber,
599
720
  });
600
- this.instrumentation.updateLastProvenBlock(lastBlockNumberInCheckpoint);
721
+ this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber);
601
722
  } else {
602
723
  this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`);
603
724
  }
@@ -608,7 +729,7 @@ export class ArchiverL1Synchronizer implements Traceable {
608
729
  // If we have 0 checkpoints locally and there are no checkpoints onchain there is nothing to do.
609
730
  const noCheckpoints = localPendingCheckpointNumber === 0 && pendingCheckpointNumber === 0;
610
731
  if (noCheckpoints) {
611
- await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
732
+ await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
612
733
  this.log.debug(
613
734
  `No checkpoints to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}, no checkpoints on chain`,
614
735
  );
@@ -620,7 +741,7 @@ export class ArchiverL1Synchronizer implements Traceable {
620
741
  // Related to the L2 reorgs of the pending chain. We are only interested in actually addressing a reorg if there
621
742
  // are any state that could be impacted by it. If we have no checkpoints, there is no impact.
622
743
  if (localPendingCheckpointNumber > 0) {
623
- const localPendingCheckpoint = await this.store.getCheckpointData(localPendingCheckpointNumber);
744
+ const localPendingCheckpoint = await this.stores.blocks.getCheckpointData(localPendingCheckpointNumber);
624
745
  if (localPendingCheckpoint === undefined) {
625
746
  throw new Error(`Missing checkpoint ${localPendingCheckpointNumber}`);
626
747
  }
@@ -635,7 +756,7 @@ export class ArchiverL1Synchronizer implements Traceable {
635
756
  // However, in the re-org scenario, our L1 node is temporarily lying to us and we end up potentially missing checkpoints.
636
757
  // We must only set this block number based on actually retrieved logs.
637
758
  // TODO(#8621): Tackle this properly when we handle L1 Re-orgs.
638
- // await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
759
+ // await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
639
760
  this.log.debug(`No checkpoints to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}`);
640
761
  return rollupStatus;
641
762
  }
@@ -655,7 +776,7 @@ export class ArchiverL1Synchronizer implements Traceable {
655
776
 
656
777
  let tipAfterUnwind = localPendingCheckpointNumber;
657
778
  while (true) {
658
- const candidateCheckpoint = await this.store.getCheckpointData(tipAfterUnwind);
779
+ const candidateCheckpoint = await this.stores.blocks.getCheckpointData(tipAfterUnwind);
659
780
  if (candidateCheckpoint === undefined) {
660
781
  break;
661
782
  }
@@ -676,11 +797,14 @@ export class ArchiverL1Synchronizer implements Traceable {
676
797
 
677
798
  const checkpointsToRemove = localPendingCheckpointNumber - tipAfterUnwind;
678
799
  await this.updater.removeCheckpointsAfter(CheckpointNumber(tipAfterUnwind));
800
+ if (checkpointsToRemove > 0) {
801
+ this.instrumentation.recordPrune('l1_mismatch');
802
+ }
679
803
 
680
804
  this.log.warn(
681
805
  `Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` +
682
806
  `due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` +
683
- `Updated L2 latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
807
+ `Updated L2 latest checkpoint is ${await this.stores.blocks.getLatestCheckpointNumber()}.`,
684
808
  );
685
809
  }
686
810
  }
@@ -690,30 +814,27 @@ export class ArchiverL1Synchronizer implements Traceable {
690
814
  let searchStartBlock: bigint = blocksSynchedTo;
691
815
  let searchEndBlock: bigint = blocksSynchedTo;
692
816
  let lastRetrievedCheckpoint: PublishedCheckpoint | undefined;
693
- let lastL1BlockWithCheckpoint: bigint | undefined = undefined;
817
+ let lastSeenCheckpoint: { checkpointNumber: CheckpointNumber; l1: L1PublishedData } | undefined;
694
818
 
695
819
  do {
696
820
  [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
697
821
 
698
822
  this.log.trace(`Retrieving checkpoints from L1 block ${searchStartBlock} to ${searchEndBlock}`);
699
823
 
700
- // TODO(md): Retrieve from blob client then from consensus client, then from peers
701
- const retrievedCheckpoints = await execInSpan(this.tracer, 'Archiver.retrieveCheckpointsFromRollup', () =>
702
- retrieveCheckpointsFromRollup(
824
+ // First fetch calldata only, no blobs yet, since we may be able to just get that data out of the proposed chain
825
+ const calldataCheckpoints = await execInSpan(this.tracer, 'Archiver.retrieveCheckpointCalldataFromRollup', () =>
826
+ retrieveCheckpointCalldataFromRollup(
703
827
  this.rollup,
704
828
  this.publicClient,
705
829
  this.debugClient,
706
- this.blobClient,
707
830
  searchStartBlock, // TODO(palla/reorg): If the L2 reorg was due to an L1 reorg, we need to start search earlier
708
831
  searchEndBlock,
709
- this.l1Addresses,
710
832
  this.instrumentation,
711
833
  this.log,
712
- !initialSyncComplete, // isHistoricalSync
713
834
  ),
714
835
  );
715
836
 
716
- if (retrievedCheckpoints.length === 0) {
837
+ if (calldataCheckpoints.length === 0) {
717
838
  // We are not calling `setBlockSynchedL1BlockNumber` because it may cause sync issues if based off infura.
718
839
  // See further details in earlier comments.
719
840
  this.log.trace(`Retrieved no new checkpoints from L1 block ${searchStartBlock} to ${searchEndBlock}`);
@@ -721,40 +842,74 @@ export class ArchiverL1Synchronizer implements Traceable {
721
842
  }
722
843
 
723
844
  this.log.debug(
724
- `Retrieved ${retrievedCheckpoints.length} new checkpoints between L1 blocks ${searchStartBlock} and ${searchEndBlock}`,
845
+ `Retrieved ${calldataCheckpoints.length} new checkpoint calldata between L1 blocks ${searchStartBlock} and ${searchEndBlock}`,
725
846
  {
726
- lastProcessedCheckpoint: retrievedCheckpoints[retrievedCheckpoints.length - 1].l1,
847
+ lastProcessedCheckpoint: calldataCheckpoints[calldataCheckpoints.length - 1].l1,
727
848
  searchStartBlock,
728
849
  searchEndBlock,
729
850
  },
730
851
  );
731
852
 
732
- const publishedCheckpoints = await Promise.all(retrievedCheckpoints.map(b => retrievedToPublishedCheckpoint(b)));
733
- const validCheckpoints: PublishedCheckpoint[] = [];
734
-
735
- for (const published of publishedCheckpoints) {
853
+ // Check if the last checkpoint matches a local pending entry (so we can skip blob fetch).
854
+ // We only check the last one; if it matches, the blob fetch is skipped for that entry.
855
+ // TODO(palla/pipelining): We may have more than a single checkpoint to promote
856
+ const lastCalldataCheckpoint = calldataCheckpoints[calldataCheckpoints.length - 1];
857
+ const promoteResult = await this.tryBuildPublishedCheckpointFromProposed(lastCalldataCheckpoint);
858
+ const checkpointToPromote = promoteResult && !('diverged' in promoteResult) ? promoteResult : undefined;
859
+ const evictProposedFrom =
860
+ promoteResult && 'diverged' in promoteResult ? promoteResult.fromCheckpointNumber : undefined;
861
+
862
+ // Validate attestations from CALLDATA before fetching any blobs. A checkpoint with invalid
863
+ // attestations (or one descending from a rejected ancestor) is rejected here without fetching its
864
+ // blobs, so a malformed blob does not throw during decode before the rejection path runs and
865
+ // stall sync. The signed consensus payload (header, archive root, fee asset price
866
+ // modifier) is fully available from calldata.
867
+ const checkpointsToIngest: RetrievedCheckpointFromCalldata[] = [];
868
+
869
+ for (const calldataCheckpoint of calldataCheckpoints) {
870
+ // Check the attestations uploaded by the publisher to L1 are correct.
871
+ // Rollup contract does not validate attestations to save on gas, so this
872
+ // falls on the nodes to verify offchain and skip those checkpoints.
736
873
  const validationResult = this.config.skipValidateCheckpointAttestations
737
874
  ? { valid: true as const }
738
- : await validateCheckpointAttestations(published, this.epochCache, this.l1Constants, this.log);
875
+ : await validateCheckpointAttestationsFromCalldata(
876
+ calldataCheckpoint,
877
+ this.epochCache,
878
+ this.l1Constants,
879
+ this.getSignatureContext(),
880
+ this.log,
881
+ );
882
+
883
+ // Also skip the checkpoint if it builds on a previously-rejected ancestor. Without
884
+ // this, addCheckpoints would throw InitialCheckpointNumberNotSequentialError when the
885
+ // ancestor was skipped earlier (e.g. due to invalid attestations), the catch handler
886
+ // would roll back the L1 sync point, and the next iteration would re-fetch and re-throw.
887
+ const rejectedAncestor = await this.stores.blocks.getRejectedCheckpointByArchiveRoot(
888
+ calldataCheckpoint.header.lastArchiveRoot,
889
+ );
739
890
 
740
- // Only update the validation result if it has changed, so we can keep track of the first invalid checkpoint
891
+ // Update the validation result if it has changed, so we can keep track of the first invalid checkpoint
741
892
  // in case there is a sequence of more than one invalid checkpoint, as we need to invalidate the first one.
742
893
  // There is an exception though: if a checkpoint is invalidated and replaced with another invalid checkpoint,
743
894
  // we need to update the validation result, since we need to be able to invalidate the new one.
744
895
  // See test 'chain progresses if an invalid checkpoint is invalidated with an invalid one' for more info.
745
- if (
746
- rollupStatus.validationResult?.valid !== validationResult.valid ||
747
- (!rollupStatus.validationResult.valid &&
748
- !validationResult.valid &&
749
- rollupStatus.validationResult.checkpoint.checkpointNumber === validationResult.checkpoint.checkpointNumber)
750
- ) {
896
+ // Do not update the validation result if there is a rejected ancestor, since in that case we want to keep the
897
+ // original invalidation, as the new checkpoint is extending from a previous invalid one.
898
+ const validStatusChanged = rollupStatus.validationResult?.valid !== validationResult.valid;
899
+ const invalidStatusWithSameCheckpointNumber =
900
+ !validationResult.valid &&
901
+ rollupStatus.validationResult &&
902
+ !rollupStatus.validationResult.valid &&
903
+ rollupStatus.validationResult.checkpoint.checkpointNumber === validationResult.checkpoint.checkpointNumber;
904
+
905
+ if (!rejectedAncestor && (validStatusChanged || invalidStatusWithSameCheckpointNumber)) {
751
906
  rollupStatus.validationResult = validationResult;
752
907
  }
753
908
 
754
909
  if (!validationResult.valid) {
755
- this.log.warn(`Skipping checkpoint ${published.checkpoint.number} due to invalid attestations`, {
756
- checkpointHash: published.checkpoint.hash(),
757
- l1BlockNumber: published.l1.blockNumber,
910
+ this.log.warn(`Skipping checkpoint ${calldataCheckpoint.checkpointNumber} due to invalid attestations`, {
911
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
912
+ l1BlockNumber: calldataCheckpoint.l1.blockNumber,
758
913
  ...pick(validationResult, 'reason'),
759
914
  });
760
915
 
@@ -764,16 +919,104 @@ export class ArchiverL1Synchronizer implements Traceable {
764
919
  validationResult,
765
920
  });
766
921
 
767
- // We keep consuming checkpoints if we find an invalid one, since we do not listen for CheckpointInvalidated events
768
- // We just pretend the invalid ones are not there and keep consuming the next checkpoints
769
- // Note that this breaks if the committee ever attests to a descendant of an invalid checkpoint
922
+ // Persist a rejected-ancestor entry so any later checkpoint that builds on this one
923
+ // is detected and skipped (rather than tripping the addCheckpoints consecutive-number
924
+ // check and causing the sync point to roll back in a loop).
925
+ await this.stores.blocks.addRejectedCheckpoint({
926
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
927
+ archiveRoot: calldataCheckpoint.archiveRoot,
928
+ parentArchiveRoot: calldataCheckpoint.header.lastArchiveRoot,
929
+ slotNumber: calldataCheckpoint.header.slotNumber,
930
+ l1: calldataCheckpoint.l1,
931
+ reason: 'invalid-attestations' as const,
932
+ });
933
+
770
934
  continue;
771
935
  }
772
936
 
937
+ if (rejectedAncestor) {
938
+ const descendantInfo: CheckpointInfo = {
939
+ archive: calldataCheckpoint.archiveRoot,
940
+ lastArchive: calldataCheckpoint.header.lastArchiveRoot,
941
+ slotNumber: calldataCheckpoint.header.slotNumber,
942
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
943
+ timestamp: calldataCheckpoint.header.timestamp,
944
+ };
945
+ this.log.warn(
946
+ `Skipping checkpoint ${calldataCheckpoint.checkpointNumber} as it is a descendant of ` +
947
+ `rejected checkpoint ${rejectedAncestor.checkpointNumber} (${rejectedAncestor.reason})`,
948
+ {
949
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
950
+ l1BlockNumber: calldataCheckpoint.l1.blockNumber,
951
+ l1BlockHash: calldataCheckpoint.l1.blockHash,
952
+ ancestorCheckpointNumber: rejectedAncestor.checkpointNumber,
953
+ ancestorArchiveRoot: rejectedAncestor.archiveRoot.toString(),
954
+ ancestorReason: rejectedAncestor.reason,
955
+ },
956
+ );
957
+
958
+ this.events.emit(L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, {
959
+ type: L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected,
960
+ checkpoint: descendantInfo,
961
+ ancestorArchiveRoot: rejectedAncestor.archiveRoot,
962
+ ancestorCheckpointNumber: rejectedAncestor.checkpointNumber,
963
+ });
964
+
965
+ // Persist this chainpoint as rejected as well, so we can construct a chain of
966
+ // skipped checkpoints starting from the first one with invalid attestations.
967
+ await this.stores.blocks.addRejectedCheckpoint({
968
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
969
+ archiveRoot: calldataCheckpoint.archiveRoot,
970
+ parentArchiveRoot: calldataCheckpoint.header.lastArchiveRoot,
971
+ slotNumber: calldataCheckpoint.header.slotNumber,
972
+ l1: calldataCheckpoint.l1,
973
+ reason: 'descends-from-invalid-attestations' as const,
974
+ });
975
+
976
+ continue;
977
+ }
978
+
979
+ checkpointsToIngest.push(calldataCheckpoint);
980
+ }
981
+
982
+ // Fetch blobs in parallel only for the surviving (attestation-valid, non-descendant) checkpoints,
983
+ // then build the full published checkpoints. The last calldata checkpoint may be promotable from a
984
+ // local proposed block (checkpointToPromote), in which case it carries no blob to fetch. A missing or
985
+ // undecodable blob throws and propagates, rolling back the L1 sync point so the fetch is retried.
986
+ const toFetchBlobs = checkpointToPromote
987
+ ? checkpointsToIngest.filter(c => c.checkpointNumber !== checkpointToPromote.checkpoint.number)
988
+ : checkpointsToIngest;
989
+ const blobFetched = await asyncPool(10, toFetchBlobs, async checkpoint =>
990
+ retrievedToPublishedCheckpoint({
991
+ ...checkpoint,
992
+ checkpointBlobData: await getCheckpointBlobDataFromBlobs(
993
+ this.blobClient,
994
+ checkpoint.l1.blockHash,
995
+ checkpoint.blobHashes,
996
+ checkpoint.checkpointNumber,
997
+ this.log,
998
+ !initialSyncComplete,
999
+ checkpoint.parentBeaconBlockRoot,
1000
+ checkpoint.l1.timestamp,
1001
+ ),
1002
+ }),
1003
+ );
1004
+
1005
+ // Index the built checkpoints by number so we can ingest them in calldata order, slotting in the
1006
+ // promoted checkpoint (built from a local proposed block rather than blobs).
1007
+ const publishedByNumber = new Map(blobFetched.map(published => [published.checkpoint.number, published]));
1008
+ if (checkpointToPromote) {
1009
+ publishedByNumber.set(checkpointToPromote.checkpoint.number, checkpointToPromote);
1010
+ }
1011
+
1012
+ const validCheckpoints: PublishedCheckpoint[] = [];
1013
+ for (const calldataCheckpoint of checkpointsToIngest) {
1014
+ const published = publishedByNumber.get(calldataCheckpoint.checkpointNumber)!;
1015
+
773
1016
  // Check the inHash of the checkpoint against the l1->l2 messages.
774
1017
  // The messages should've been synced up to the currentL1BlockNumber and must be available for the published
775
1018
  // checkpoints we just retrieved.
776
- const l1ToL2Messages = await this.store.getL1ToL2Messages(published.checkpoint.number);
1019
+ const l1ToL2Messages = await this.stores.messages.getL1ToL2Messages(published.checkpoint.number);
777
1020
  const computedInHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
778
1021
  const publishedInHash = published.checkpoint.header.inHash;
779
1022
  if (!computedInHash.equals(publishedInHash)) {
@@ -801,29 +1044,64 @@ export class ArchiverL1Synchronizer implements Traceable {
801
1044
  );
802
1045
  }
803
1046
 
1047
+ for (const published of validCheckpoints) {
1048
+ this.instrumentation.processCheckpointL1Timing({
1049
+ slotNumber: published.checkpoint.header.slotNumber,
1050
+ l1Timestamp: published.l1.timestamp,
1051
+ l1Constants: this.l1Constants,
1052
+ });
1053
+ }
1054
+
804
1055
  try {
805
1056
  const updatedValidationResult =
806
1057
  rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
1058
+
1059
+ // Split valid checkpoints: the promoted one (if any) is persisted via the proposed-promotion path,
1060
+ // the rest via addCheckpoints. Both paths run within the same store transaction for atomicity.
1061
+ const [[maybeValidCheckpointToPromote], checkpointsToAdd] = partition(
1062
+ validCheckpoints,
1063
+ c => c.checkpoint.number === checkpointToPromote?.checkpoint.number,
1064
+ );
1065
+
807
1066
  const [processDuration, result] = await elapsed(() =>
808
1067
  execInSpan(this.tracer, 'Archiver.addCheckpoints', () =>
809
- this.updater.addCheckpoints(validCheckpoints, updatedValidationResult),
1068
+ this.updater.addCheckpoints(
1069
+ checkpointsToAdd,
1070
+ updatedValidationResult,
1071
+ maybeValidCheckpointToPromote && {
1072
+ l1: lastCalldataCheckpoint.l1,
1073
+ attestations: lastCalldataCheckpoint.attestations,
1074
+ checkpoint: maybeValidCheckpointToPromote,
1075
+ },
1076
+ evictProposedFrom,
1077
+ ),
810
1078
  ),
811
1079
  );
812
- this.instrumentation.processNewBlocks(
813
- processDuration / validCheckpoints.length,
814
- validCheckpoints.flatMap(c => c.checkpoint.blocks),
815
- );
1080
+
1081
+ if (validCheckpoints.length > 0) {
1082
+ this.instrumentation.processNewCheckpointedBlocks(
1083
+ processDuration / validCheckpoints.length,
1084
+ validCheckpoints.flatMap(c => c.checkpoint.blocks),
1085
+ );
1086
+ }
1087
+
1088
+ // Record blocks newly fetched from L1 checkpoint payloads as added. The promoted checkpoint (if any) is
1089
+ // excluded: its blocks were already in local archiver storage (added via the proposed-block path) so the
1090
+ // block stream does not need them re-downloaded.
1091
+ blocksAdded.push(...checkpointsToAdd.flatMap(c => c.checkpoint.blocks));
816
1092
 
817
1093
  // If blocks were pruned due to conflict with L1 checkpoints, emit event
818
1094
  if (result.prunedBlocks && result.prunedBlocks.length > 0) {
819
1095
  const prunedCheckpointNumber = result.prunedBlocks[0].checkpointNumber;
820
1096
  const prunedSlotNumber = result.prunedBlocks[0].header.globalVariables.slotNumber;
821
1097
 
822
- this.log.warn(
1098
+ this.log.info(
823
1099
  `Pruned ${result.prunedBlocks.length} mismatching blocks for checkpoint ${prunedCheckpointNumber}`,
824
1100
  { prunedBlocks: result.prunedBlocks.map(b => b.toBlockInfo()), prunedSlotNumber, prunedCheckpointNumber },
825
1101
  );
826
1102
 
1103
+ this.instrumentation.recordPrune('l1_conflict');
1104
+
827
1105
  // Emit event for listening services to react to the prune.
828
1106
  // Note: slotNumber comes from the first pruned block. If pruned blocks theoretically spanned multiple slots,
829
1107
  // only one slot number would be reported (though in practice all blocks in a checkpoint span a single slot).
@@ -837,14 +1115,23 @@ export class ArchiverL1Synchronizer implements Traceable {
837
1115
  if (err instanceof InitialCheckpointNumberNotSequentialError) {
838
1116
  const { previousCheckpointNumber, newCheckpointNumber } = err;
839
1117
  const previousCheckpoint = previousCheckpointNumber
840
- ? await this.store.getCheckpointData(CheckpointNumber(previousCheckpointNumber))
1118
+ ? await this.stores.blocks.getCheckpointData(CheckpointNumber(previousCheckpointNumber))
841
1119
  : undefined;
842
- const updatedL1SyncPoint = previousCheckpoint?.l1.blockNumber ?? this.l1Constants.l1StartBlock;
843
- await this.store.setCheckpointSynchedL1BlockNumber(updatedL1SyncPoint);
1120
+ const lastFinalizedCheckpoint = await this.stores.blocks.getCheckpointData(
1121
+ await this.stores.blocks.getFinalizedCheckpointNumber(),
1122
+ );
1123
+ const updatedL1SyncPoint =
1124
+ previousCheckpoint?.l1.blockNumber ??
1125
+ lastFinalizedCheckpoint?.l1.blockNumber ??
1126
+ this.l1Constants.l1StartBlock;
1127
+ await this.stores.blocks.setSynchedL1BlockNumber(updatedL1SyncPoint);
844
1128
  this.log.warn(
845
1129
  `Attempting to insert checkpoint ${newCheckpointNumber} with previous block ${previousCheckpointNumber}. Rolling back L1 sync point to ${updatedL1SyncPoint} to try and fetch the missing blocks.`,
846
1130
  {
847
1131
  previousCheckpointNumber,
1132
+ previousCheckpoint: previousCheckpoint?.header.toInspect(),
1133
+ lastFinalizedCheckpoint: lastFinalizedCheckpoint?.header.toInspect(),
1134
+ l1StartBlock: this.l1Constants.l1StartBlock,
848
1135
  newCheckpointNumber,
849
1136
  updatedL1SyncPoint,
850
1137
  },
@@ -865,13 +1152,107 @@ export class ArchiverL1Synchronizer implements Traceable {
865
1152
  });
866
1153
  }
867
1154
  lastRetrievedCheckpoint = validCheckpoints.at(-1) ?? lastRetrievedCheckpoint;
868
- lastL1BlockWithCheckpoint = retrievedCheckpoints.at(-1)?.l1.blockNumber ?? lastL1BlockWithCheckpoint;
1155
+ // The last checkpoint seen on L1 this batch (valid or rejected), tracked from calldata since
1156
+ // rejected checkpoints are no longer built into PublishedCheckpoints.
1157
+ lastSeenCheckpoint = lastCalldataCheckpoint;
869
1158
  } while (searchEndBlock < currentL1BlockNumber);
870
1159
 
871
1160
  // Important that we update AFTER inserting the blocks.
872
1161
  await updateProvenCheckpoint();
873
1162
 
874
- return { ...rollupStatus, lastRetrievedCheckpoint, lastL1BlockWithCheckpoint };
1163
+ return { ...rollupStatus, lastRetrievedCheckpoint, lastSeenCheckpoint };
1164
+ }
1165
+
1166
+ /**
1167
+ * Checks if a specific checkpoint matches a local pending entry, and if so, loads local data to build
1168
+ * a synthetic published checkpoint (skipping blob fetch).
1169
+ *
1170
+ * Returns { diverged: true, fromCheckpointNumber } when the L1 checkpoint does NOT match local pending
1171
+ * data for that number, so the caller can evict the entire pending suffix >= fromCheckpointNumber
1172
+ * (those entries chain off the now-invalid local state) within the same addCheckpoints transaction.
1173
+ */
1174
+ private async tryBuildPublishedCheckpointFromProposed(
1175
+ calldataCheckpoint: RetrievedCheckpointFromCalldata | undefined,
1176
+ ): Promise<PublishedCheckpoint | { diverged: true; fromCheckpointNumber: CheckpointNumber } | undefined> {
1177
+ if (this.config.skipPromoteProposedCheckpointDuringL1Sync || !calldataCheckpoint) {
1178
+ return undefined;
1179
+ }
1180
+
1181
+ // Look up the specific pending entry for the checkpoint being mined, not just the tip
1182
+ const proposed = await this.stores.blocks.getProposedCheckpointByNumber(calldataCheckpoint.checkpointNumber);
1183
+ if (!proposed) {
1184
+ return undefined;
1185
+ }
1186
+
1187
+ if (
1188
+ !proposed.header.equals(calldataCheckpoint.header) ||
1189
+ !proposed.archive.root.equals(calldataCheckpoint.archiveRoot)
1190
+ ) {
1191
+ this.log.warn(
1192
+ `Local proposed checkpoint ${proposed.checkpointNumber} does not match checkpoint retrieved from L1, overriding with L1 data`,
1193
+ {
1194
+ proposedCheckpointNumber: proposed.checkpointNumber,
1195
+ proposedHeader: proposed.header.toInspect(),
1196
+ proposedArchiveRoot: proposed.archive.root.toString(),
1197
+ calldataCheckpointNumber: calldataCheckpoint.checkpointNumber,
1198
+ calldataHeader: calldataCheckpoint.header.toInspect(),
1199
+ calldataArchiveRoot: calldataCheckpoint.archiveRoot.toString(),
1200
+ },
1201
+ );
1202
+ // Both the locally-proposed checkpoint and the L1-confirmed one are signed by the
1203
+ // slot proposer; emit a divergence event so the slasher can attribute equivocation.
1204
+ // Only emit when the slots match — uncheckpointed entries are pruned above so this
1205
+ // should always hold, but guard defensively to avoid mis-attributing a slash.
1206
+ if (proposed.header.slotNumber === calldataCheckpoint.header.slotNumber) {
1207
+ this.events.emit(L2BlockSourceEvents.CheckpointEquivocationDetected, {
1208
+ type: L2BlockSourceEvents.CheckpointEquivocationDetected,
1209
+ slotNumber: calldataCheckpoint.header.slotNumber,
1210
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
1211
+ l1ArchiveRoot: calldataCheckpoint.archiveRoot,
1212
+ proposedArchiveRoot: proposed.archive.root,
1213
+ });
1214
+ }
1215
+ // Return a divergence signal so the caller can evict pending >= this number
1216
+ return { diverged: true, fromCheckpointNumber: proposed.checkpointNumber };
1217
+ }
1218
+
1219
+ this.log.debug(
1220
+ `Building published checkpoint from proposed ${calldataCheckpoint.checkpointNumber} (skipping blob fetch)`,
1221
+ { proposedHeader: proposed.header.toInspect(), proposedArchiveRoot: proposed.archive.root.toString() },
1222
+ );
1223
+
1224
+ const blocks = await this.stores.blocks.getBlocks({
1225
+ from: BlockNumber(proposed.startBlock),
1226
+ limit: proposed.blockCount,
1227
+ });
1228
+ if (blocks.length !== proposed.blockCount) {
1229
+ this.log.warn(
1230
+ `Local proposed checkpoint ${proposed.checkpointNumber} has wrong block count (expected ${proposed.blockCount} blocks starting at ${proposed.startBlock} but got ${blocks.length})`,
1231
+ {
1232
+ proposedCheckpointNumber: proposed.checkpointNumber,
1233
+ proposedStartBlock: proposed.startBlock,
1234
+ proposedBlockCount: proposed.blockCount,
1235
+ retrievedBlocks: blocks.map(b => b.number),
1236
+ },
1237
+ );
1238
+ return undefined;
1239
+ }
1240
+
1241
+ const checkpoint = Checkpoint.from({
1242
+ archive: proposed.archive,
1243
+ header: proposed.header,
1244
+ blocks,
1245
+ number: proposed.checkpointNumber,
1246
+ feeAssetPriceModifier: proposed.feeAssetPriceModifier,
1247
+ });
1248
+ const promotedCheckpoint = PublishedCheckpoint.from({
1249
+ checkpoint,
1250
+ l1: calldataCheckpoint.l1,
1251
+ attestations: calldataCheckpoint.attestations,
1252
+ });
1253
+ this.instrumentation.processCheckpointPromoted();
1254
+
1255
+ return promotedCheckpoint;
875
1256
  }
876
1257
 
877
1258
  private async checkForNewCheckpointsBeforeL1SyncPoint(
@@ -879,39 +1260,48 @@ export class ArchiverL1Synchronizer implements Traceable {
879
1260
  blocksSynchedTo: bigint,
880
1261
  currentL1BlockNumber: bigint,
881
1262
  ): Promise<void> {
882
- const { lastRetrievedCheckpoint, pendingCheckpointNumber } = status;
883
- // Compare the last checkpoint we have (either retrieved in this round or loaded from store) with what the
884
- // rollup contract told us was the latest one (pinned at the currentL1BlockNumber).
1263
+ const { lastSeenCheckpoint, pendingCheckpointNumber } = status;
1264
+ // Compare the last checkpoint (valid or not) we have (either retrieved in this round or loaded from store)
1265
+ // with what the rollup contract told us was the latest one (pinned at the currentL1BlockNumber).
885
1266
  const latestLocalCheckpointNumber =
886
- lastRetrievedCheckpoint?.checkpoint.number ?? (await this.store.getSynchedCheckpointNumber());
1267
+ lastSeenCheckpoint?.checkpointNumber ??
1268
+ CheckpointNumber.max(
1269
+ await this.stores.blocks.getLatestCheckpointNumber(),
1270
+ await this.stores.blocks.getLatestRejectedCheckpointNumber(),
1271
+ ) ??
1272
+ CheckpointNumber.ZERO;
1273
+
887
1274
  if (latestLocalCheckpointNumber < pendingCheckpointNumber) {
888
1275
  // Here we have consumed all logs until the `currentL1Block` we pinned at the beginning of the archiver loop,
889
1276
  // but still haven't reached the pending checkpoint according to the call to the rollup contract.
890
1277
  // We suspect an L1 reorg that added checkpoints *behind* us. If that is the case, it must have happened between
891
1278
  // the last checkpoint we saw and the current one, so we reset the last synched L1 block number. In the edge case
892
1279
  // we don't have one, we go back 2 L1 epochs, which is the deepest possible reorg (assuming Casper is working).
893
- let latestLocalCheckpointArchive: string | undefined = undefined;
894
- let targetL1BlockNumber = maxBigint(currentL1BlockNumber - 64n, 0n);
895
- if (lastRetrievedCheckpoint) {
896
- latestLocalCheckpointArchive = lastRetrievedCheckpoint.checkpoint.archive.root.toString();
897
- targetL1BlockNumber = lastRetrievedCheckpoint.l1.blockNumber;
898
- } else if (latestLocalCheckpointNumber > 0) {
899
- const checkpoint = await this.store.getRangeOfCheckpoints(latestLocalCheckpointNumber, 1).then(([c]) => c);
900
- latestLocalCheckpointArchive = checkpoint.archive.root.toString();
901
- targetL1BlockNumber = checkpoint.l1.blockNumber;
902
- }
1280
+ const latestLocalCheckpoint:
1281
+ | { checkpointNumber: CheckpointNumber; l1: L1PublishedData }
1282
+ | CheckpointData
1283
+ | RejectedCheckpoint
1284
+ | undefined =
1285
+ lastSeenCheckpoint ??
1286
+ (await this.stores.blocks.getCheckpointData(latestLocalCheckpointNumber)) ??
1287
+ (await this.stores.blocks.getRejectedCheckpointByNumber(latestLocalCheckpointNumber));
1288
+
1289
+ const targetL1BlockNumber =
1290
+ latestLocalCheckpoint?.l1.blockNumber ??
1291
+ maxBigint(currentL1BlockNumber - 64n, this.l1Constants.l1StartBlock, 0n);
1292
+
903
1293
  this.log.warn(
904
1294
  `Failed to reach checkpoint ${pendingCheckpointNumber} at ${currentL1BlockNumber} (latest is ${latestLocalCheckpointNumber}). ` +
905
1295
  `Rolling back last synched L1 block number to ${targetL1BlockNumber}.`,
906
1296
  {
907
1297
  latestLocalCheckpointNumber,
908
- latestLocalCheckpointArchive,
1298
+ latestLocalCheckpointL1: latestLocalCheckpoint?.l1,
909
1299
  blocksSynchedTo,
910
1300
  currentL1BlockNumber,
911
1301
  ...status,
912
1302
  },
913
1303
  );
914
- await this.store.setCheckpointSynchedL1BlockNumber(targetL1BlockNumber);
1304
+ await this.stores.blocks.setSynchedL1BlockNumber(targetL1BlockNumber);
915
1305
  } else {
916
1306
  this.log.trace(`No new checkpoints behind L1 sync point to retrieve.`, {
917
1307
  latestLocalCheckpointNumber,
@@ -921,7 +1311,7 @@ export class ArchiverL1Synchronizer implements Traceable {
921
1311
  }
922
1312
 
923
1313
  private async getCheckpointHeader(number: CheckpointNumber) {
924
- const checkpoint = await this.store.getCheckpointData(number);
1314
+ const checkpoint = await this.stores.blocks.getCheckpointData(number);
925
1315
  if (!checkpoint) {
926
1316
  return undefined;
927
1317
  }