@aztec/archiver 0.0.1-commit.86469d5 → 0.0.1-commit.8655d4a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +19 -11
  2. package/dest/archiver.d.ts +39 -17
  3. package/dest/archiver.d.ts.map +1 -1
  4. package/dest/archiver.js +260 -160
  5. package/dest/config.d.ts +6 -3
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +23 -15
  8. package/dest/errors.d.ts +55 -9
  9. package/dest/errors.d.ts.map +1 -1
  10. package/dest/errors.js +81 -14
  11. package/dest/factory.d.ts +13 -9
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +51 -37
  14. package/dest/index.d.ts +12 -3
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +11 -2
  17. package/dest/l1/bin/retrieve-calldata.js +36 -33
  18. package/dest/l1/calldata_retriever.d.ts +74 -50
  19. package/dest/l1/calldata_retriever.d.ts.map +1 -1
  20. package/dest/l1/calldata_retriever.js +201 -260
  21. package/dest/l1/data_retrieval.d.ts +26 -17
  22. package/dest/l1/data_retrieval.d.ts.map +1 -1
  23. package/dest/l1/data_retrieval.js +42 -47
  24. package/dest/l1/spire_proposer.d.ts +5 -5
  25. package/dest/l1/spire_proposer.d.ts.map +1 -1
  26. package/dest/l1/spire_proposer.js +9 -17
  27. package/dest/l1/trace_tx.d.ts +12 -66
  28. package/dest/l1/trace_tx.d.ts.map +1 -1
  29. package/dest/l1/validate_historical_logs.d.ts +23 -0
  30. package/dest/l1/validate_historical_logs.d.ts.map +1 -0
  31. package/dest/l1/validate_historical_logs.js +108 -0
  32. package/dest/modules/contract_data_source_adapter.d.ts +25 -0
  33. package/dest/modules/contract_data_source_adapter.d.ts.map +1 -0
  34. package/dest/modules/contract_data_source_adapter.js +40 -0
  35. package/dest/modules/data_source_base.d.ts +71 -42
  36. package/dest/modules/data_source_base.d.ts.map +1 -1
  37. package/dest/modules/data_source_base.js +270 -179
  38. package/dest/modules/data_store_updater.d.ts +49 -17
  39. package/dest/modules/data_store_updater.d.ts.map +1 -1
  40. package/dest/modules/data_store_updater.js +211 -121
  41. package/dest/modules/instrumentation.d.ts +21 -3
  42. package/dest/modules/instrumentation.d.ts.map +1 -1
  43. package/dest/modules/instrumentation.js +44 -9
  44. package/dest/modules/l1_synchronizer.d.ts +14 -12
  45. package/dest/modules/l1_synchronizer.d.ts.map +1 -1
  46. package/dest/modules/l1_synchronizer.js +443 -211
  47. package/dest/modules/validation.d.ts +4 -3
  48. package/dest/modules/validation.d.ts.map +1 -1
  49. package/dest/modules/validation.js +6 -6
  50. package/dest/store/block_store.d.ts +174 -66
  51. package/dest/store/block_store.d.ts.map +1 -1
  52. package/dest/store/block_store.js +743 -245
  53. package/dest/store/contract_class_store.d.ts +17 -4
  54. package/dest/store/contract_class_store.d.ts.map +1 -1
  55. package/dest/store/contract_class_store.js +24 -68
  56. package/dest/store/contract_instance_store.d.ts +28 -1
  57. package/dest/store/contract_instance_store.d.ts.map +1 -1
  58. package/dest/store/contract_instance_store.js +37 -2
  59. package/dest/store/data_stores.d.ts +68 -0
  60. package/dest/store/data_stores.d.ts.map +1 -0
  61. package/dest/store/data_stores.js +54 -0
  62. package/dest/store/function_names_cache.d.ts +17 -0
  63. package/dest/store/function_names_cache.d.ts.map +1 -0
  64. package/dest/store/function_names_cache.js +30 -0
  65. package/dest/store/l2_tips_cache.d.ts +25 -0
  66. package/dest/store/l2_tips_cache.d.ts.map +1 -0
  67. package/dest/store/l2_tips_cache.js +26 -0
  68. package/dest/store/log_store.d.ts +42 -37
  69. package/dest/store/log_store.d.ts.map +1 -1
  70. package/dest/store/log_store.js +262 -388
  71. package/dest/store/log_store_codec.d.ts +70 -0
  72. package/dest/store/log_store_codec.d.ts.map +1 -0
  73. package/dest/store/log_store_codec.js +101 -0
  74. package/dest/store/message_store.d.ts +11 -1
  75. package/dest/store/message_store.d.ts.map +1 -1
  76. package/dest/store/message_store.js +51 -9
  77. package/dest/test/fake_l1_state.d.ts +25 -1
  78. package/dest/test/fake_l1_state.d.ts.map +1 -1
  79. package/dest/test/fake_l1_state.js +166 -32
  80. package/dest/test/mock_archiver.d.ts +1 -1
  81. package/dest/test/mock_archiver.d.ts.map +1 -1
  82. package/dest/test/mock_archiver.js +3 -2
  83. package/dest/test/mock_l1_to_l2_message_source.d.ts +1 -1
  84. package/dest/test/mock_l1_to_l2_message_source.d.ts.map +1 -1
  85. package/dest/test/mock_l1_to_l2_message_source.js +2 -1
  86. package/dest/test/mock_l2_block_source.d.ts +62 -41
  87. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  88. package/dest/test/mock_l2_block_source.js +321 -202
  89. package/dest/test/mock_structs.d.ts +4 -1
  90. package/dest/test/mock_structs.d.ts.map +1 -1
  91. package/dest/test/mock_structs.js +13 -1
  92. package/dest/test/noop_l1_archiver.d.ts +12 -6
  93. package/dest/test/noop_l1_archiver.d.ts.map +1 -1
  94. package/dest/test/noop_l1_archiver.js +26 -9
  95. package/package.json +14 -14
  96. package/src/archiver.ts +319 -181
  97. package/src/config.ts +32 -12
  98. package/src/errors.ts +122 -21
  99. package/src/factory.ts +75 -36
  100. package/src/index.ts +19 -2
  101. package/src/l1/README.md +25 -68
  102. package/src/l1/bin/retrieve-calldata.ts +46 -39
  103. package/src/l1/calldata_retriever.ts +260 -379
  104. package/src/l1/data_retrieval.ts +58 -69
  105. package/src/l1/spire_proposer.ts +7 -15
  106. package/src/l1/validate_historical_logs.ts +140 -0
  107. package/src/modules/contract_data_source_adapter.ts +55 -0
  108. package/src/modules/data_source_base.ts +347 -221
  109. package/src/modules/data_store_updater.ts +248 -153
  110. package/src/modules/instrumentation.ts +56 -9
  111. package/src/modules/l1_synchronizer.ts +585 -258
  112. package/src/modules/validation.ts +10 -9
  113. package/src/store/block_store.ts +924 -300
  114. package/src/store/contract_class_store.ts +31 -103
  115. package/src/store/contract_instance_store.ts +51 -5
  116. package/src/store/data_stores.ts +104 -0
  117. package/src/store/function_names_cache.ts +37 -0
  118. package/src/store/l2_tips_cache.ts +35 -0
  119. package/src/store/log_store.ts +303 -499
  120. package/src/store/log_store_codec.ts +132 -0
  121. package/src/store/message_store.ts +60 -10
  122. package/src/structs/inbox_message.ts +1 -1
  123. package/src/test/fake_l1_state.ts +213 -42
  124. package/src/test/mock_archiver.ts +3 -2
  125. package/src/test/mock_l1_to_l2_message_source.ts +1 -0
  126. package/src/test/mock_l2_block_source.ts +394 -210
  127. package/src/test/mock_structs.ts +20 -6
  128. package/src/test/noop_l1_archiver.ts +39 -9
  129. package/dest/store/kv_archiver_store.d.ts +0 -340
  130. package/dest/store/kv_archiver_store.d.ts.map +0 -1
  131. package/dest/store/kv_archiver_store.js +0 -446
  132. package/src/store/kv_archiver_store.ts +0 -639
package/src/archiver.ts CHANGED
@@ -1,57 +1,68 @@
1
1
  import type { BlobClientInterface } from '@aztec/blob-client/client';
2
- import { GENESIS_BLOCK_HEADER_HASH, INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
3
2
  import { EpochCache } from '@aztec/epoch-cache';
4
3
  import { BlockTagTooOldError, RollupContract } from '@aztec/ethereum/contracts';
5
4
  import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
6
5
  import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
7
6
  import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
8
7
  import { Buffer32 } from '@aztec/foundation/buffer';
9
- import { merge } from '@aztec/foundation/collection';
8
+ import { merge, pick } from '@aztec/foundation/collection';
10
9
  import { Fr } from '@aztec/foundation/curves/bn254';
11
10
  import { EthAddress } from '@aztec/foundation/eth-address';
12
11
  import { type Logger, createLogger } from '@aztec/foundation/log';
13
12
  import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise';
14
13
  import { RunningPromise, makeLoggingErrorHandler } from '@aztec/foundation/running-promise';
15
- import { DateProvider } from '@aztec/foundation/timer';
14
+ import { DateProvider, elapsed } from '@aztec/foundation/timer';
16
15
  import {
17
16
  type ArchiverEmitter,
18
- type CheckpointId,
19
- GENESIS_CHECKPOINT_HEADER_HASH,
17
+ type BlockHash,
20
18
  L2Block,
21
19
  type L2BlockSink,
20
+ L2BlockSourceEvents,
22
21
  type L2Tips,
23
22
  type ValidateCheckpointResult,
24
23
  } from '@aztec/stdlib/block';
25
- import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
24
+ import { type ProposedCheckpointInput, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
26
25
  import {
27
26
  type L1RollupConstants,
28
- getEpochNumberAtTimestamp,
27
+ getEpochAtSlot,
29
28
  getSlotAtNextL1Block,
30
- getSlotAtTimestamp,
31
29
  getSlotRangeForEpoch,
30
+ getTimestampForSlot,
32
31
  getTimestampRangeForEpoch,
33
32
  } from '@aztec/stdlib/epoch-helpers';
33
+ import type { BlockHeader } from '@aztec/stdlib/tx';
34
34
  import { type TelemetryClient, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
35
35
 
36
36
  import { type ArchiverConfig, mapArchiverConfig } from './config.js';
37
- import { NoBlobBodiesFoundError } from './errors.js';
37
+ import { BlockAlreadyCheckpointedError, BlockOrCheckpointSlotExpiredError, NoBlobBodiesFoundError } from './errors.js';
38
+ import { validateAndLogHistoricalLogsAvailability } from './l1/validate_historical_logs.js';
38
39
  import { validateAndLogTraceAvailability } from './l1/validate_trace.js';
39
40
  import { ArchiverDataSourceBase } from './modules/data_source_base.js';
40
41
  import { ArchiverDataStoreUpdater } from './modules/data_store_updater.js';
41
42
  import type { ArchiverInstrumentation } from './modules/instrumentation.js';
42
43
  import type { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js';
43
- import type { KVArchiverDataStore } from './store/kv_archiver_store.js';
44
+ import { type ArchiverDataStores, backupArchiverDataStores, getArchiverSynchPoint } from './store/data_stores.js';
45
+ import { L2TipsCache } from './store/l2_tips_cache.js';
44
46
 
45
47
  /** Export ArchiverEmitter for use in factory and tests. */
46
48
  export type { ArchiverEmitter };
47
49
 
48
50
  /** Request to add a block to the archiver, queued for processing by the sync loop. */
49
51
  type AddBlockRequest = {
52
+ type: 'block';
50
53
  block: L2Block;
51
54
  resolve: () => void;
52
55
  reject: (err: Error) => void;
53
56
  };
54
57
 
58
+ /** Request to add a proposed checkpoint to the archiver, queued for processing by the sync loop. */
59
+ type AddProposedCheckpointRequest = {
60
+ type: 'checkpoint';
61
+ checkpoint: ProposedCheckpointInput;
62
+ resolve: () => void;
63
+ reject: (err: Error) => void;
64
+ };
65
+
55
66
  export type ArchiverDeps = {
56
67
  telemetry?: TelemetryClient;
57
68
  blobClient: BlobClientInterface;
@@ -77,28 +88,38 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
77
88
  private initialSyncComplete: boolean = false;
78
89
  private initialSyncPromise: PromiseWithResolvers<void>;
79
90
 
80
- /** Queue of blocks to be added to the store, processed by the sync loop. */
81
- private blockQueue: AddBlockRequest[] = [];
91
+ /** Queue of blocks and checkpoints to be added to the store, processed by the sync loop. */
92
+ private inboundQueue: (AddBlockRequest | AddProposedCheckpointRequest)[] = [];
82
93
 
83
94
  /** Helper to handle updates to the store */
84
95
  private readonly updater: ArchiverDataStoreUpdater;
85
96
 
97
+ /** In-memory cache for L2 chain tips. */
98
+ private readonly l2TipsCache: L2TipsCache;
99
+
86
100
  public readonly tracer: Tracer;
87
101
 
102
+ private readonly instrumentation: ArchiverInstrumentation;
103
+
88
104
  /**
89
105
  * Creates a new instance of the Archiver.
90
106
  * @param publicClient - A client for interacting with the Ethereum node.
91
107
  * @param debugClient - A client for interacting with the Ethereum node for debug/trace methods.
92
108
  * @param rollup - Rollup contract instance.
93
109
  * @param inbox - Inbox contract instance.
94
- * @param l1Addresses - L1 contract addresses (registry, governance proposer, slash factory, slashing proposer).
95
- * @param dataStore - An archiver data store for storage & retrieval of blocks, encrypted logs & contract data.
110
+ * @param l1Addresses - L1 contract addresses (registry, governance proposer, slashing proposer).
111
+ * @param dataStores - Archiver substores for storage & retrieval of blocks, encrypted logs & contract data.
96
112
  * @param config - Archiver configuration options.
97
113
  * @param blobClient - Client for retrieving blob data.
98
- * @param epochCache - Cache for epoch-related data.
99
- * @param dateProvider - Provider for current date/time.
100
114
  * @param instrumentation - Instrumentation for metrics and tracing.
101
115
  * @param l1Constants - L1 rollup constants.
116
+ * @param synchronizer - L1 synchronizer that handles fetching checkpoints and messages from L1.
117
+ * @param events - Event emitter shared with the synchronizer.
118
+ * @param initialHeader - Genesis block header.
119
+ * @param initialBlockHash - Precomputed hash of the genesis block header.
120
+ * @param l2TipsCache - In-memory cache for L2 chain tips.
121
+ * @param epochCache - Cache used to compute the proposer pipelining offset.
122
+ * @param dateProvider - Provider for current date/time, used for wall-clock orphan-block pruning.
102
123
  * @param log - A logger.
103
124
  */
104
125
  constructor(
@@ -107,30 +128,46 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
107
128
  private readonly rollup: RollupContract,
108
129
  private readonly l1Addresses: Pick<
109
130
  L1ContractAddresses,
110
- 'registryAddress' | 'governanceProposerAddress' | 'slashFactoryAddress'
111
- > & { slashingProposerAddress: EthAddress },
112
- readonly dataStore: KVArchiverDataStore,
131
+ 'rollupAddress' | 'registryAddress' | 'inboxAddress' | 'governanceProposerAddress'
132
+ > & {
133
+ slashingProposerAddress: EthAddress;
134
+ },
135
+ readonly dataStores: ArchiverDataStores,
113
136
  private config: {
114
137
  pollingIntervalMs: number;
115
138
  batchSize: number;
116
139
  skipValidateCheckpointAttestations?: boolean;
117
140
  maxAllowedEthClientDriftSeconds: number;
118
141
  ethereumAllowNoDebugHosts?: boolean;
142
+ skipHistoricalLogsCheck?: boolean;
143
+ orphanProposedBlockPruneGraceSeconds: number;
119
144
  },
120
145
  private readonly blobClient: BlobClientInterface,
121
146
  instrumentation: ArchiverInstrumentation,
122
- protected override readonly l1Constants: L1RollupConstants & { l1StartBlockHash: Buffer32; genesisArchiveRoot: Fr },
147
+ protected override readonly l1Constants: L1RollupConstants & {
148
+ l1StartBlockHash: Buffer32;
149
+ genesisArchiveRoot: Fr;
150
+ },
123
151
  synchronizer: ArchiverL1Synchronizer,
124
152
  events: ArchiverEmitter,
153
+ initialHeader: BlockHeader,
154
+ initialBlockHash: BlockHash,
155
+ l2TipsCache: L2TipsCache,
156
+ private readonly epochCache: EpochCache,
157
+ private readonly dateProvider: DateProvider,
125
158
  private readonly log: Logger = createLogger('archiver'),
126
159
  ) {
127
- super(dataStore, l1Constants);
160
+ super(dataStores, l1Constants, initialHeader, initialBlockHash, l1Constants.genesisArchiveRoot);
128
161
 
129
162
  this.tracer = instrumentation.tracer;
163
+ this.instrumentation = instrumentation;
130
164
  this.initialSyncPromise = promiseWithResolvers();
131
165
  this.synchronizer = synchronizer;
132
166
  this.events = events;
133
- this.updater = new ArchiverDataStoreUpdater(this.dataStore);
167
+ this.l2TipsCache = l2TipsCache;
168
+ this.updater = new ArchiverDataStoreUpdater(this.dataStores, this.l2TipsCache, {
169
+ rollupManaLimit: l1Constants.rollupManaLimit,
170
+ });
134
171
 
135
172
  // Running promise starts with a small interval inbetween runs, so all iterations needed for the initial sync
136
173
  // are done as fast as possible. This then gets updated once the initial sync completes.
@@ -164,11 +201,24 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
164
201
  this.config.ethereumAllowNoDebugHosts ?? false,
165
202
  this.log.getBindings(),
166
203
  );
204
+ await validateAndLogHistoricalLogsAvailability(
205
+ this.publicClient,
206
+ {
207
+ rollupAddress: this.l1Addresses.rollupAddress,
208
+ inboxAddress: this.l1Addresses.inboxAddress,
209
+ registryAddress: this.l1Addresses.registryAddress,
210
+ governanceProposerAddress: this.l1Addresses.governanceProposerAddress,
211
+ },
212
+ this.config.skipHistoricalLogsCheck ?? false,
213
+ this.log.getBindings(),
214
+ );
167
215
 
168
216
  // Log initial state for the archiver
169
217
  const { l1StartBlock } = this.l1Constants;
170
- const { blocksSynchedTo = l1StartBlock, messagesSynchedTo = l1StartBlock } = await this.store.getSynchPoint();
171
- const currentL2Checkpoint = await this.getSynchedCheckpointNumber();
218
+ const { blocksSynchedTo = l1StartBlock, messagesSynchedTo = l1StartBlock } = await getArchiverSynchPoint(
219
+ this.stores,
220
+ );
221
+ const currentL2Checkpoint = await this.getCheckpointNumber();
172
222
  this.log.info(
173
223
  `Starting archiver sync to rollup contract ${this.rollup.address} from L1 block ${blocksSynchedTo} and L2 checkpoint ${currentL2Checkpoint}`,
174
224
  { blocksSynchedTo, messagesSynchedTo, currentL2Checkpoint },
@@ -185,6 +235,14 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
185
235
  return this.runningPromise.trigger();
186
236
  }
187
237
 
238
+ public trySyncImmediate() {
239
+ try {
240
+ return this.syncImmediate();
241
+ } catch (err) {
242
+ this.log.error(`Failed to trigger immediate archiver sync: ${err}`, err);
243
+ }
244
+ }
245
+
188
246
  /**
189
247
  * Queues a block to be added to the archiver store and triggers processing.
190
248
  * The block will be processed by the sync loop.
@@ -193,53 +251,85 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
193
251
  * @returns A promise that resolves when the block has been added to the store, or rejects on error.
194
252
  */
195
253
  public addBlock(block: L2Block): Promise<void> {
196
- return new Promise<void>((resolve, reject) => {
197
- this.blockQueue.push({ block, resolve, reject });
198
- this.log.debug(`Queued block ${block.number} for processing`);
199
- // Trigger an immediate sync, but don't wait for it - the promise resolves when the block is processed
200
- this.syncImmediate().catch(err => {
201
- this.log.error(`Sync immediate call failed: ${err}`);
202
- });
203
- });
254
+ const promise = promiseWithResolvers<void>();
255
+ this.inboundQueue.push({ block, ...promise, type: 'block' });
256
+ this.log.debug(`Queued block ${block.number} for processing`);
257
+ void this.trySyncImmediate();
258
+ return promise.promise;
204
259
  }
205
260
 
206
261
  /**
207
- * Processes all queued blocks, adding them to the store.
262
+ * Queues a new proposed checkpoint into the archiver store.
263
+ * Checks that the checkpoint is not for an L2 slot already synced from L1.
264
+ * Resolves once the checkpoint has been processed.
265
+ */
266
+ public addProposedCheckpoint(pending: ProposedCheckpointInput): Promise<void> {
267
+ const promise = promiseWithResolvers<void>();
268
+ this.inboundQueue.push({ checkpoint: pending, ...promise, type: 'checkpoint' });
269
+ this.log.debug(`Queued checkpoint ${pending.checkpointNumber} for processing`);
270
+ void this.trySyncImmediate();
271
+ return promise.promise;
272
+ }
273
+
274
+ /**
275
+ * Processes all queued blocks and checkpoints, adding them to the store.
208
276
  * Called at the beginning of each sync iteration.
209
- * Blocks are processed in the order they were queued.
277
+ * Items are processed in the order they were queued.
210
278
  */
211
- private async processQueuedBlocks(): Promise<void> {
212
- if (this.blockQueue.length === 0) {
279
+ private async processInboundQueue(): Promise<void> {
280
+ if (this.inboundQueue.length === 0) {
213
281
  return;
214
282
  }
215
283
 
216
- // Take all blocks from the queue
217
- const queuedItems = this.blockQueue.splice(0, this.blockQueue.length);
218
- this.log.debug(`Processing ${queuedItems.length} queued block(s)`);
284
+ // Take all items from the queue
285
+ const queuedItems = this.inboundQueue.splice(0, this.inboundQueue.length);
286
+ this.log.debug(`Processing ${queuedItems.length} queued inbound items`);
219
287
 
220
288
  // Calculate slot threshold for validation
221
289
  const l1Timestamp = this.synchronizer.getL1Timestamp();
222
290
  const slotAtNextL1Block =
223
291
  l1Timestamp === undefined ? undefined : getSlotAtNextL1Block(l1Timestamp, this.l1Constants);
224
292
 
225
- // Process each block individually to properly resolve/reject each promise
226
- for (const { block, resolve, reject } of queuedItems) {
227
- const blockSlot = block.header.globalVariables.slotNumber;
228
- if (slotAtNextL1Block !== undefined && blockSlot < slotAtNextL1Block) {
293
+ // Helpers for manipulating blocks and checkpoints in the queue
294
+ const getSlot: (item: AddBlockRequest | AddProposedCheckpointRequest) => SlotNumber = item =>
295
+ item.type === 'block' ? item.block.header.globalVariables.slotNumber : item.checkpoint.header.slotNumber;
296
+ const getNumber: (item: AddBlockRequest | AddProposedCheckpointRequest) => number = item =>
297
+ item.type === 'block' ? item.block.number : item.checkpoint.checkpointNumber;
298
+
299
+ // Process each item individually to properly resolve/reject each promise
300
+ for (const item of queuedItems) {
301
+ const { resolve, reject, type } = item;
302
+ const itemSlot = getSlot(item);
303
+ const itemNumber = getNumber(item);
304
+ if (slotAtNextL1Block !== undefined && itemSlot < slotAtNextL1Block) {
305
+ const nextSlotTimestamp = getTimestampForSlot(slotAtNextL1Block, this.l1Constants);
229
306
  this.log.warn(
230
- `Rejecting proposed block ${block.number} for past slot ${blockSlot} (current is ${slotAtNextL1Block})`,
231
- { block: block.toBlockInfo(), l1Timestamp, slotAtNextL1Block },
307
+ `Rejecting proposed ${type} ${itemNumber} for past slot ${itemSlot} (current ${slotAtNextL1Block})`,
308
+ { number: itemNumber, type, l1Timestamp, slotAtNextL1Block, nextSlotTimestamp },
232
309
  );
233
- reject(new Error(`Block ${block.number} is for past slot ${blockSlot} (current is ${slotAtNextL1Block})`));
310
+ reject(new BlockOrCheckpointSlotExpiredError(itemSlot, nextSlotTimestamp, l1Timestamp));
234
311
  continue;
235
312
  }
236
313
 
237
314
  try {
238
- await this.updater.addProposedBlocks([block]);
239
- this.log.debug(`Added block ${block.number} to store`);
315
+ if (type === 'block') {
316
+ const [durationMs] = await elapsed(() => this.updater.addProposedBlock(item.block));
317
+ this.instrumentation.processNewProposedBlock(durationMs, item.block);
318
+ } else {
319
+ await this.updater.addProposedCheckpoint(item.checkpoint);
320
+ }
321
+ this.log.debug(`Added ${type} ${itemNumber} to store`);
240
322
  resolve();
241
323
  } catch (err: any) {
242
- this.log.error(`Failed to add block ${block.number} to store: ${err.message}`);
324
+ if (err instanceof BlockAlreadyCheckpointedError) {
325
+ this.log.debug(`Proposed block ${itemNumber} matches already checkpointed block, ignoring late proposal`);
326
+ resolve();
327
+ continue;
328
+ }
329
+ this.log.error(`Failed to add ${type} ${itemNumber} to store: ${err.message}`, err, {
330
+ number: itemNumber,
331
+ type,
332
+ });
243
333
  reject(err);
244
334
  }
245
335
  }
@@ -255,9 +345,104 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
255
345
  @trackSpan('Archiver.sync')
256
346
  private async sync() {
257
347
  // Process any queued blocks first, before doing L1 sync
258
- await this.processQueuedBlocks();
348
+ await this.processInboundQueue();
259
349
  // Now perform L1 sync
260
350
  await this.syncFromL1();
351
+ // Prune proposed blocks with no corresponding proposed checkpoint by the end of their build slot.
352
+ await this.pruneOrphanProposedBlocks();
353
+ }
354
+
355
+ /**
356
+ * Prunes a block-only local tip that was built atop a checkpoint that was never itself proposed.
357
+ *
358
+ * Under pipelining, a proposer publishes the blocks for a checkpoint (block-only proposals) before
359
+ * assembling and publishing the enclosing proposed checkpoint at the end of the build slot. A node
360
+ * that received those blocks but never the proposed checkpoint is left with an orphan tip it must
361
+ * not build on. We prune it once enough wall-clock time has elapsed that the proposed checkpoint
362
+ * should have arrived. This runs on wall-clock time (not L1 block advancement) so it fires during
363
+ * quiet L1 periods, and is the liveness counterpart to the sequencer's checkSync guard.
364
+ *
365
+ * The uncheckpointed suffix is scanned in order. Blocks covered by proposed checkpoints are left in
366
+ * place; the first block not covered by a proposed checkpoint starts the orphan suffix to prune.
367
+ */
368
+ private async pruneOrphanProposedBlocks(): Promise<void> {
369
+ const tips = await this.getL2Tips();
370
+ const now = BigInt(this.dateProvider.nowInSeconds());
371
+ const pipeliningOffset = this.epochCache.pipeliningOffset();
372
+
373
+ // This only applies under pipelining
374
+ if (pipeliningOffset === 0) {
375
+ this.log.trace(`Pipelining offset is 0, skipping orphan proposed block pruning`);
376
+ return;
377
+ }
378
+
379
+ // The proposed tip is a proposed-checkpointed block, so there are no orphan proposed blocks to prune
380
+ if (tips.proposedCheckpoint.block.number === tips.proposed.number) {
381
+ this.log.trace(
382
+ `No orphan proposed blocks to prune: proposed tip ${tips.proposed.number} is checkpointed`,
383
+ pick(tips, 'proposed', 'proposedCheckpoint'),
384
+ );
385
+ return;
386
+ }
387
+
388
+ // Load the blocks that are candidates for pruning (ie blocks without a proposed checkpoint covering them)
389
+ const blocksWithoutProposedCheckpoint = await this.stores.blocks.getBlocksData({
390
+ from: BlockNumber(tips.proposedCheckpoint.block.number + 1),
391
+ limit: tips.proposed.number - tips.proposedCheckpoint.block.number,
392
+ });
393
+
394
+ // Iterate through them in order, the first one with a slot that should have received a proposed checkpoint
395
+ // is the first orphan block, and all blocks after it are also orphaned and should be pruned.
396
+ let lastSlotChecked = undefined;
397
+ for (const blockData of blocksWithoutProposedCheckpoint) {
398
+ // No need to recheck if this block had the same slot as the previous one.
399
+ const blockSlot = blockData.header.getSlot();
400
+ const blockNumber = blockData.header.getBlockNumber();
401
+ if (lastSlotChecked !== undefined && blockSlot === lastSlotChecked) {
402
+ continue;
403
+ }
404
+ lastSlotChecked = blockSlot;
405
+
406
+ // The proposed checkpoint should have landed by the start of the slot after the block's build slot
407
+ // (build slot = blockSlot - pipeliningOffset). Wait a grace period beyond that to tolerate propagation.
408
+ const expectedCheckpointedBySlot = SlotNumber(Number(blockSlot) - pipeliningOffset + 1);
409
+ const expectedCheckpointedByTime =
410
+ getTimestampForSlot(expectedCheckpointedBySlot, this.l1Constants) +
411
+ BigInt(this.config.orphanProposedBlockPruneGraceSeconds);
412
+
413
+ // If it's not checkpointed by the expected time, prune it along with all blocks after it.
414
+ if (now >= expectedCheckpointedByTime) {
415
+ const pruneAfterBlockNumber = BlockNumber(blockNumber - 1);
416
+ this.log.warn(
417
+ `Pruning orphan blocks after block ${pruneAfterBlockNumber}: block at slot ${blockSlot} belongs to ` +
418
+ `checkpoint ${blockData.checkpointNumber} which has no matching proposed checkpoint`,
419
+ {
420
+ firstUncheckpointedBlockHeader: blockData.header.toInspect(),
421
+ blockCheckpointNumber: blockData.checkpointNumber,
422
+ blockNumber,
423
+ blockSlot,
424
+ pipeliningOffset,
425
+ expectedCheckpointedBySlot,
426
+ expectedCheckpointedByTime,
427
+ now,
428
+ },
429
+ );
430
+
431
+ const prunedBlocks = await this.updater.removeBlocksWithoutProposedCheckpointAfter(pruneAfterBlockNumber);
432
+ if (prunedBlocks.length > 0) {
433
+ this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
434
+ type: L2BlockSourceEvents.L2PruneUncheckpointed,
435
+ slotNumber: blockSlot,
436
+ blocks: prunedBlocks,
437
+ });
438
+ }
439
+ return;
440
+ }
441
+ }
442
+
443
+ this.log.trace('No orphan proposed blocks to prune: all uncheckpointed blocks are still within the grace period', {
444
+ blocksWithoutProposedCheckpoint: blocksWithoutProposedCheckpoint.map(b => b.header.toInspect()),
445
+ });
261
446
  }
262
447
 
263
448
  private async syncFromL1() {
@@ -271,7 +456,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
271
456
  if (currentL1BlockNumber + 1n >= l1BlockNumberAtEnd) {
272
457
  this.log.info(`Initial archiver sync to L1 block ${currentL1BlockNumber} complete`, {
273
458
  l1BlockNumber: currentL1BlockNumber,
274
- syncPoint: await this.store.getSynchPoint(),
459
+ syncPoint: await getArchiverSynchPoint(this.stores),
275
460
  ...(await this.getL2Tips()),
276
461
  });
277
462
  this.runningPromise.setPollingIntervalMS(this.config.pollingIntervalMs);
@@ -303,7 +488,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
303
488
  }
304
489
 
305
490
  public backupTo(destPath: string): Promise<string> {
306
- return this.dataStore.backupTo(destPath);
491
+ return backupArchiverDataStores(this.dataStores, destPath);
307
492
  }
308
493
 
309
494
  public getL1Constants(): Promise<L1RollupConstants> {
@@ -330,24 +515,56 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
330
515
  return Promise.resolve(this.synchronizer.getL1Timestamp());
331
516
  }
332
517
 
333
- public getL2SlotNumber(): Promise<SlotNumber | undefined> {
518
+ public async getSyncedL2SlotNumber(): Promise<SlotNumber | undefined> {
519
+ // The synced L2 slot is the latest slot for which we have all L1 data,
520
+ // either because we have seen all L1 blocks for that slot, or because
521
+ // we have seen the corresponding checkpoint.
522
+
523
+ let slotFromL1Sync: SlotNumber | undefined;
334
524
  const l1Timestamp = this.synchronizer.getL1Timestamp();
335
- return Promise.resolve(l1Timestamp === undefined ? undefined : getSlotAtTimestamp(l1Timestamp, this.l1Constants));
525
+ if (l1Timestamp !== undefined) {
526
+ const nextL1BlockSlot = getSlotAtNextL1Block(l1Timestamp, this.l1Constants);
527
+ if (Number(nextL1BlockSlot) > 0) {
528
+ slotFromL1Sync = SlotNumber.add(nextL1BlockSlot, -1);
529
+ }
530
+ }
531
+
532
+ let slotFromCheckpoint: SlotNumber | undefined;
533
+ const latestCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
534
+ if (latestCheckpointNumber > 0) {
535
+ const checkpointData = await this.stores.blocks.getCheckpointData(latestCheckpointNumber);
536
+ if (checkpointData) {
537
+ slotFromCheckpoint = checkpointData.header.slotNumber;
538
+ }
539
+ }
540
+
541
+ if (slotFromL1Sync === undefined && slotFromCheckpoint === undefined) {
542
+ return undefined;
543
+ }
544
+ return SlotNumber(Math.max(slotFromL1Sync ?? 0, slotFromCheckpoint ?? 0));
336
545
  }
337
546
 
338
- public getL2EpochNumber(): Promise<EpochNumber | undefined> {
339
- const l1Timestamp = this.synchronizer.getL1Timestamp();
340
- return Promise.resolve(
341
- l1Timestamp === undefined ? undefined : getEpochNumberAtTimestamp(l1Timestamp, this.l1Constants),
342
- );
547
+ public async getSyncedL2EpochNumber(): Promise<EpochNumber | undefined> {
548
+ const syncedSlot = await this.getSyncedL2SlotNumber();
549
+ if (syncedSlot === undefined) {
550
+ return undefined;
551
+ }
552
+ // An epoch is fully synced when all its slots are synced.
553
+ // We check if syncedSlot is the last slot of its epoch; if so, that epoch is fully synced.
554
+ // Otherwise, only the previous epoch is fully synced.
555
+ const epoch = getEpochAtSlot(syncedSlot, this.l1Constants);
556
+ const [, endSlot] = getSlotRangeForEpoch(epoch, this.l1Constants);
557
+ if (syncedSlot >= endSlot) {
558
+ return epoch;
559
+ }
560
+ return Number(epoch) > 0 ? EpochNumber(Number(epoch) - 1) : undefined;
343
561
  }
344
562
 
345
563
  public async isEpochComplete(epochNumber: EpochNumber): Promise<boolean> {
346
564
  // The epoch is complete if the current checkpointed L2 block is the last one in the epoch (or later).
347
565
  // We use the checkpointed block number (synced from L1) instead of 'latest' to avoid returning true
348
566
  // prematurely when proposed blocks have been pushed to the archiver but not yet checkpointed on L1.
349
- const checkpointedBlockNumber = await this.getCheckpointedL2BlockNumber();
350
- const header = checkpointedBlockNumber > 0 ? await this.getBlockHeader(checkpointedBlockNumber) : undefined;
567
+ const header = (await this.getBlockData({ tag: 'checkpointed' }))?.header;
351
568
  const slot = header ? header.globalVariables.slotNumber : undefined;
352
569
  const [_startSlot, endSlot] = getSlotRangeForEpoch(epochNumber, this.l1Constants);
353
570
  if (slot && slot >= endSlot) {
@@ -391,115 +608,11 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
391
608
  return true;
392
609
  }
393
610
 
394
- public async getL2Tips(): Promise<L2Tips> {
395
- const [latestBlockNumber, provenBlockNumber, checkpointedBlockNumber, finalizedBlockNumber] = await Promise.all([
396
- this.getBlockNumber(),
397
- this.getProvenBlockNumber(),
398
- this.getCheckpointedL2BlockNumber(),
399
- this.getFinalizedL2BlockNumber(),
400
- ] as const);
401
-
402
- const beforeInitialblockNumber = BlockNumber(INITIAL_L2_BLOCK_NUM - 1);
403
-
404
- // Get the latest block header and checkpointed blocks for proven, finalised and checkpointed blocks
405
- const [latestBlockHeader, provenCheckpointedBlock, finalizedCheckpointedBlock, checkpointedBlock] =
406
- await Promise.all([
407
- latestBlockNumber > beforeInitialblockNumber ? this.getBlockHeader(latestBlockNumber) : undefined,
408
- provenBlockNumber > beforeInitialblockNumber ? this.getCheckpointedBlock(provenBlockNumber) : undefined,
409
- finalizedBlockNumber > beforeInitialblockNumber ? this.getCheckpointedBlock(finalizedBlockNumber) : undefined,
410
- checkpointedBlockNumber > beforeInitialblockNumber
411
- ? this.getCheckpointedBlock(checkpointedBlockNumber)
412
- : undefined,
413
- ] as const);
414
-
415
- if (latestBlockNumber > beforeInitialblockNumber && !latestBlockHeader) {
416
- throw new Error(`Failed to retrieve latest block header for block ${latestBlockNumber}`);
417
- }
418
-
419
- // Checkpointed blocks must exist for proven, finalized and checkpointed tips if they are beyond the initial block number.
420
- if (checkpointedBlockNumber > beforeInitialblockNumber && !checkpointedBlock?.block.header) {
421
- throw new Error(
422
- `Failed to retrieve checkpointed block header for block ${checkpointedBlockNumber} (latest block is ${latestBlockNumber})`,
423
- );
424
- }
425
-
426
- if (provenBlockNumber > beforeInitialblockNumber && !provenCheckpointedBlock?.block.header) {
427
- throw new Error(
428
- `Failed to retrieve proven checkpointed for block ${provenBlockNumber} (latest block is ${latestBlockNumber})`,
429
- );
430
- }
431
-
432
- if (finalizedBlockNumber > beforeInitialblockNumber && !finalizedCheckpointedBlock?.block.header) {
433
- throw new Error(
434
- `Failed to retrieve finalized block header for block ${finalizedBlockNumber} (latest block is ${latestBlockNumber})`,
435
- );
436
- }
437
-
438
- const latestBlockHeaderHash = (await latestBlockHeader?.hash()) ?? GENESIS_BLOCK_HEADER_HASH;
439
- const provenBlockHeaderHash = (await provenCheckpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH;
440
- const finalizedBlockHeaderHash =
441
- (await finalizedCheckpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH;
442
- const checkpointedBlockHeaderHash = (await checkpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH;
443
-
444
- // Now attempt to retrieve checkpoints for proven, finalised and checkpointed blocks
445
- const [[provenBlockCheckpoint], [finalizedBlockCheckpoint], [checkpointedBlockCheckpoint]] = await Promise.all([
446
- provenCheckpointedBlock !== undefined
447
- ? await this.getCheckpoints(provenCheckpointedBlock?.checkpointNumber, 1)
448
- : [undefined],
449
- finalizedCheckpointedBlock !== undefined
450
- ? await this.getCheckpoints(finalizedCheckpointedBlock?.checkpointNumber, 1)
451
- : [undefined],
452
- checkpointedBlock !== undefined ? await this.getCheckpoints(checkpointedBlock?.checkpointNumber, 1) : [undefined],
453
- ]);
454
-
455
- const initialcheckpointId: CheckpointId = {
456
- number: CheckpointNumber.ZERO,
457
- hash: GENESIS_CHECKPOINT_HEADER_HASH.toString(),
458
- };
459
-
460
- const makeCheckpointId = (checkpoint: PublishedCheckpoint | undefined) => {
461
- if (checkpoint === undefined) {
462
- return initialcheckpointId;
463
- }
464
- return {
465
- number: checkpoint.checkpoint.number,
466
- hash: checkpoint.checkpoint.hash().toString(),
467
- };
468
- };
469
-
470
- const l2Tips: L2Tips = {
471
- proposed: {
472
- number: latestBlockNumber,
473
- hash: latestBlockHeaderHash.toString(),
474
- },
475
- proven: {
476
- block: {
477
- number: provenBlockNumber,
478
- hash: provenBlockHeaderHash.toString(),
479
- },
480
- checkpoint: makeCheckpointId(provenBlockCheckpoint),
481
- },
482
- finalized: {
483
- block: {
484
- number: finalizedBlockNumber,
485
- hash: finalizedBlockHeaderHash.toString(),
486
- },
487
- checkpoint: makeCheckpointId(finalizedBlockCheckpoint),
488
- },
489
- checkpointed: {
490
- block: {
491
- number: checkpointedBlockNumber,
492
- hash: checkpointedBlockHeaderHash.toString(),
493
- },
494
- checkpoint: makeCheckpointId(checkpointedBlockCheckpoint),
495
- },
496
- };
497
-
498
- return l2Tips;
611
+ public getL2Tips(): Promise<L2Tips> {
612
+ return this.l2TipsCache.getL2Tips();
499
613
  }
500
614
 
501
615
  public async rollbackTo(targetL2BlockNumber: BlockNumber): Promise<void> {
502
- // TODO(pw/mbps): This still assumes 1 block per checkpoint
503
616
  const currentBlocks = await this.getL2Tips();
504
617
  const currentL2Block = currentBlocks.proposed.number;
505
618
  const currentProvenBlock = currentBlocks.proven.block.number;
@@ -507,12 +620,34 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
507
620
  if (targetL2BlockNumber >= currentL2Block) {
508
621
  throw new Error(`Target L2 block ${targetL2BlockNumber} must be less than current L2 block ${currentL2Block}`);
509
622
  }
510
- const targetL2Block = await this.store.getCheckpointedBlock(targetL2BlockNumber);
511
- if (!targetL2Block) {
623
+ const checkpointedTip = await this.stores.blocks.getCheckpointedL2BlockNumber();
624
+ if (targetL2BlockNumber > checkpointedTip) {
625
+ throw new Error(`Target L2 block ${targetL2BlockNumber} is not checkpointed yet`);
626
+ }
627
+ const targetBlockData = await this.stores.blocks.getBlockData({ number: targetL2BlockNumber });
628
+ if (!targetBlockData) {
512
629
  throw new Error(`Target L2 block ${targetL2BlockNumber} not found`);
513
630
  }
514
- const targetL1BlockNumber = targetL2Block.l1.blockNumber;
515
- const targetCheckpointNumber = targetL2Block.checkpointNumber;
631
+ const targetCheckpointNumber = targetBlockData.checkpointNumber;
632
+
633
+ // Rollback operates at checkpoint granularity: the target block must be the last block of its checkpoint.
634
+ const checkpointData = await this.stores.blocks.getCheckpointData(targetCheckpointNumber);
635
+ if (!checkpointData) {
636
+ throw new Error(`Checkpoint ${targetCheckpointNumber} not found for block ${targetL2BlockNumber}`);
637
+ }
638
+ const lastBlockInCheckpoint = BlockNumber(checkpointData.startBlock + checkpointData.blockCount - 1);
639
+ if (targetL2BlockNumber !== lastBlockInCheckpoint) {
640
+ const previousCheckpointBoundary =
641
+ checkpointData.startBlock > 1 ? BlockNumber(checkpointData.startBlock - 1) : BlockNumber(0);
642
+ throw new Error(
643
+ `Target L2 block ${targetL2BlockNumber} is not at a checkpoint boundary. ` +
644
+ `Checkpoint ${targetCheckpointNumber} spans blocks ${checkpointData.startBlock} to ${lastBlockInCheckpoint}. ` +
645
+ `Use block ${lastBlockInCheckpoint} to roll back to this checkpoint, ` +
646
+ `or block ${previousCheckpointBoundary} to roll back to the previous one.`,
647
+ );
648
+ }
649
+
650
+ const targetL1BlockNumber = checkpointData.l1.blockNumber;
516
651
  const targetL1Block = await this.publicClient.getBlock({
517
652
  blockNumber: targetL1BlockNumber,
518
653
  includeTransactions: false,
@@ -526,18 +661,21 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
526
661
  );
527
662
  await this.updater.removeCheckpointsAfter(targetCheckpointNumber);
528
663
  this.log.info(`Rolling back L1 to L2 messages to checkpoint ${targetCheckpointNumber}`);
529
- await this.store.rollbackL1ToL2MessagesToCheckpoint(targetCheckpointNumber);
664
+ await this.stores.messages.rollbackL1ToL2MessagesToCheckpoint(targetCheckpointNumber);
530
665
  this.log.info(`Setting L1 syncpoints to ${targetL1BlockNumber}`);
531
- await this.store.setCheckpointSynchedL1BlockNumber(targetL1BlockNumber);
532
- await this.store.setMessageSynchedL1Block({ l1BlockNumber: targetL1BlockNumber, l1BlockHash: targetL1BlockHash });
666
+ await this.stores.blocks.setSynchedL1BlockNumber(targetL1BlockNumber);
667
+ await this.stores.messages.setMessageSyncState(
668
+ { l1BlockNumber: targetL1BlockNumber, l1BlockHash: targetL1BlockHash },
669
+ undefined,
670
+ );
533
671
  if (targetL2BlockNumber < currentProvenBlock) {
534
- this.log.info(`Clearing proven L2 block number`);
535
- await this.store.setProvenCheckpointNumber(CheckpointNumber.ZERO);
672
+ this.log.info(`Rolling back proven L2 checkpoint to ${targetCheckpointNumber}`);
673
+ await this.updater.setProvenCheckpointNumber(targetCheckpointNumber);
674
+ }
675
+ const currentFinalizedBlock = currentBlocks.finalized.block.number;
676
+ if (targetL2BlockNumber < currentFinalizedBlock) {
677
+ this.log.info(`Rolling back finalized L2 checkpoint to ${targetCheckpointNumber}`);
678
+ await this.updater.setFinalizedCheckpointNumber(targetCheckpointNumber);
536
679
  }
537
- // TODO(palla/reorg): Set the finalized block when we add support for it.
538
- // if (targetL2BlockNumber < currentFinalizedBlock) {
539
- // this.log.info(`Clearing finalized L2 block number`);
540
- // await this.store.setFinalizedL2BlockNumber(0);
541
- // }
542
680
  }
543
681
  }