@aztec/archiver 5.0.0-private.20260319 → 5.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/README.md +19 -13
  2. package/dest/archiver.d.ts +71 -15
  3. package/dest/archiver.d.ts.map +1 -1
  4. package/dest/archiver.js +268 -74
  5. package/dest/config.d.ts +7 -3
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +39 -16
  8. package/dest/errors.d.ts +44 -10
  9. package/dest/errors.d.ts.map +1 -1
  10. package/dest/errors.js +66 -12
  11. package/dest/factory.d.ts +12 -7
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +40 -25
  14. package/dest/index.d.ts +11 -3
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +10 -2
  17. package/dest/l1/calldata_retriever.d.ts +2 -1
  18. package/dest/l1/calldata_retriever.d.ts.map +1 -1
  19. package/dest/l1/calldata_retriever.js +15 -5
  20. package/dest/l1/data_retrieval.d.ts +19 -10
  21. package/dest/l1/data_retrieval.d.ts.map +1 -1
  22. package/dest/l1/data_retrieval.js +25 -32
  23. package/dest/l1/trace_tx.d.ts +12 -66
  24. package/dest/l1/trace_tx.d.ts.map +1 -1
  25. package/dest/l1/validate_historical_logs.d.ts +23 -0
  26. package/dest/l1/validate_historical_logs.d.ts.map +1 -0
  27. package/dest/l1/validate_historical_logs.js +108 -0
  28. package/dest/modules/contract_data_source_adapter.d.ts +25 -0
  29. package/dest/modules/contract_data_source_adapter.d.ts.map +1 -0
  30. package/dest/modules/contract_data_source_adapter.js +32 -0
  31. package/dest/modules/data_source_base.d.ts +72 -48
  32. package/dest/modules/data_source_base.d.ts.map +1 -1
  33. package/dest/modules/data_source_base.js +266 -143
  34. package/dest/modules/data_store_updater.d.ts +30 -13
  35. package/dest/modules/data_store_updater.d.ts.map +1 -1
  36. package/dest/modules/data_store_updater.js +172 -117
  37. package/dest/modules/instrumentation.d.ts +7 -2
  38. package/dest/modules/instrumentation.d.ts.map +1 -1
  39. package/dest/modules/instrumentation.js +25 -7
  40. package/dest/modules/l1_synchronizer.d.ts +11 -6
  41. package/dest/modules/l1_synchronizer.d.ts.map +1 -1
  42. package/dest/modules/l1_synchronizer.js +437 -217
  43. package/dest/modules/outbox_trees_resolver.d.ts +62 -0
  44. package/dest/modules/outbox_trees_resolver.d.ts.map +1 -0
  45. package/dest/modules/outbox_trees_resolver.js +162 -0
  46. package/dest/modules/validation.d.ts +24 -5
  47. package/dest/modules/validation.d.ts.map +1 -1
  48. package/dest/modules/validation.js +40 -15
  49. package/dest/store/block_store.d.ts +164 -66
  50. package/dest/store/block_store.d.ts.map +1 -1
  51. package/dest/store/block_store.js +638 -233
  52. package/dest/store/contract_class_store.d.ts +17 -4
  53. package/dest/store/contract_class_store.d.ts.map +1 -1
  54. package/dest/store/contract_class_store.js +36 -68
  55. package/dest/store/contract_instance_store.d.ts +28 -1
  56. package/dest/store/contract_instance_store.d.ts.map +1 -1
  57. package/dest/store/contract_instance_store.js +48 -2
  58. package/dest/store/data_stores.d.ts +68 -0
  59. package/dest/store/data_stores.d.ts.map +1 -0
  60. package/dest/store/data_stores.js +54 -0
  61. package/dest/store/function_names_cache.d.ts +17 -0
  62. package/dest/store/function_names_cache.d.ts.map +1 -0
  63. package/dest/store/function_names_cache.js +30 -0
  64. package/dest/store/l2_tips_cache.d.ts +13 -7
  65. package/dest/store/l2_tips_cache.d.ts.map +1 -1
  66. package/dest/store/l2_tips_cache.js +13 -76
  67. package/dest/store/log_store.d.ts +42 -40
  68. package/dest/store/log_store.d.ts.map +1 -1
  69. package/dest/store/log_store.js +257 -480
  70. package/dest/store/log_store_codec.d.ts +78 -0
  71. package/dest/store/log_store_codec.d.ts.map +1 -0
  72. package/dest/store/log_store_codec.js +110 -0
  73. package/dest/store/message_store.d.ts +9 -3
  74. package/dest/store/message_store.d.ts.map +1 -1
  75. package/dest/store/message_store.js +39 -10
  76. package/dest/test/fake_l1_state.d.ts +15 -3
  77. package/dest/test/fake_l1_state.d.ts.map +1 -1
  78. package/dest/test/fake_l1_state.js +80 -18
  79. package/dest/test/mock_l2_block_source.d.ts +48 -50
  80. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  81. package/dest/test/mock_l2_block_source.js +223 -179
  82. package/dest/test/mock_structs.d.ts +1 -1
  83. package/dest/test/mock_structs.d.ts.map +1 -1
  84. package/dest/test/mock_structs.js +2 -2
  85. package/dest/test/noop_l1_archiver.d.ts +14 -6
  86. package/dest/test/noop_l1_archiver.d.ts.map +1 -1
  87. package/dest/test/noop_l1_archiver.js +25 -9
  88. package/package.json +14 -14
  89. package/src/archiver.ts +353 -80
  90. package/src/config.ts +43 -20
  91. package/src/errors.ts +103 -18
  92. package/src/factory.ts +56 -24
  93. package/src/index.ts +18 -2
  94. package/src/l1/calldata_retriever.ts +16 -5
  95. package/src/l1/data_retrieval.ts +38 -46
  96. package/src/l1/validate_historical_logs.ts +140 -0
  97. package/src/modules/contract_data_source_adapter.ts +46 -0
  98. package/src/modules/data_source_base.ts +340 -189
  99. package/src/modules/data_store_updater.ts +211 -148
  100. package/src/modules/instrumentation.ts +28 -8
  101. package/src/modules/l1_synchronizer.ts +583 -255
  102. package/src/modules/outbox_trees_resolver.ts +199 -0
  103. package/src/modules/validation.ts +81 -21
  104. package/src/store/block_store.ts +792 -285
  105. package/src/store/contract_class_store.ts +43 -103
  106. package/src/store/contract_instance_store.ts +62 -5
  107. package/src/store/data_stores.ts +104 -0
  108. package/src/store/function_names_cache.ts +37 -0
  109. package/src/store/l2_tips_cache.ts +16 -70
  110. package/src/store/log_store.ts +296 -652
  111. package/src/store/log_store_codec.ts +143 -0
  112. package/src/store/message_store.ts +43 -12
  113. package/src/structs/inbox_message.ts +1 -1
  114. package/src/test/fake_l1_state.ts +99 -27
  115. package/src/test/mock_l2_block_source.ts +279 -233
  116. package/src/test/mock_structs.ts +5 -2
  117. package/src/test/noop_l1_archiver.ts +52 -9
  118. package/dest/store/kv_archiver_store.d.ts +0 -367
  119. package/dest/store/kv_archiver_store.d.ts.map +0 -1
  120. package/dest/store/kv_archiver_store.js +0 -481
  121. package/src/store/kv_archiver_store.ts +0 -697
package/src/archiver.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import type { BlobClientInterface } from '@aztec/blob-client/client';
2
2
  import { EpochCache } from '@aztec/epoch-cache';
3
- import { BlockTagTooOldError, RollupContract } from '@aztec/ethereum/contracts';
3
+ import { BlockTagTooOldError, OutboxContract, RollupContract } from '@aztec/ethereum/contracts';
4
4
  import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
5
5
  import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
6
6
  import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
@@ -11,32 +11,40 @@ import { EthAddress } from '@aztec/foundation/eth-address';
11
11
  import { type Logger, createLogger } from '@aztec/foundation/log';
12
12
  import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise';
13
13
  import { RunningPromise, makeLoggingErrorHandler } from '@aztec/foundation/running-promise';
14
- import { DateProvider } from '@aztec/foundation/timer';
14
+ import { DateProvider, elapsed } from '@aztec/foundation/timer';
15
15
  import {
16
16
  type ArchiverEmitter,
17
+ type BlockHash,
17
18
  L2Block,
18
19
  type L2BlockSink,
20
+ L2BlockSourceEvents,
19
21
  type L2Tips,
20
22
  type ValidateCheckpointResult,
21
23
  } from '@aztec/stdlib/block';
22
- import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
24
+ import { type ProposedCheckpointInput, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
23
25
  import {
24
26
  type L1RollupConstants,
25
27
  getEpochAtSlot,
26
28
  getSlotAtNextL1Block,
27
29
  getSlotRangeForEpoch,
30
+ getTimestampForSlot,
28
31
  getTimestampRangeForEpoch,
29
32
  } from '@aztec/stdlib/epoch-helpers';
33
+ import type { L2ToL1MembershipWitness } from '@aztec/stdlib/messaging';
34
+ import { ConsensusTimetable } from '@aztec/stdlib/timetable';
35
+ import type { BlockHeader, TxHash } from '@aztec/stdlib/tx';
30
36
  import { type TelemetryClient, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
31
37
 
32
38
  import { type ArchiverConfig, mapArchiverConfig } from './config.js';
33
- import { BlockAlreadyCheckpointedError, NoBlobBodiesFoundError } from './errors.js';
39
+ import { BlockAlreadyCheckpointedError, BlockOrCheckpointSlotExpiredError, NoBlobBodiesFoundError } from './errors.js';
40
+ import { validateAndLogHistoricalLogsAvailability } from './l1/validate_historical_logs.js';
34
41
  import { validateAndLogTraceAvailability } from './l1/validate_trace.js';
35
42
  import { ArchiverDataSourceBase } from './modules/data_source_base.js';
36
43
  import { ArchiverDataStoreUpdater } from './modules/data_store_updater.js';
37
44
  import type { ArchiverInstrumentation } from './modules/instrumentation.js';
38
45
  import type { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js';
39
- import type { KVArchiverDataStore } from './store/kv_archiver_store.js';
46
+ import { OutboxTreesResolver } from './modules/outbox_trees_resolver.js';
47
+ import { type ArchiverDataStores, backupArchiverDataStores, getArchiverSynchPoint } from './store/data_stores.js';
40
48
  import { L2TipsCache } from './store/l2_tips_cache.js';
41
49
 
42
50
  /** Export ArchiverEmitter for use in factory and tests. */
@@ -44,11 +52,20 @@ export type { ArchiverEmitter };
44
52
 
45
53
  /** Request to add a block to the archiver, queued for processing by the sync loop. */
46
54
  type AddBlockRequest = {
55
+ type: 'block';
47
56
  block: L2Block;
48
57
  resolve: () => void;
49
58
  reject: (err: Error) => void;
50
59
  };
51
60
 
61
+ /** Request to add a proposed checkpoint to the archiver, queued for processing by the sync loop. */
62
+ type AddProposedCheckpointRequest = {
63
+ type: 'checkpoint';
64
+ checkpoint: ProposedCheckpointInput;
65
+ resolve: () => void;
66
+ reject: (err: Error) => void;
67
+ };
68
+
52
69
  export type ArchiverDeps = {
53
70
  telemetry?: TelemetryClient;
54
71
  blobClient: BlobClientInterface;
@@ -56,6 +73,16 @@ export type ArchiverDeps = {
56
73
  dateProvider?: DateProvider;
57
74
  };
58
75
 
76
+ /** Minimal dependency for observing whether a checkpoint proposal was received for a slot. */
77
+ export type CheckpointProposalPresence = {
78
+ /** Returns true when a checkpoint proposal for `slot` has been retained locally. */
79
+ hasCheckpointProposalForSlot(slot: SlotNumber): Promise<boolean>;
80
+ };
81
+
82
+ const noCheckpointProposalPresence: CheckpointProposalPresence = {
83
+ hasCheckpointProposalForSlot: () => Promise.resolve(false),
84
+ };
85
+
59
86
  /**
60
87
  * Pulls checkpoints in a non-blocking manner and provides interface for their retrieval.
61
88
  * Responsible for handling robust L1 polling so that other components do not need to
@@ -71,11 +98,14 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
71
98
  /** L1 synchronizer that handles fetching checkpoints and messages from L1. */
72
99
  private readonly synchronizer: ArchiverL1Synchronizer;
73
100
 
101
+ /** Resolver for L2-to-L1 message membership witnesses, built over this archiver's read view. */
102
+ private readonly outboxTreesResolver: OutboxTreesResolver;
103
+
74
104
  private initialSyncComplete: boolean = false;
75
105
  private initialSyncPromise: PromiseWithResolvers<void>;
76
106
 
77
- /** Queue of blocks to be added to the store, processed by the sync loop. */
78
- private blockQueue: AddBlockRequest[] = [];
107
+ /** Queue of blocks and checkpoints to be added to the store, processed by the sync loop. */
108
+ private inboundQueue: (AddBlockRequest | AddProposedCheckpointRequest)[] = [];
79
109
 
80
110
  /** Helper to handle updates to the store */
81
111
  private readonly updater: ArchiverDataStoreUpdater;
@@ -83,38 +113,57 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
83
113
  /** In-memory cache for L2 chain tips. */
84
114
  private readonly l2TipsCache: L2TipsCache;
85
115
 
116
+ /** Consensus timing model used for proposed-checkpoint arrival expectations. */
117
+ private readonly timetable: ConsensusTimetable;
118
+
86
119
  public readonly tracer: Tracer;
87
120
 
121
+ private readonly instrumentation: ArchiverInstrumentation;
122
+
88
123
  /**
89
124
  * Creates a new instance of the Archiver.
90
125
  * @param publicClient - A client for interacting with the Ethereum node.
91
126
  * @param debugClient - A client for interacting with the Ethereum node for debug/trace methods.
92
127
  * @param rollup - Rollup contract instance.
128
+ * @param outbox - Outbox contract instance, used to read per-epoch L2-to-L1 message roots.
93
129
  * @param inbox - Inbox contract instance.
94
- * @param l1Addresses - L1 contract addresses (registry, governance proposer, slash factory, slashing proposer).
95
- * @param dataStore - An archiver data store for storage & retrieval of blocks, encrypted logs & contract data.
130
+ * @param l1Addresses - L1 contract addresses (registry, governance proposer, slashing proposer).
131
+ * @param dataStores - Archiver substores for storage & retrieval of blocks, encrypted logs & contract data.
96
132
  * @param config - Archiver configuration options.
97
133
  * @param blobClient - Client for retrieving blob data.
98
- * @param dateProvider - Provider for current date/time.
99
134
  * @param instrumentation - Instrumentation for metrics and tracing.
100
135
  * @param l1Constants - L1 rollup constants.
136
+ * @param synchronizer - L1 synchronizer that handles fetching checkpoints and messages from L1.
137
+ * @param events - Event emitter shared with the synchronizer.
138
+ * @param initialHeader - Genesis block header.
139
+ * @param initialBlockHash - Precomputed hash of the genesis block header.
140
+ * @param l2TipsCache - In-memory cache for L2 chain tips.
141
+ * @param dateProvider - Provider for current date/time, used for wall-clock orphan-block pruning.
101
142
  * @param log - A logger.
102
143
  */
103
144
  constructor(
104
145
  private readonly publicClient: ViemPublicClient,
105
146
  private readonly debugClient: ViemPublicDebugClient,
106
147
  private readonly rollup: RollupContract,
148
+ private readonly outbox: OutboxContract,
107
149
  private readonly l1Addresses: Pick<
108
150
  L1ContractAddresses,
109
- 'registryAddress' | 'governanceProposerAddress' | 'slashFactoryAddress'
110
- > & { slashingProposerAddress: EthAddress },
111
- readonly dataStore: KVArchiverDataStore,
151
+ 'rollupAddress' | 'registryAddress' | 'inboxAddress' | 'governanceProposerAddress'
152
+ > & {
153
+ slashingProposerAddress: EthAddress;
154
+ },
155
+ readonly dataStores: ArchiverDataStores,
112
156
  private config: {
113
157
  pollingIntervalMs: number;
114
158
  batchSize: number;
115
159
  skipValidateCheckpointAttestations?: boolean;
116
160
  maxAllowedEthClientDriftSeconds: number;
117
161
  ethereumAllowNoDebugHosts?: boolean;
162
+ skipHistoricalLogsCheck?: boolean;
163
+ checkpointProposalSyncGrace: number;
164
+ orphanPruneNoProposalTolerance: number;
165
+ skipOrphanProposedBlockPruning: boolean;
166
+ blockDuration: number;
118
167
  },
119
168
  private readonly blobClient: BlobClientInterface,
120
169
  instrumentation: ArchiverInstrumentation,
@@ -124,20 +173,39 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
124
173
  },
125
174
  synchronizer: ArchiverL1Synchronizer,
126
175
  events: ArchiverEmitter,
127
- l2TipsCache?: L2TipsCache,
176
+ initialHeader: BlockHeader,
177
+ initialBlockHash: BlockHash,
178
+ l2TipsCache: L2TipsCache,
179
+ private readonly dateProvider: DateProvider,
180
+ private checkpointProposalPresence: CheckpointProposalPresence = noCheckpointProposalPresence,
128
181
  private readonly log: Logger = createLogger('archiver'),
129
182
  ) {
130
- super(dataStore, l1Constants);
183
+ super(dataStores, l1Constants, initialHeader, initialBlockHash, l1Constants.genesisArchiveRoot);
131
184
 
132
185
  this.tracer = instrumentation.tracer;
186
+ this.instrumentation = instrumentation;
133
187
  this.initialSyncPromise = promiseWithResolvers();
134
188
  this.synchronizer = synchronizer;
135
189
  this.events = events;
136
- this.l2TipsCache = l2TipsCache ?? new L2TipsCache(this.dataStore.blockStore);
137
- this.updater = new ArchiverDataStoreUpdater(this.dataStore, this.l2TipsCache, {
190
+ this.l2TipsCache = l2TipsCache;
191
+ this.timetable = new ConsensusTimetable({
192
+ l1Constants,
193
+ blockDuration: this.config.blockDuration,
194
+ checkpointProposalSyncGrace: this.config.checkpointProposalSyncGrace,
195
+ });
196
+ this.updater = new ArchiverDataStoreUpdater(this.dataStores, this.l2TipsCache, {
138
197
  rollupManaLimit: l1Constants.rollupManaLimit,
139
198
  });
140
199
 
200
+ // The resolver reads its witness data from this archiver and pins its lazy Outbox-root fetches
201
+ // to the archiver's synced L1 block. No method on `this` is invoked during construction.
202
+ this.outboxTreesResolver = new OutboxTreesResolver(
203
+ this.outbox,
204
+ this,
205
+ () => Promise.resolve(this.getL1BlockNumber()),
206
+ l1Constants.epochDuration,
207
+ );
208
+
141
209
  // Running promise starts with a small interval inbetween runs, so all iterations needed for the initial sync
142
210
  // are done as fast as possible. This then gets updated once the initial sync completes.
143
211
  this.runningPromise = new RunningPromise(
@@ -148,6 +216,27 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
148
216
  );
149
217
  }
150
218
 
219
+ /**
220
+ * Returns the L2-to-L1 membership witness for `message` emitted by tx `txHash`. The node selects
221
+ * the smallest partial-proof root on the Outbox that covers the tx's checkpoint and builds the
222
+ * witness against it.
223
+ *
224
+ * The node reads the Outbox roots lazily, pinned to its synced L1 block, so the witness reflects
225
+ * the node's synced view. Returns `undefined` if the tx isn't yet in a block/epoch or no covering
226
+ * root has landed on L1 as of the synced block.
227
+ *
228
+ * Caveat: roots are cached only for the current synced L1 block and re-fetched once the node
229
+ * advances, so a reorg is picked up on the next synced-block advance rather than re-validated per
230
+ * request.
231
+ */
232
+ public getL2ToL1MembershipWitness(
233
+ txHash: TxHash,
234
+ message: Fr,
235
+ messageIndexInTx?: number,
236
+ ): Promise<L2ToL1MembershipWitness | undefined> {
237
+ return this.outboxTreesResolver.getL2ToL1MembershipWitness(txHash, message, messageIndexInTx);
238
+ }
239
+
151
240
  /** Updates archiver config */
152
241
  public updateConfig(newConfig: Partial<ArchiverConfig>) {
153
242
  this.config = merge(this.config, mapArchiverConfig(newConfig));
@@ -170,11 +259,24 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
170
259
  this.config.ethereumAllowNoDebugHosts ?? false,
171
260
  this.log.getBindings(),
172
261
  );
262
+ await validateAndLogHistoricalLogsAvailability(
263
+ this.publicClient,
264
+ {
265
+ rollupAddress: this.l1Addresses.rollupAddress,
266
+ inboxAddress: this.l1Addresses.inboxAddress,
267
+ registryAddress: this.l1Addresses.registryAddress,
268
+ governanceProposerAddress: this.l1Addresses.governanceProposerAddress,
269
+ },
270
+ this.config.skipHistoricalLogsCheck ?? false,
271
+ this.log.getBindings(),
272
+ );
173
273
 
174
274
  // Log initial state for the archiver
175
275
  const { l1StartBlock } = this.l1Constants;
176
- const { blocksSynchedTo = l1StartBlock, messagesSynchedTo = l1StartBlock } = await this.store.getSynchPoint();
177
- const currentL2Checkpoint = await this.getSynchedCheckpointNumber();
276
+ const { blocksSynchedTo = l1StartBlock, messagesSynchedTo = l1StartBlock } = await getArchiverSynchPoint(
277
+ this.stores,
278
+ );
279
+ const currentL2Checkpoint = await this.getCheckpointNumber();
178
280
  this.log.info(
179
281
  `Starting archiver sync to rollup contract ${this.rollup.address} from L1 block ${blocksSynchedTo} and L2 checkpoint ${currentL2Checkpoint}`,
180
282
  { blocksSynchedTo, messagesSynchedTo, currentL2Checkpoint },
@@ -191,6 +293,19 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
191
293
  return this.runningPromise.trigger();
192
294
  }
193
295
 
296
+ /** Sets the proposal-presence provider used by orphan proposed-block pruning. */
297
+ public setCheckpointProposalPresence(checkpointProposalPresence: CheckpointProposalPresence): void {
298
+ this.checkpointProposalPresence = checkpointProposalPresence;
299
+ }
300
+
301
+ public trySyncImmediate() {
302
+ try {
303
+ return this.syncImmediate();
304
+ } catch (err) {
305
+ this.log.error(`Failed to trigger immediate archiver sync: ${err}`, err);
306
+ }
307
+ }
308
+
194
309
  /**
195
310
  * Queues a block to be added to the archiver store and triggers processing.
196
311
  * The block will be processed by the sync loop.
@@ -199,58 +314,85 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
199
314
  * @returns A promise that resolves when the block has been added to the store, or rejects on error.
200
315
  */
201
316
  public addBlock(block: L2Block): Promise<void> {
202
- return new Promise<void>((resolve, reject) => {
203
- this.blockQueue.push({ block, resolve, reject });
204
- this.log.debug(`Queued block ${block.number} for processing`);
205
- // Trigger an immediate sync, but don't wait for it - the promise resolves when the block is processed
206
- this.syncImmediate().catch(err => {
207
- this.log.error(`Sync immediate call failed: ${err}`);
208
- });
209
- });
317
+ const promise = promiseWithResolvers<void>();
318
+ this.inboundQueue.push({ block, ...promise, type: 'block' });
319
+ this.log.debug(`Queued block ${block.number} for processing`);
320
+ void this.trySyncImmediate();
321
+ return promise.promise;
322
+ }
323
+
324
+ /**
325
+ * Queues a new proposed checkpoint into the archiver store.
326
+ * Checks that the checkpoint is not for an L2 slot already synced from L1.
327
+ * Resolves once the checkpoint has been processed.
328
+ */
329
+ public addProposedCheckpoint(pending: ProposedCheckpointInput): Promise<void> {
330
+ const promise = promiseWithResolvers<void>();
331
+ this.inboundQueue.push({ checkpoint: pending, ...promise, type: 'checkpoint' });
332
+ this.log.debug(`Queued checkpoint ${pending.checkpointNumber} for processing`);
333
+ void this.trySyncImmediate();
334
+ return promise.promise;
210
335
  }
211
336
 
212
337
  /**
213
- * Processes all queued blocks, adding them to the store.
338
+ * Processes all queued blocks and checkpoints, adding them to the store.
214
339
  * Called at the beginning of each sync iteration.
215
- * Blocks are processed in the order they were queued.
340
+ * Items are processed in the order they were queued.
216
341
  */
217
- private async processQueuedBlocks(): Promise<void> {
218
- if (this.blockQueue.length === 0) {
342
+ private async processInboundQueue(): Promise<void> {
343
+ if (this.inboundQueue.length === 0) {
219
344
  return;
220
345
  }
221
346
 
222
- // Take all blocks from the queue
223
- const queuedItems = this.blockQueue.splice(0, this.blockQueue.length);
224
- this.log.debug(`Processing ${queuedItems.length} queued block(s)`);
347
+ // Take all items from the queue
348
+ const queuedItems = this.inboundQueue.splice(0, this.inboundQueue.length);
349
+ this.log.debug(`Processing ${queuedItems.length} queued inbound items`);
225
350
 
226
351
  // Calculate slot threshold for validation
227
352
  const l1Timestamp = this.synchronizer.getL1Timestamp();
228
353
  const slotAtNextL1Block =
229
354
  l1Timestamp === undefined ? undefined : getSlotAtNextL1Block(l1Timestamp, this.l1Constants);
230
355
 
231
- // Process each block individually to properly resolve/reject each promise
232
- for (const { block, resolve, reject } of queuedItems) {
233
- const blockSlot = block.header.globalVariables.slotNumber;
234
- if (slotAtNextL1Block !== undefined && blockSlot < slotAtNextL1Block) {
356
+ // Helpers for manipulating blocks and checkpoints in the queue
357
+ const getSlot: (item: AddBlockRequest | AddProposedCheckpointRequest) => SlotNumber = item =>
358
+ item.type === 'block' ? item.block.header.globalVariables.slotNumber : item.checkpoint.header.slotNumber;
359
+ const getNumber: (item: AddBlockRequest | AddProposedCheckpointRequest) => number = item =>
360
+ item.type === 'block' ? item.block.number : item.checkpoint.checkpointNumber;
361
+
362
+ // Process each item individually to properly resolve/reject each promise
363
+ for (const item of queuedItems) {
364
+ const { resolve, reject, type } = item;
365
+ const itemSlot = getSlot(item);
366
+ const itemNumber = getNumber(item);
367
+ if (slotAtNextL1Block !== undefined && itemSlot < slotAtNextL1Block) {
368
+ const nextSlotTimestamp = getTimestampForSlot(slotAtNextL1Block, this.l1Constants);
235
369
  this.log.warn(
236
- `Rejecting proposed block ${block.number} for past slot ${blockSlot} (current is ${slotAtNextL1Block})`,
237
- { block: block.toBlockInfo(), l1Timestamp, slotAtNextL1Block },
370
+ `Rejecting proposed ${type} ${itemNumber} for past slot ${itemSlot} (current ${slotAtNextL1Block})`,
371
+ { number: itemNumber, type, l1Timestamp, slotAtNextL1Block, nextSlotTimestamp },
238
372
  );
239
- reject(new Error(`Block ${block.number} is for past slot ${blockSlot} (current is ${slotAtNextL1Block})`));
373
+ reject(new BlockOrCheckpointSlotExpiredError(itemSlot, nextSlotTimestamp, l1Timestamp));
240
374
  continue;
241
375
  }
242
376
 
243
377
  try {
244
- await this.updater.addProposedBlock(block);
245
- this.log.debug(`Added block ${block.number} to store`);
378
+ if (type === 'block') {
379
+ const [durationMs] = await elapsed(() => this.updater.addProposedBlock(item.block));
380
+ this.instrumentation.processNewProposedBlock(durationMs, item.block);
381
+ } else {
382
+ await this.updater.addProposedCheckpoint(item.checkpoint);
383
+ }
384
+ this.log.debug(`Added ${type} ${itemNumber} to store`);
246
385
  resolve();
247
386
  } catch (err: any) {
248
387
  if (err instanceof BlockAlreadyCheckpointedError) {
249
- this.log.debug(`Proposed block ${block.number} matches already checkpointed block, ignoring late proposal`);
388
+ this.log.debug(`Proposed block ${itemNumber} matches already checkpointed block, ignoring late proposal`);
250
389
  resolve();
251
390
  continue;
252
391
  }
253
- this.log.error(`Failed to add block ${block.number} to store: ${err.message}`);
392
+ this.log.error(`Failed to add ${type} ${itemNumber} to store: ${err.message}`, err, {
393
+ number: itemNumber,
394
+ type,
395
+ });
254
396
  reject(err);
255
397
  }
256
398
  }
@@ -266,9 +408,110 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
266
408
  @trackSpan('Archiver.sync')
267
409
  private async sync() {
268
410
  // Process any queued blocks first, before doing L1 sync
269
- await this.processQueuedBlocks();
411
+ await this.processInboundQueue();
270
412
  // Now perform L1 sync
271
413
  await this.syncFromL1();
414
+ // Prune proposed blocks with no corresponding proposed checkpoint after the appropriate materialization deadline.
415
+ await this.pruneOrphanProposedBlocks();
416
+ }
417
+
418
+ /**
419
+ * Prunes a block-only local tip that was built atop a checkpoint that was never itself proposed.
420
+ *
421
+ * Under pipelining, a proposer publishes the blocks for a checkpoint (block-only proposals) before
422
+ * assembling and publishing the enclosing proposed checkpoint at the end of the build slot. A node
423
+ * that received those blocks but never the proposed checkpoint is left with an orphan tip it must not build on.
424
+ * If no checkpoint proposal was received for the orphan slot, we prune after the receive deadline plus local
425
+ * tolerance. If a checkpoint proposal was received but has not materialized into proposed archiver state,
426
+ * we prune after the consensus materialization deadline.
427
+ *
428
+ * The uncheckpointed suffix is scanned in order. Blocks covered by proposed checkpoints are left in
429
+ * place; the first block not covered by a proposed checkpoint starts the orphan suffix to prune.
430
+ */
431
+ private async pruneOrphanProposedBlocks(): Promise<void> {
432
+ if (this.config.skipOrphanProposedBlockPruning) {
433
+ return;
434
+ }
435
+ const tips = await this.getL2Tips();
436
+ const now = BigInt(this.dateProvider.nowInSeconds());
437
+
438
+ // Frontier block covered by a proposed (or, falling back, confirmed) checkpoint. Blocks beyond it
439
+ // have no enclosing checkpoint proposal and are the orphan-pruning candidates.
440
+ const proposedCheckpointBlockNumber = await this.stores.blocks.getProposedCheckpointL2BlockNumber();
441
+
442
+ // The proposed tip is a proposed-checkpointed block, so there are no orphan proposed blocks to prune
443
+ if (proposedCheckpointBlockNumber === tips.proposed.number) {
444
+ this.log.trace(`No orphan proposed blocks to prune: proposed tip ${tips.proposed.number} is checkpointed`, {
445
+ proposed: tips.proposed,
446
+ proposedCheckpointBlockNumber,
447
+ });
448
+ return;
449
+ }
450
+
451
+ // Load the blocks that are candidates for pruning (ie blocks without a proposed checkpoint covering them)
452
+ const blocksWithoutProposedCheckpoint = await this.stores.blocks.getBlocksData({
453
+ from: BlockNumber(proposedCheckpointBlockNumber + 1),
454
+ limit: tips.proposed.number - proposedCheckpointBlockNumber,
455
+ });
456
+
457
+ // Iterate through them in order, the first one with a slot that should have received a proposed checkpoint
458
+ // is the first orphan block, and all blocks after it are also orphaned and should be pruned.
459
+ let lastSlotChecked = undefined;
460
+ for (const blockData of blocksWithoutProposedCheckpoint) {
461
+ // No need to recheck if this block had the same slot as the previous one.
462
+ const blockSlot = blockData.header.getSlot();
463
+ const blockNumber = blockData.header.getBlockNumber();
464
+ if (lastSlotChecked !== undefined && blockSlot === lastSlotChecked) {
465
+ continue;
466
+ }
467
+ lastSlotChecked = blockSlot;
468
+
469
+ const hasCheckpointProposal = await this.checkpointProposalPresence.hasCheckpointProposalForSlot(blockSlot);
470
+ const deadlineType = hasCheckpointProposal ? 'checkpoint-proposal-synced' : 'checkpoint-proposal-received';
471
+ const selectedDeadline = BigInt(
472
+ hasCheckpointProposal
473
+ ? this.timetable.getCheckpointProposalSyncedDeadline(blockSlot)
474
+ : Math.ceil(
475
+ this.timetable.getCheckpointProposalReceiveDeadline(blockSlot) +
476
+ this.config.orphanPruneNoProposalTolerance,
477
+ ),
478
+ );
479
+
480
+ // If it's still not checkpointed once strictly past the selected deadline, prune it along with all blocks after
481
+ // it. A proposal may still legitimately arrive or materialize at exactly its deadline, so the tip is only
482
+ // orphaned once that instant has fully elapsed.
483
+ if (now > selectedDeadline) {
484
+ const pruneAfterBlockNumber = BlockNumber(blockNumber - 1);
485
+ this.log.warn(
486
+ `Pruning orphan blocks after block ${pruneAfterBlockNumber}: block at slot ${blockSlot} belongs to ` +
487
+ `checkpoint ${blockData.checkpointNumber} which has no matching proposed checkpoint`,
488
+ {
489
+ firstUncheckpointedBlockHeader: blockData.header.toInspect(),
490
+ blockCheckpointNumber: blockData.checkpointNumber,
491
+ blockNumber,
492
+ blockSlot,
493
+ hasCheckpointProposal,
494
+ deadlineType,
495
+ selectedDeadline,
496
+ now,
497
+ },
498
+ );
499
+
500
+ const prunedBlocks = await this.updater.removeBlocksWithoutProposedCheckpointAfter(pruneAfterBlockNumber);
501
+ if (prunedBlocks.length > 0) {
502
+ this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
503
+ type: L2BlockSourceEvents.L2PruneUncheckpointed,
504
+ slotNumber: blockSlot,
505
+ blocks: prunedBlocks,
506
+ });
507
+ }
508
+ return;
509
+ }
510
+ }
511
+
512
+ this.log.trace('No orphan proposed blocks to prune: all uncheckpointed blocks are still within deadline', {
513
+ blocksWithoutProposedCheckpoint: blocksWithoutProposedCheckpoint.map(b => b.header.toInspect()),
514
+ });
272
515
  }
273
516
 
274
517
  private async syncFromL1() {
@@ -282,7 +525,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
282
525
  if (currentL1BlockNumber + 1n >= l1BlockNumberAtEnd) {
283
526
  this.log.info(`Initial archiver sync to L1 block ${currentL1BlockNumber} complete`, {
284
527
  l1BlockNumber: currentL1BlockNumber,
285
- syncPoint: await this.store.getSynchPoint(),
528
+ syncPoint: await getArchiverSynchPoint(this.stores),
286
529
  ...(await this.getL2Tips()),
287
530
  });
288
531
  this.runningPromise.setPollingIntervalMS(this.config.pollingIntervalMs);
@@ -314,7 +557,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
314
557
  }
315
558
 
316
559
  public backupTo(destPath: string): Promise<string> {
317
- return this.dataStore.backupTo(destPath);
560
+ return backupArchiverDataStores(this.dataStores, destPath);
318
561
  }
319
562
 
320
563
  public getL1Constants(): Promise<L1RollupConstants> {
@@ -341,19 +584,33 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
341
584
  return Promise.resolve(this.synchronizer.getL1Timestamp());
342
585
  }
343
586
 
344
- public getSyncedL2SlotNumber(): Promise<SlotNumber | undefined> {
587
+ public async getSyncedL2SlotNumber(): Promise<SlotNumber | undefined> {
588
+ // The synced L2 slot is the latest slot for which we have all L1 data,
589
+ // either because we have seen all L1 blocks for that slot, or because
590
+ // we have seen the corresponding checkpoint.
591
+
592
+ let slotFromL1Sync: SlotNumber | undefined;
345
593
  const l1Timestamp = this.synchronizer.getL1Timestamp();
346
- if (l1Timestamp === undefined) {
347
- return Promise.resolve(undefined);
594
+ if (l1Timestamp !== undefined) {
595
+ const nextL1BlockSlot = getSlotAtNextL1Block(l1Timestamp, this.l1Constants);
596
+ if (Number(nextL1BlockSlot) > 0) {
597
+ slotFromL1Sync = SlotNumber.add(nextL1BlockSlot, -1);
598
+ }
348
599
  }
349
- // The synced slot is the last L2 slot whose all L1 blocks have been processed.
350
- // If the next L1 block (at l1Timestamp + ethereumSlotDuration) falls in slot N,
351
- // then we've fully synced slot N-1.
352
- const nextL1BlockSlot = getSlotAtNextL1Block(l1Timestamp, this.l1Constants);
353
- if (Number(nextL1BlockSlot) === 0) {
354
- return Promise.resolve(undefined);
600
+
601
+ let slotFromCheckpoint: SlotNumber | undefined;
602
+ const latestCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
603
+ if (latestCheckpointNumber > 0) {
604
+ const checkpointData = await this.stores.blocks.getCheckpointData(latestCheckpointNumber);
605
+ if (checkpointData) {
606
+ slotFromCheckpoint = checkpointData.header.slotNumber;
607
+ }
608
+ }
609
+
610
+ if (slotFromL1Sync === undefined && slotFromCheckpoint === undefined) {
611
+ return undefined;
355
612
  }
356
- return Promise.resolve(SlotNumber(nextL1BlockSlot - 1));
613
+ return SlotNumber(Math.max(slotFromL1Sync ?? 0, slotFromCheckpoint ?? 0));
357
614
  }
358
615
 
359
616
  public async getSyncedL2EpochNumber(): Promise<EpochNumber | undefined> {
@@ -376,8 +633,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
376
633
  // The epoch is complete if the current checkpointed L2 block is the last one in the epoch (or later).
377
634
  // We use the checkpointed block number (synced from L1) instead of 'latest' to avoid returning true
378
635
  // prematurely when proposed blocks have been pushed to the archiver but not yet checkpointed on L1.
379
- const checkpointedBlockNumber = await this.getCheckpointedL2BlockNumber();
380
- const header = checkpointedBlockNumber > 0 ? await this.getBlockHeader(checkpointedBlockNumber) : undefined;
636
+ const header = (await this.getBlockData({ tag: 'checkpointed' }))?.header;
381
637
  const slot = header ? header.globalVariables.slotNumber : undefined;
382
638
  const [_startSlot, endSlot] = getSlotRangeForEpoch(epochNumber, this.l1Constants);
383
639
  if (slot && slot >= endSlot) {
@@ -412,6 +668,15 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
412
668
  return this.updater.removeCheckpointsAfter(checkpointNumber);
413
669
  }
414
670
 
671
+ /**
672
+ * Removes all uncheckpointed blocks strictly after `blockNumber`, along with the proposed checkpoints
673
+ * that referenced them. Used by the AutomineSequencer to undo a local insert whose propose tx failed
674
+ * to land on L1 (no reorg needed — nothing reached L1). Refuses to touch checkpointed blocks.
675
+ */
676
+ public removeUncheckpointedBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
677
+ return this.updater.removeUncheckpointedBlocksAfter(blockNumber);
678
+ }
679
+
415
680
  /** Used by TXE to add checkpoints directly without syncing from L1. */
416
681
  public async addCheckpoints(
417
682
  checkpoints: PublishedCheckpoint[],
@@ -433,29 +698,34 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
433
698
  if (targetL2BlockNumber >= currentL2Block) {
434
699
  throw new Error(`Target L2 block ${targetL2BlockNumber} must be less than current L2 block ${currentL2Block}`);
435
700
  }
436
- const targetL2Block = await this.store.getCheckpointedBlock(targetL2BlockNumber);
437
- if (!targetL2Block) {
701
+ const checkpointedTip = await this.stores.blocks.getCheckpointedL2BlockNumber();
702
+ if (targetL2BlockNumber > checkpointedTip) {
703
+ throw new Error(`Target L2 block ${targetL2BlockNumber} is not checkpointed yet`);
704
+ }
705
+ const targetBlockData = await this.stores.blocks.getBlockData({ number: targetL2BlockNumber });
706
+ if (!targetBlockData) {
438
707
  throw new Error(`Target L2 block ${targetL2BlockNumber} not found`);
439
708
  }
440
- const targetCheckpointNumber = targetL2Block.checkpointNumber;
709
+ const targetCheckpointNumber = targetBlockData.checkpointNumber;
441
710
 
442
711
  // Rollback operates at checkpoint granularity: the target block must be the last block of its checkpoint.
443
- const checkpointData = await this.store.getCheckpointData(targetCheckpointNumber);
444
- if (checkpointData) {
445
- const lastBlockInCheckpoint = BlockNumber(checkpointData.startBlock + checkpointData.blockCount - 1);
446
- if (targetL2BlockNumber !== lastBlockInCheckpoint) {
447
- const previousCheckpointBoundary =
448
- checkpointData.startBlock > 1 ? BlockNumber(checkpointData.startBlock - 1) : BlockNumber(0);
449
- throw new Error(
450
- `Target L2 block ${targetL2BlockNumber} is not at a checkpoint boundary. ` +
451
- `Checkpoint ${targetCheckpointNumber} spans blocks ${checkpointData.startBlock} to ${lastBlockInCheckpoint}. ` +
452
- `Use block ${lastBlockInCheckpoint} to roll back to this checkpoint, ` +
453
- `or block ${previousCheckpointBoundary} to roll back to the previous one.`,
454
- );
455
- }
712
+ const checkpointData = await this.stores.blocks.getCheckpointData(targetCheckpointNumber);
713
+ if (!checkpointData) {
714
+ throw new Error(`Checkpoint ${targetCheckpointNumber} not found for block ${targetL2BlockNumber}`);
715
+ }
716
+ const lastBlockInCheckpoint = BlockNumber(checkpointData.startBlock + checkpointData.blockCount - 1);
717
+ if (targetL2BlockNumber !== lastBlockInCheckpoint) {
718
+ const previousCheckpointBoundary =
719
+ checkpointData.startBlock > 1 ? BlockNumber(checkpointData.startBlock - 1) : BlockNumber(0);
720
+ throw new Error(
721
+ `Target L2 block ${targetL2BlockNumber} is not at a checkpoint boundary. ` +
722
+ `Checkpoint ${targetCheckpointNumber} spans blocks ${checkpointData.startBlock} to ${lastBlockInCheckpoint}. ` +
723
+ `Use block ${lastBlockInCheckpoint} to roll back to this checkpoint, ` +
724
+ `or block ${previousCheckpointBoundary} to roll back to the previous one.`,
725
+ );
456
726
  }
457
727
 
458
- const targetL1BlockNumber = targetL2Block.l1.blockNumber;
728
+ const targetL1BlockNumber = checkpointData.l1.blockNumber;
459
729
  const targetL1Block = await this.publicClient.getBlock({
460
730
  blockNumber: targetL1BlockNumber,
461
731
  includeTransactions: false,
@@ -469,10 +739,13 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
469
739
  );
470
740
  await this.updater.removeCheckpointsAfter(targetCheckpointNumber);
471
741
  this.log.info(`Rolling back L1 to L2 messages to checkpoint ${targetCheckpointNumber}`);
472
- await this.store.rollbackL1ToL2MessagesToCheckpoint(targetCheckpointNumber);
742
+ await this.stores.messages.rollbackL1ToL2MessagesToCheckpoint(targetCheckpointNumber);
473
743
  this.log.info(`Setting L1 syncpoints to ${targetL1BlockNumber}`);
474
- await this.store.setCheckpointSynchedL1BlockNumber(targetL1BlockNumber);
475
- await this.store.setMessageSynchedL1Block({ l1BlockNumber: targetL1BlockNumber, l1BlockHash: targetL1BlockHash });
744
+ await this.stores.blocks.setSynchedL1BlockNumber(targetL1BlockNumber);
745
+ await this.stores.messages.setMessageSyncState(
746
+ { l1BlockNumber: targetL1BlockNumber, l1BlockHash: targetL1BlockHash },
747
+ undefined,
748
+ );
476
749
  if (targetL2BlockNumber < currentProvenBlock) {
477
750
  this.log.info(`Rolling back proven L2 checkpoint to ${targetCheckpointNumber}`);
478
751
  await this.updater.setProvenCheckpointNumber(targetCheckpointNumber);