@aztec/archiver 0.0.1-commit.2448fdb → 0.0.1-commit.2606882

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/README.md +19 -11
  2. package/dest/archiver.d.ts +53 -16
  3. package/dest/archiver.d.ts.map +1 -1
  4. package/dest/archiver.js +229 -68
  5. package/dest/config.d.ts +4 -1
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +21 -14
  8. package/dest/errors.d.ts +50 -2
  9. package/dest/errors.d.ts.map +1 -1
  10. package/dest/errors.js +74 -2
  11. package/dest/factory.d.ts +13 -7
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +33 -24
  14. package/dest/index.d.ts +11 -3
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +10 -2
  17. package/dest/l1/calldata_retriever.d.ts +2 -1
  18. package/dest/l1/calldata_retriever.d.ts.map +1 -1
  19. package/dest/l1/calldata_retriever.js +15 -5
  20. package/dest/l1/data_retrieval.d.ts +24 -12
  21. package/dest/l1/data_retrieval.d.ts.map +1 -1
  22. package/dest/l1/data_retrieval.js +36 -37
  23. package/dest/l1/trace_tx.d.ts +12 -66
  24. package/dest/l1/trace_tx.d.ts.map +1 -1
  25. package/dest/l1/validate_historical_logs.d.ts +23 -0
  26. package/dest/l1/validate_historical_logs.d.ts.map +1 -0
  27. package/dest/l1/validate_historical_logs.js +108 -0
  28. package/dest/modules/contract_data_source_adapter.d.ts +25 -0
  29. package/dest/modules/contract_data_source_adapter.d.ts.map +1 -0
  30. package/dest/modules/contract_data_source_adapter.js +40 -0
  31. package/dest/modules/data_source_base.d.ts +71 -47
  32. package/dest/modules/data_source_base.d.ts.map +1 -1
  33. package/dest/modules/data_source_base.js +269 -137
  34. package/dest/modules/data_store_updater.d.ts +30 -13
  35. package/dest/modules/data_store_updater.d.ts.map +1 -1
  36. package/dest/modules/data_store_updater.js +126 -109
  37. package/dest/modules/instrumentation.d.ts +18 -2
  38. package/dest/modules/instrumentation.d.ts.map +1 -1
  39. package/dest/modules/instrumentation.js +35 -7
  40. package/dest/modules/l1_synchronizer.d.ts +11 -6
  41. package/dest/modules/l1_synchronizer.d.ts.map +1 -1
  42. package/dest/modules/l1_synchronizer.js +418 -216
  43. package/dest/modules/outbox_trees_resolver.d.ts +62 -0
  44. package/dest/modules/outbox_trees_resolver.d.ts.map +1 -0
  45. package/dest/modules/outbox_trees_resolver.js +162 -0
  46. package/dest/modules/validation.d.ts +4 -3
  47. package/dest/modules/validation.d.ts.map +1 -1
  48. package/dest/modules/validation.js +6 -6
  49. package/dest/store/block_store.d.ts +164 -66
  50. package/dest/store/block_store.d.ts.map +1 -1
  51. package/dest/store/block_store.js +605 -235
  52. package/dest/store/contract_class_store.d.ts +17 -4
  53. package/dest/store/contract_class_store.d.ts.map +1 -1
  54. package/dest/store/contract_class_store.js +18 -66
  55. package/dest/store/contract_instance_store.d.ts +28 -1
  56. package/dest/store/contract_instance_store.d.ts.map +1 -1
  57. package/dest/store/contract_instance_store.js +31 -0
  58. package/dest/store/data_stores.d.ts +68 -0
  59. package/dest/store/data_stores.d.ts.map +1 -0
  60. package/dest/store/data_stores.js +54 -0
  61. package/dest/store/function_names_cache.d.ts +17 -0
  62. package/dest/store/function_names_cache.d.ts.map +1 -0
  63. package/dest/store/function_names_cache.js +30 -0
  64. package/dest/store/l2_tips_cache.d.ts +13 -7
  65. package/dest/store/l2_tips_cache.d.ts.map +1 -1
  66. package/dest/store/l2_tips_cache.js +13 -76
  67. package/dest/store/log_store.d.ts +42 -37
  68. package/dest/store/log_store.d.ts.map +1 -1
  69. package/dest/store/log_store.js +259 -443
  70. package/dest/store/log_store_codec.d.ts +70 -0
  71. package/dest/store/log_store_codec.d.ts.map +1 -0
  72. package/dest/store/log_store_codec.js +101 -0
  73. package/dest/store/message_store.d.ts +11 -1
  74. package/dest/store/message_store.d.ts.map +1 -1
  75. package/dest/store/message_store.js +50 -8
  76. package/dest/test/fake_l1_state.d.ts +8 -3
  77. package/dest/test/fake_l1_state.d.ts.map +1 -1
  78. package/dest/test/fake_l1_state.js +75 -18
  79. package/dest/test/mock_l1_to_l2_message_source.d.ts +1 -1
  80. package/dest/test/mock_l1_to_l2_message_source.d.ts.map +1 -1
  81. package/dest/test/mock_l1_to_l2_message_source.js +2 -1
  82. package/dest/test/mock_l2_block_source.d.ts +51 -50
  83. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  84. package/dest/test/mock_l2_block_source.js +244 -181
  85. package/dest/test/noop_l1_archiver.d.ts +10 -6
  86. package/dest/test/noop_l1_archiver.d.ts.map +1 -1
  87. package/dest/test/noop_l1_archiver.js +22 -9
  88. package/package.json +14 -14
  89. package/src/archiver.ts +293 -75
  90. package/src/config.ts +24 -11
  91. package/src/errors.ts +117 -2
  92. package/src/factory.ts +40 -23
  93. package/src/index.ts +18 -2
  94. package/src/l1/calldata_retriever.ts +16 -5
  95. package/src/l1/data_retrieval.ts +52 -53
  96. package/src/l1/validate_historical_logs.ts +140 -0
  97. package/src/modules/contract_data_source_adapter.ts +55 -0
  98. package/src/modules/data_source_base.ts +341 -174
  99. package/src/modules/data_store_updater.ts +152 -137
  100. package/src/modules/instrumentation.ts +48 -8
  101. package/src/modules/l1_synchronizer.ts +556 -262
  102. package/src/modules/outbox_trees_resolver.ts +199 -0
  103. package/src/modules/validation.ts +10 -9
  104. package/src/store/block_store.ts +750 -285
  105. package/src/store/contract_class_store.ts +24 -100
  106. package/src/store/contract_instance_store.ts +43 -0
  107. package/src/store/data_stores.ts +104 -0
  108. package/src/store/function_names_cache.ts +37 -0
  109. package/src/store/l2_tips_cache.ts +16 -70
  110. package/src/store/log_store.ts +297 -609
  111. package/src/store/log_store_codec.ts +132 -0
  112. package/src/store/message_store.ts +59 -9
  113. package/src/structs/inbox_message.ts +1 -1
  114. package/src/test/fake_l1_state.ts +96 -29
  115. package/src/test/mock_l1_to_l2_message_source.ts +1 -0
  116. package/src/test/mock_l2_block_source.ts +299 -232
  117. package/src/test/noop_l1_archiver.ts +42 -9
  118. package/dest/store/kv_archiver_store.d.ts +0 -362
  119. package/dest/store/kv_archiver_store.d.ts.map +0 -1
  120. package/dest/store/kv_archiver_store.js +0 -475
  121. package/src/store/kv_archiver_store.ts +0 -685
package/src/archiver.ts CHANGED
@@ -1,42 +1,49 @@
1
1
  import type { BlobClientInterface } from '@aztec/blob-client/client';
2
- import { EpochCache } from '@aztec/epoch-cache';
3
- import { BlockTagTooOldError, RollupContract } from '@aztec/ethereum/contracts';
2
+ import { EpochCache, PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
3
+ import { BlockTagTooOldError, OutboxContract, RollupContract } from '@aztec/ethereum/contracts';
4
4
  import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
5
5
  import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
6
6
  import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
7
7
  import { Buffer32 } from '@aztec/foundation/buffer';
8
- import { merge } from '@aztec/foundation/collection';
8
+ import { merge, pick } from '@aztec/foundation/collection';
9
9
  import { Fr } from '@aztec/foundation/curves/bn254';
10
10
  import { EthAddress } from '@aztec/foundation/eth-address';
11
11
  import { type Logger, createLogger } from '@aztec/foundation/log';
12
12
  import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise';
13
13
  import { RunningPromise, makeLoggingErrorHandler } from '@aztec/foundation/running-promise';
14
- import { DateProvider } from '@aztec/foundation/timer';
14
+ import { DateProvider, elapsed } from '@aztec/foundation/timer';
15
15
  import {
16
16
  type ArchiverEmitter,
17
+ type BlockHash,
17
18
  L2Block,
18
19
  type L2BlockSink,
20
+ L2BlockSourceEvents,
19
21
  type L2Tips,
20
22
  type ValidateCheckpointResult,
21
23
  } from '@aztec/stdlib/block';
22
- import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
24
+ import { type ProposedCheckpointInput, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
23
25
  import {
24
26
  type L1RollupConstants,
25
27
  getEpochAtSlot,
26
28
  getSlotAtNextL1Block,
27
29
  getSlotRangeForEpoch,
30
+ getTimestampForSlot,
28
31
  getTimestampRangeForEpoch,
29
32
  } from '@aztec/stdlib/epoch-helpers';
33
+ import type { L2ToL1MembershipWitness } from '@aztec/stdlib/messaging';
34
+ import type { BlockHeader, TxHash } from '@aztec/stdlib/tx';
30
35
  import { type TelemetryClient, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
31
36
 
32
37
  import { type ArchiverConfig, mapArchiverConfig } from './config.js';
33
- import { BlockAlreadyCheckpointedError, NoBlobBodiesFoundError } from './errors.js';
38
+ import { BlockAlreadyCheckpointedError, BlockOrCheckpointSlotExpiredError, NoBlobBodiesFoundError } from './errors.js';
39
+ import { validateAndLogHistoricalLogsAvailability } from './l1/validate_historical_logs.js';
34
40
  import { validateAndLogTraceAvailability } from './l1/validate_trace.js';
35
41
  import { ArchiverDataSourceBase } from './modules/data_source_base.js';
36
42
  import { ArchiverDataStoreUpdater } from './modules/data_store_updater.js';
37
43
  import type { ArchiverInstrumentation } from './modules/instrumentation.js';
38
44
  import type { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js';
39
- import type { KVArchiverDataStore } from './store/kv_archiver_store.js';
45
+ import { OutboxTreesResolver } from './modules/outbox_trees_resolver.js';
46
+ import { type ArchiverDataStores, backupArchiverDataStores, getArchiverSynchPoint } from './store/data_stores.js';
40
47
  import { L2TipsCache } from './store/l2_tips_cache.js';
41
48
 
42
49
  /** Export ArchiverEmitter for use in factory and tests. */
@@ -44,11 +51,20 @@ export type { ArchiverEmitter };
44
51
 
45
52
  /** Request to add a block to the archiver, queued for processing by the sync loop. */
46
53
  type AddBlockRequest = {
54
+ type: 'block';
47
55
  block: L2Block;
48
56
  resolve: () => void;
49
57
  reject: (err: Error) => void;
50
58
  };
51
59
 
60
+ /** Request to add a proposed checkpoint to the archiver, queued for processing by the sync loop. */
61
+ type AddProposedCheckpointRequest = {
62
+ type: 'checkpoint';
63
+ checkpoint: ProposedCheckpointInput;
64
+ resolve: () => void;
65
+ reject: (err: Error) => void;
66
+ };
67
+
52
68
  export type ArchiverDeps = {
53
69
  telemetry?: TelemetryClient;
54
70
  blobClient: BlobClientInterface;
@@ -71,11 +87,14 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
71
87
  /** L1 synchronizer that handles fetching checkpoints and messages from L1. */
72
88
  private readonly synchronizer: ArchiverL1Synchronizer;
73
89
 
90
+ /** Resolver for L2-to-L1 message membership witnesses, built over this archiver's read view. */
91
+ private readonly outboxTreesResolver: OutboxTreesResolver;
92
+
74
93
  private initialSyncComplete: boolean = false;
75
94
  private initialSyncPromise: PromiseWithResolvers<void>;
76
95
 
77
- /** Queue of blocks to be added to the store, processed by the sync loop. */
78
- private blockQueue: AddBlockRequest[] = [];
96
+ /** Queue of blocks and checkpoints to be added to the store, processed by the sync loop. */
97
+ private inboundQueue: (AddBlockRequest | AddProposedCheckpointRequest)[] = [];
79
98
 
80
99
  /** Helper to handle updates to the store */
81
100
  private readonly updater: ArchiverDataStoreUpdater;
@@ -85,37 +104,50 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
85
104
 
86
105
  public readonly tracer: Tracer;
87
106
 
107
+ private readonly instrumentation: ArchiverInstrumentation;
108
+
88
109
  /**
89
110
  * Creates a new instance of the Archiver.
90
111
  * @param publicClient - A client for interacting with the Ethereum node.
91
112
  * @param debugClient - A client for interacting with the Ethereum node for debug/trace methods.
92
113
  * @param rollup - Rollup contract instance.
114
+ * @param outbox - Outbox contract instance, used to read per-epoch L2-to-L1 message roots.
93
115
  * @param inbox - Inbox contract instance.
94
- * @param l1Addresses - L1 contract addresses (registry, governance proposer, slash factory, slashing proposer).
95
- * @param dataStore - An archiver data store for storage & retrieval of blocks, encrypted logs & contract data.
116
+ * @param l1Addresses - L1 contract addresses (registry, governance proposer, slashing proposer).
117
+ * @param dataStores - Archiver substores for storage & retrieval of blocks, encrypted logs & contract data.
96
118
  * @param config - Archiver configuration options.
97
119
  * @param blobClient - Client for retrieving blob data.
98
- * @param epochCache - Cache for epoch-related data.
99
- * @param dateProvider - Provider for current date/time.
100
120
  * @param instrumentation - Instrumentation for metrics and tracing.
101
121
  * @param l1Constants - L1 rollup constants.
122
+ * @param synchronizer - L1 synchronizer that handles fetching checkpoints and messages from L1.
123
+ * @param events - Event emitter shared with the synchronizer.
124
+ * @param initialHeader - Genesis block header.
125
+ * @param initialBlockHash - Precomputed hash of the genesis block header.
126
+ * @param l2TipsCache - In-memory cache for L2 chain tips.
127
+ * @param dateProvider - Provider for current date/time, used for wall-clock orphan-block pruning.
102
128
  * @param log - A logger.
103
129
  */
104
130
  constructor(
105
131
  private readonly publicClient: ViemPublicClient,
106
132
  private readonly debugClient: ViemPublicDebugClient,
107
133
  private readonly rollup: RollupContract,
134
+ private readonly outbox: OutboxContract,
108
135
  private readonly l1Addresses: Pick<
109
136
  L1ContractAddresses,
110
- 'registryAddress' | 'governanceProposerAddress' | 'slashFactoryAddress'
111
- > & { slashingProposerAddress: EthAddress },
112
- readonly dataStore: KVArchiverDataStore,
137
+ 'rollupAddress' | 'registryAddress' | 'inboxAddress' | 'governanceProposerAddress'
138
+ > & {
139
+ slashingProposerAddress: EthAddress;
140
+ },
141
+ readonly dataStores: ArchiverDataStores,
113
142
  private config: {
114
143
  pollingIntervalMs: number;
115
144
  batchSize: number;
116
145
  skipValidateCheckpointAttestations?: boolean;
117
146
  maxAllowedEthClientDriftSeconds: number;
118
147
  ethereumAllowNoDebugHosts?: boolean;
148
+ skipHistoricalLogsCheck?: boolean;
149
+ orphanProposedBlockPruneGraceSeconds: number;
150
+ enableOrphanProposedBlockPruning: boolean;
119
151
  },
120
152
  private readonly blobClient: BlobClientInterface,
121
153
  instrumentation: ArchiverInstrumentation,
@@ -125,20 +157,33 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
125
157
  },
126
158
  synchronizer: ArchiverL1Synchronizer,
127
159
  events: ArchiverEmitter,
128
- l2TipsCache?: L2TipsCache,
160
+ initialHeader: BlockHeader,
161
+ initialBlockHash: BlockHash,
162
+ l2TipsCache: L2TipsCache,
163
+ private readonly dateProvider: DateProvider,
129
164
  private readonly log: Logger = createLogger('archiver'),
130
165
  ) {
131
- super(dataStore, l1Constants);
166
+ super(dataStores, l1Constants, initialHeader, initialBlockHash, l1Constants.genesisArchiveRoot);
132
167
 
133
168
  this.tracer = instrumentation.tracer;
169
+ this.instrumentation = instrumentation;
134
170
  this.initialSyncPromise = promiseWithResolvers();
135
171
  this.synchronizer = synchronizer;
136
172
  this.events = events;
137
- this.l2TipsCache = l2TipsCache ?? new L2TipsCache(this.dataStore.blockStore);
138
- this.updater = new ArchiverDataStoreUpdater(this.dataStore, this.l2TipsCache, {
173
+ this.l2TipsCache = l2TipsCache;
174
+ this.updater = new ArchiverDataStoreUpdater(this.dataStores, this.l2TipsCache, {
139
175
  rollupManaLimit: l1Constants.rollupManaLimit,
140
176
  });
141
177
 
178
+ // The resolver reads its witness data from this archiver and pins its lazy Outbox-root fetches
179
+ // to the archiver's synced L1 block. No method on `this` is invoked during construction.
180
+ this.outboxTreesResolver = new OutboxTreesResolver(
181
+ this.outbox,
182
+ this,
183
+ () => Promise.resolve(this.getL1BlockNumber()),
184
+ l1Constants.epochDuration,
185
+ );
186
+
142
187
  // Running promise starts with a small interval inbetween runs, so all iterations needed for the initial sync
143
188
  // are done as fast as possible. This then gets updated once the initial sync completes.
144
189
  this.runningPromise = new RunningPromise(
@@ -149,6 +194,27 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
149
194
  );
150
195
  }
151
196
 
197
+ /**
198
+ * Returns the L2-to-L1 membership witness for `message` emitted by tx `txHash`. The node selects
199
+ * the smallest partial-proof root on the Outbox that covers the tx's checkpoint and builds the
200
+ * witness against it.
201
+ *
202
+ * The node reads the Outbox roots lazily, pinned to its synced L1 block, so the witness reflects
203
+ * the node's synced view. Returns `undefined` if the tx isn't yet in a block/epoch or no covering
204
+ * root has landed on L1 as of the synced block.
205
+ *
206
+ * Caveat: roots are cached only for the current synced L1 block and re-fetched once the node
207
+ * advances, so a reorg is picked up on the next synced-block advance rather than re-validated per
208
+ * request.
209
+ */
210
+ public getL2ToL1MembershipWitness(
211
+ txHash: TxHash,
212
+ message: Fr,
213
+ messageIndexInTx?: number,
214
+ ): Promise<L2ToL1MembershipWitness | undefined> {
215
+ return this.outboxTreesResolver.getL2ToL1MembershipWitness(txHash, message, messageIndexInTx);
216
+ }
217
+
152
218
  /** Updates archiver config */
153
219
  public updateConfig(newConfig: Partial<ArchiverConfig>) {
154
220
  this.config = merge(this.config, mapArchiverConfig(newConfig));
@@ -171,11 +237,24 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
171
237
  this.config.ethereumAllowNoDebugHosts ?? false,
172
238
  this.log.getBindings(),
173
239
  );
240
+ await validateAndLogHistoricalLogsAvailability(
241
+ this.publicClient,
242
+ {
243
+ rollupAddress: this.l1Addresses.rollupAddress,
244
+ inboxAddress: this.l1Addresses.inboxAddress,
245
+ registryAddress: this.l1Addresses.registryAddress,
246
+ governanceProposerAddress: this.l1Addresses.governanceProposerAddress,
247
+ },
248
+ this.config.skipHistoricalLogsCheck ?? false,
249
+ this.log.getBindings(),
250
+ );
174
251
 
175
252
  // Log initial state for the archiver
176
253
  const { l1StartBlock } = this.l1Constants;
177
- const { blocksSynchedTo = l1StartBlock, messagesSynchedTo = l1StartBlock } = await this.store.getSynchPoint();
178
- const currentL2Checkpoint = await this.getSynchedCheckpointNumber();
254
+ const { blocksSynchedTo = l1StartBlock, messagesSynchedTo = l1StartBlock } = await getArchiverSynchPoint(
255
+ this.stores,
256
+ );
257
+ const currentL2Checkpoint = await this.getCheckpointNumber();
179
258
  this.log.info(
180
259
  `Starting archiver sync to rollup contract ${this.rollup.address} from L1 block ${blocksSynchedTo} and L2 checkpoint ${currentL2Checkpoint}`,
181
260
  { blocksSynchedTo, messagesSynchedTo, currentL2Checkpoint },
@@ -192,6 +271,14 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
192
271
  return this.runningPromise.trigger();
193
272
  }
194
273
 
274
+ public trySyncImmediate() {
275
+ try {
276
+ return this.syncImmediate();
277
+ } catch (err) {
278
+ this.log.error(`Failed to trigger immediate archiver sync: ${err}`, err);
279
+ }
280
+ }
281
+
195
282
  /**
196
283
  * Queues a block to be added to the archiver store and triggers processing.
197
284
  * The block will be processed by the sync loop.
@@ -200,58 +287,85 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
200
287
  * @returns A promise that resolves when the block has been added to the store, or rejects on error.
201
288
  */
202
289
  public addBlock(block: L2Block): Promise<void> {
203
- return new Promise<void>((resolve, reject) => {
204
- this.blockQueue.push({ block, resolve, reject });
205
- this.log.debug(`Queued block ${block.number} for processing`);
206
- // Trigger an immediate sync, but don't wait for it - the promise resolves when the block is processed
207
- this.syncImmediate().catch(err => {
208
- this.log.error(`Sync immediate call failed: ${err}`);
209
- });
210
- });
290
+ const promise = promiseWithResolvers<void>();
291
+ this.inboundQueue.push({ block, ...promise, type: 'block' });
292
+ this.log.debug(`Queued block ${block.number} for processing`);
293
+ void this.trySyncImmediate();
294
+ return promise.promise;
211
295
  }
212
296
 
213
297
  /**
214
- * Processes all queued blocks, adding them to the store.
298
+ * Queues a new proposed checkpoint into the archiver store.
299
+ * Checks that the checkpoint is not for an L2 slot already synced from L1.
300
+ * Resolves once the checkpoint has been processed.
301
+ */
302
+ public addProposedCheckpoint(pending: ProposedCheckpointInput): Promise<void> {
303
+ const promise = promiseWithResolvers<void>();
304
+ this.inboundQueue.push({ checkpoint: pending, ...promise, type: 'checkpoint' });
305
+ this.log.debug(`Queued checkpoint ${pending.checkpointNumber} for processing`);
306
+ void this.trySyncImmediate();
307
+ return promise.promise;
308
+ }
309
+
310
+ /**
311
+ * Processes all queued blocks and checkpoints, adding them to the store.
215
312
  * Called at the beginning of each sync iteration.
216
- * Blocks are processed in the order they were queued.
313
+ * Items are processed in the order they were queued.
217
314
  */
218
- private async processQueuedBlocks(): Promise<void> {
219
- if (this.blockQueue.length === 0) {
315
+ private async processInboundQueue(): Promise<void> {
316
+ if (this.inboundQueue.length === 0) {
220
317
  return;
221
318
  }
222
319
 
223
- // Take all blocks from the queue
224
- const queuedItems = this.blockQueue.splice(0, this.blockQueue.length);
225
- this.log.debug(`Processing ${queuedItems.length} queued block(s)`);
320
+ // Take all items from the queue
321
+ const queuedItems = this.inboundQueue.splice(0, this.inboundQueue.length);
322
+ this.log.debug(`Processing ${queuedItems.length} queued inbound items`);
226
323
 
227
324
  // Calculate slot threshold for validation
228
325
  const l1Timestamp = this.synchronizer.getL1Timestamp();
229
326
  const slotAtNextL1Block =
230
327
  l1Timestamp === undefined ? undefined : getSlotAtNextL1Block(l1Timestamp, this.l1Constants);
231
328
 
232
- // Process each block individually to properly resolve/reject each promise
233
- for (const { block, resolve, reject } of queuedItems) {
234
- const blockSlot = block.header.globalVariables.slotNumber;
235
- if (slotAtNextL1Block !== undefined && blockSlot < slotAtNextL1Block) {
329
+ // Helpers for manipulating blocks and checkpoints in the queue
330
+ const getSlot: (item: AddBlockRequest | AddProposedCheckpointRequest) => SlotNumber = item =>
331
+ item.type === 'block' ? item.block.header.globalVariables.slotNumber : item.checkpoint.header.slotNumber;
332
+ const getNumber: (item: AddBlockRequest | AddProposedCheckpointRequest) => number = item =>
333
+ item.type === 'block' ? item.block.number : item.checkpoint.checkpointNumber;
334
+
335
+ // Process each item individually to properly resolve/reject each promise
336
+ for (const item of queuedItems) {
337
+ const { resolve, reject, type } = item;
338
+ const itemSlot = getSlot(item);
339
+ const itemNumber = getNumber(item);
340
+ if (slotAtNextL1Block !== undefined && itemSlot < slotAtNextL1Block) {
341
+ const nextSlotTimestamp = getTimestampForSlot(slotAtNextL1Block, this.l1Constants);
236
342
  this.log.warn(
237
- `Rejecting proposed block ${block.number} for past slot ${blockSlot} (current is ${slotAtNextL1Block})`,
238
- { block: block.toBlockInfo(), l1Timestamp, slotAtNextL1Block },
343
+ `Rejecting proposed ${type} ${itemNumber} for past slot ${itemSlot} (current ${slotAtNextL1Block})`,
344
+ { number: itemNumber, type, l1Timestamp, slotAtNextL1Block, nextSlotTimestamp },
239
345
  );
240
- reject(new Error(`Block ${block.number} is for past slot ${blockSlot} (current is ${slotAtNextL1Block})`));
346
+ reject(new BlockOrCheckpointSlotExpiredError(itemSlot, nextSlotTimestamp, l1Timestamp));
241
347
  continue;
242
348
  }
243
349
 
244
350
  try {
245
- await this.updater.addProposedBlock(block);
246
- this.log.debug(`Added block ${block.number} to store`);
351
+ if (type === 'block') {
352
+ const [durationMs] = await elapsed(() => this.updater.addProposedBlock(item.block));
353
+ this.instrumentation.processNewProposedBlock(durationMs, item.block);
354
+ } else {
355
+ await this.updater.addProposedCheckpoint(item.checkpoint);
356
+ }
357
+ this.log.debug(`Added ${type} ${itemNumber} to store`);
247
358
  resolve();
248
359
  } catch (err: any) {
249
360
  if (err instanceof BlockAlreadyCheckpointedError) {
250
- this.log.debug(`Proposed block ${block.number} matches already checkpointed block, ignoring late proposal`);
361
+ this.log.debug(`Proposed block ${itemNumber} matches already checkpointed block, ignoring late proposal`);
251
362
  resolve();
252
363
  continue;
253
364
  }
254
- this.log.error(`Failed to add block ${block.number} to store: ${err.message}`);
365
+ this.log.error(`Failed to add ${type} ${itemNumber} to store: ${err.message}`, err, {
366
+ number: itemNumber,
367
+ type,
368
+ });
255
369
  reject(err);
256
370
  }
257
371
  }
@@ -267,9 +381,106 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
267
381
  @trackSpan('Archiver.sync')
268
382
  private async sync() {
269
383
  // Process any queued blocks first, before doing L1 sync
270
- await this.processQueuedBlocks();
384
+ await this.processInboundQueue();
271
385
  // Now perform L1 sync
272
386
  await this.syncFromL1();
387
+ // Prune proposed blocks with no corresponding proposed checkpoint by the end of their build slot.
388
+ await this.pruneOrphanProposedBlocks();
389
+ }
390
+
391
+ /**
392
+ * Prunes a block-only local tip that was built atop a checkpoint that was never itself proposed.
393
+ *
394
+ * Under pipelining, a proposer publishes the blocks for a checkpoint (block-only proposals) before
395
+ * assembling and publishing the enclosing proposed checkpoint at the end of the build slot. A node
396
+ * that received those blocks but never the proposed checkpoint is left with an orphan tip it must
397
+ * not build on. We prune it once enough wall-clock time has elapsed that the proposed checkpoint
398
+ * should have arrived. This runs on wall-clock time (not L1 block advancement) so it fires during
399
+ * quiet L1 periods, and is the liveness counterpart to the sequencer's checkSync guard.
400
+ *
401
+ * The uncheckpointed suffix is scanned in order. Blocks covered by proposed checkpoints are left in
402
+ * place; the first block not covered by a proposed checkpoint starts the orphan suffix to prune.
403
+ */
404
+ private async pruneOrphanProposedBlocks(): Promise<void> {
405
+ // Orphan pruning is a pipelining-only liveness mechanism: it cleans up block-only proposals
406
+ // whose enclosing checkpoint never arrived over gossip. The test-only automine sequencer is
407
+ // non-pipelined — it publishes each checkpoint synchronously in the slot it builds — so it
408
+ // never produces orphan proposed blocks. Running the prune there would race its two-step local
409
+ // push (addBlock then addProposedCheckpoint) and delete the block before its checkpoint attaches.
410
+ if (!this.config.enableOrphanProposedBlockPruning) {
411
+ return;
412
+ }
413
+ const tips = await this.getL2Tips();
414
+ const now = BigInt(this.dateProvider.nowInSeconds());
415
+ const pipeliningOffset = PROPOSER_PIPELINING_SLOT_OFFSET;
416
+
417
+ // The proposed tip is a proposed-checkpointed block, so there are no orphan proposed blocks to prune
418
+ if (tips.proposedCheckpoint.block.number === tips.proposed.number) {
419
+ this.log.trace(
420
+ `No orphan proposed blocks to prune: proposed tip ${tips.proposed.number} is checkpointed`,
421
+ pick(tips, 'proposed', 'proposedCheckpoint'),
422
+ );
423
+ return;
424
+ }
425
+
426
+ // Load the blocks that are candidates for pruning (ie blocks without a proposed checkpoint covering them)
427
+ const blocksWithoutProposedCheckpoint = await this.stores.blocks.getBlocksData({
428
+ from: BlockNumber(tips.proposedCheckpoint.block.number + 1),
429
+ limit: tips.proposed.number - tips.proposedCheckpoint.block.number,
430
+ });
431
+
432
+ // Iterate through them in order, the first one with a slot that should have received a proposed checkpoint
433
+ // is the first orphan block, and all blocks after it are also orphaned and should be pruned.
434
+ let lastSlotChecked = undefined;
435
+ for (const blockData of blocksWithoutProposedCheckpoint) {
436
+ // No need to recheck if this block had the same slot as the previous one.
437
+ const blockSlot = blockData.header.getSlot();
438
+ const blockNumber = blockData.header.getBlockNumber();
439
+ if (lastSlotChecked !== undefined && blockSlot === lastSlotChecked) {
440
+ continue;
441
+ }
442
+ lastSlotChecked = blockSlot;
443
+
444
+ // The proposed checkpoint should have landed by the start of the slot after the block's build slot
445
+ // (build slot = blockSlot - pipeliningOffset). Wait a grace period beyond that to tolerate propagation.
446
+ const expectedCheckpointedBySlot = SlotNumber(Number(blockSlot) - pipeliningOffset + 1);
447
+ const expectedCheckpointedByTime =
448
+ getTimestampForSlot(expectedCheckpointedBySlot, this.l1Constants) +
449
+ BigInt(this.config.orphanProposedBlockPruneGraceSeconds);
450
+
451
+ // If it's not checkpointed by the expected time, prune it along with all blocks after it.
452
+ if (now >= expectedCheckpointedByTime) {
453
+ const pruneAfterBlockNumber = BlockNumber(blockNumber - 1);
454
+ this.log.warn(
455
+ `Pruning orphan blocks after block ${pruneAfterBlockNumber}: block at slot ${blockSlot} belongs to ` +
456
+ `checkpoint ${blockData.checkpointNumber} which has no matching proposed checkpoint`,
457
+ {
458
+ firstUncheckpointedBlockHeader: blockData.header.toInspect(),
459
+ blockCheckpointNumber: blockData.checkpointNumber,
460
+ blockNumber,
461
+ blockSlot,
462
+ pipeliningOffset,
463
+ expectedCheckpointedBySlot,
464
+ expectedCheckpointedByTime,
465
+ now,
466
+ },
467
+ );
468
+
469
+ const prunedBlocks = await this.updater.removeBlocksWithoutProposedCheckpointAfter(pruneAfterBlockNumber);
470
+ if (prunedBlocks.length > 0) {
471
+ this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
472
+ type: L2BlockSourceEvents.L2PruneUncheckpointed,
473
+ slotNumber: blockSlot,
474
+ blocks: prunedBlocks,
475
+ });
476
+ }
477
+ return;
478
+ }
479
+ }
480
+
481
+ this.log.trace('No orphan proposed blocks to prune: all uncheckpointed blocks are still within the grace period', {
482
+ blocksWithoutProposedCheckpoint: blocksWithoutProposedCheckpoint.map(b => b.header.toInspect()),
483
+ });
273
484
  }
274
485
 
275
486
  private async syncFromL1() {
@@ -283,7 +494,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
283
494
  if (currentL1BlockNumber + 1n >= l1BlockNumberAtEnd) {
284
495
  this.log.info(`Initial archiver sync to L1 block ${currentL1BlockNumber} complete`, {
285
496
  l1BlockNumber: currentL1BlockNumber,
286
- syncPoint: await this.store.getSynchPoint(),
497
+ syncPoint: await getArchiverSynchPoint(this.stores),
287
498
  ...(await this.getL2Tips()),
288
499
  });
289
500
  this.runningPromise.setPollingIntervalMS(this.config.pollingIntervalMs);
@@ -315,7 +526,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
315
526
  }
316
527
 
317
528
  public backupTo(destPath: string): Promise<string> {
318
- return this.dataStore.backupTo(destPath);
529
+ return backupArchiverDataStores(this.dataStores, destPath);
319
530
  }
320
531
 
321
532
  public getL1Constants(): Promise<L1RollupConstants> {
@@ -357,9 +568,9 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
357
568
  }
358
569
 
359
570
  let slotFromCheckpoint: SlotNumber | undefined;
360
- const latestCheckpointNumber = await this.store.getSynchedCheckpointNumber();
571
+ const latestCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
361
572
  if (latestCheckpointNumber > 0) {
362
- const checkpointData = await this.store.getCheckpointData(latestCheckpointNumber);
573
+ const checkpointData = await this.stores.blocks.getCheckpointData(latestCheckpointNumber);
363
574
  if (checkpointData) {
364
575
  slotFromCheckpoint = checkpointData.header.slotNumber;
365
576
  }
@@ -391,8 +602,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
391
602
  // The epoch is complete if the current checkpointed L2 block is the last one in the epoch (or later).
392
603
  // We use the checkpointed block number (synced from L1) instead of 'latest' to avoid returning true
393
604
  // prematurely when proposed blocks have been pushed to the archiver but not yet checkpointed on L1.
394
- const checkpointedBlockNumber = await this.getCheckpointedL2BlockNumber();
395
- const header = checkpointedBlockNumber > 0 ? await this.getBlockHeader(checkpointedBlockNumber) : undefined;
605
+ const header = (await this.getBlockData({ tag: 'checkpointed' }))?.header;
396
606
  const slot = header ? header.globalVariables.slotNumber : undefined;
397
607
  const [_startSlot, endSlot] = getSlotRangeForEpoch(epochNumber, this.l1Constants);
398
608
  if (slot && slot >= endSlot) {
@@ -448,29 +658,34 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
448
658
  if (targetL2BlockNumber >= currentL2Block) {
449
659
  throw new Error(`Target L2 block ${targetL2BlockNumber} must be less than current L2 block ${currentL2Block}`);
450
660
  }
451
- const targetL2Block = await this.store.getCheckpointedBlock(targetL2BlockNumber);
452
- if (!targetL2Block) {
661
+ const checkpointedTip = await this.stores.blocks.getCheckpointedL2BlockNumber();
662
+ if (targetL2BlockNumber > checkpointedTip) {
663
+ throw new Error(`Target L2 block ${targetL2BlockNumber} is not checkpointed yet`);
664
+ }
665
+ const targetBlockData = await this.stores.blocks.getBlockData({ number: targetL2BlockNumber });
666
+ if (!targetBlockData) {
453
667
  throw new Error(`Target L2 block ${targetL2BlockNumber} not found`);
454
668
  }
455
- const targetCheckpointNumber = targetL2Block.checkpointNumber;
669
+ const targetCheckpointNumber = targetBlockData.checkpointNumber;
456
670
 
457
671
  // Rollback operates at checkpoint granularity: the target block must be the last block of its checkpoint.
458
- const checkpointData = await this.store.getCheckpointData(targetCheckpointNumber);
459
- if (checkpointData) {
460
- const lastBlockInCheckpoint = BlockNumber(checkpointData.startBlock + checkpointData.blockCount - 1);
461
- if (targetL2BlockNumber !== lastBlockInCheckpoint) {
462
- const previousCheckpointBoundary =
463
- checkpointData.startBlock > 1 ? BlockNumber(checkpointData.startBlock - 1) : BlockNumber(0);
464
- throw new Error(
465
- `Target L2 block ${targetL2BlockNumber} is not at a checkpoint boundary. ` +
466
- `Checkpoint ${targetCheckpointNumber} spans blocks ${checkpointData.startBlock} to ${lastBlockInCheckpoint}. ` +
467
- `Use block ${lastBlockInCheckpoint} to roll back to this checkpoint, ` +
468
- `or block ${previousCheckpointBoundary} to roll back to the previous one.`,
469
- );
470
- }
672
+ const checkpointData = await this.stores.blocks.getCheckpointData(targetCheckpointNumber);
673
+ if (!checkpointData) {
674
+ throw new Error(`Checkpoint ${targetCheckpointNumber} not found for block ${targetL2BlockNumber}`);
675
+ }
676
+ const lastBlockInCheckpoint = BlockNumber(checkpointData.startBlock + checkpointData.blockCount - 1);
677
+ if (targetL2BlockNumber !== lastBlockInCheckpoint) {
678
+ const previousCheckpointBoundary =
679
+ checkpointData.startBlock > 1 ? BlockNumber(checkpointData.startBlock - 1) : BlockNumber(0);
680
+ throw new Error(
681
+ `Target L2 block ${targetL2BlockNumber} is not at a checkpoint boundary. ` +
682
+ `Checkpoint ${targetCheckpointNumber} spans blocks ${checkpointData.startBlock} to ${lastBlockInCheckpoint}. ` +
683
+ `Use block ${lastBlockInCheckpoint} to roll back to this checkpoint, ` +
684
+ `or block ${previousCheckpointBoundary} to roll back to the previous one.`,
685
+ );
471
686
  }
472
687
 
473
- const targetL1BlockNumber = targetL2Block.l1.blockNumber;
688
+ const targetL1BlockNumber = checkpointData.l1.blockNumber;
474
689
  const targetL1Block = await this.publicClient.getBlock({
475
690
  blockNumber: targetL1BlockNumber,
476
691
  includeTransactions: false,
@@ -484,10 +699,13 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
484
699
  );
485
700
  await this.updater.removeCheckpointsAfter(targetCheckpointNumber);
486
701
  this.log.info(`Rolling back L1 to L2 messages to checkpoint ${targetCheckpointNumber}`);
487
- await this.store.rollbackL1ToL2MessagesToCheckpoint(targetCheckpointNumber);
702
+ await this.stores.messages.rollbackL1ToL2MessagesToCheckpoint(targetCheckpointNumber);
488
703
  this.log.info(`Setting L1 syncpoints to ${targetL1BlockNumber}`);
489
- await this.store.setCheckpointSynchedL1BlockNumber(targetL1BlockNumber);
490
- await this.store.setMessageSynchedL1Block({ l1BlockNumber: targetL1BlockNumber, l1BlockHash: targetL1BlockHash });
704
+ await this.stores.blocks.setSynchedL1BlockNumber(targetL1BlockNumber);
705
+ await this.stores.messages.setMessageSyncState(
706
+ { l1BlockNumber: targetL1BlockNumber, l1BlockHash: targetL1BlockHash },
707
+ undefined,
708
+ );
491
709
  if (targetL2BlockNumber < currentProvenBlock) {
492
710
  this.log.info(`Rolling back proven L2 checkpoint to ${targetCheckpointNumber}`);
493
711
  await this.updater.setProvenCheckpointNumber(targetCheckpointNumber);