@aztec/archiver 0.0.1-commit.04852196a → 0.0.1-commit.04d373f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -11
- package/dest/archiver.d.ts +36 -19
- package/dest/archiver.d.ts.map +1 -1
- package/dest/archiver.js +250 -79
- package/dest/config.d.ts +6 -3
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +23 -15
- package/dest/errors.d.ts +55 -9
- package/dest/errors.d.ts.map +1 -1
- package/dest/errors.js +81 -14
- package/dest/factory.d.ts +12 -8
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +42 -32
- package/dest/index.d.ts +11 -3
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +10 -2
- package/dest/l1/calldata_retriever.d.ts +2 -1
- package/dest/l1/calldata_retriever.d.ts.map +1 -1
- package/dest/l1/calldata_retriever.js +15 -5
- package/dest/l1/data_retrieval.d.ts +19 -10
- package/dest/l1/data_retrieval.d.ts.map +1 -1
- package/dest/l1/data_retrieval.js +25 -32
- package/dest/l1/trace_tx.d.ts +12 -66
- package/dest/l1/trace_tx.d.ts.map +1 -1
- package/dest/l1/validate_historical_logs.d.ts +23 -0
- package/dest/l1/validate_historical_logs.d.ts.map +1 -0
- package/dest/l1/validate_historical_logs.js +108 -0
- package/dest/modules/contract_data_source_adapter.d.ts +25 -0
- package/dest/modules/contract_data_source_adapter.d.ts.map +1 -0
- package/dest/modules/contract_data_source_adapter.js +40 -0
- package/dest/modules/data_source_base.d.ts +70 -46
- package/dest/modules/data_source_base.d.ts.map +1 -1
- package/dest/modules/data_source_base.js +270 -135
- package/dest/modules/data_store_updater.d.ts +39 -17
- package/dest/modules/data_store_updater.d.ts.map +1 -1
- package/dest/modules/data_store_updater.js +183 -122
- package/dest/modules/instrumentation.d.ts +7 -2
- package/dest/modules/instrumentation.d.ts.map +1 -1
- package/dest/modules/instrumentation.js +25 -7
- package/dest/modules/l1_synchronizer.d.ts +12 -7
- package/dest/modules/l1_synchronizer.d.ts.map +1 -1
- package/dest/modules/l1_synchronizer.js +430 -205
- package/dest/modules/validation.d.ts +4 -3
- package/dest/modules/validation.d.ts.map +1 -1
- package/dest/modules/validation.js +6 -6
- package/dest/store/block_store.d.ts +174 -70
- package/dest/store/block_store.d.ts.map +1 -1
- package/dest/store/block_store.js +696 -250
- package/dest/store/contract_class_store.d.ts +17 -4
- package/dest/store/contract_class_store.d.ts.map +1 -1
- package/dest/store/contract_class_store.js +24 -68
- package/dest/store/contract_instance_store.d.ts +28 -1
- package/dest/store/contract_instance_store.d.ts.map +1 -1
- package/dest/store/contract_instance_store.js +37 -2
- package/dest/store/data_stores.d.ts +68 -0
- package/dest/store/data_stores.d.ts.map +1 -0
- package/dest/store/data_stores.js +54 -0
- package/dest/store/function_names_cache.d.ts +17 -0
- package/dest/store/function_names_cache.d.ts.map +1 -0
- package/dest/store/function_names_cache.js +30 -0
- package/dest/store/l2_tips_cache.d.ts +13 -7
- package/dest/store/l2_tips_cache.d.ts.map +1 -1
- package/dest/store/l2_tips_cache.js +13 -76
- package/dest/store/log_store.d.ts +42 -37
- package/dest/store/log_store.d.ts.map +1 -1
- package/dest/store/log_store.js +262 -408
- package/dest/store/log_store_codec.d.ts +70 -0
- package/dest/store/log_store_codec.d.ts.map +1 -0
- package/dest/store/log_store_codec.js +101 -0
- package/dest/store/message_store.d.ts +11 -1
- package/dest/store/message_store.d.ts.map +1 -1
- package/dest/store/message_store.js +50 -8
- package/dest/test/fake_l1_state.d.ts +20 -1
- package/dest/test/fake_l1_state.d.ts.map +1 -1
- package/dest/test/fake_l1_state.js +114 -18
- package/dest/test/mock_l1_to_l2_message_source.d.ts +1 -1
- package/dest/test/mock_l1_to_l2_message_source.d.ts.map +1 -1
- package/dest/test/mock_l1_to_l2_message_source.js +2 -1
- package/dest/test/mock_l2_block_source.d.ts +51 -46
- package/dest/test/mock_l2_block_source.d.ts.map +1 -1
- package/dest/test/mock_l2_block_source.js +243 -170
- package/dest/test/noop_l1_archiver.d.ts +12 -6
- package/dest/test/noop_l1_archiver.d.ts.map +1 -1
- package/dest/test/noop_l1_archiver.js +26 -9
- package/package.json +14 -14
- package/src/archiver.ts +301 -86
- package/src/config.ts +32 -12
- package/src/errors.ts +122 -21
- package/src/factory.ts +50 -28
- package/src/index.ts +18 -2
- package/src/l1/calldata_retriever.ts +16 -5
- package/src/l1/data_retrieval.ts +36 -45
- package/src/l1/validate_historical_logs.ts +140 -0
- package/src/modules/contract_data_source_adapter.ts +55 -0
- package/src/modules/data_source_base.ts +336 -171
- package/src/modules/data_store_updater.ts +219 -154
- package/src/modules/instrumentation.ts +28 -8
- package/src/modules/l1_synchronizer.ts +566 -248
- package/src/modules/validation.ts +10 -9
- package/src/store/block_store.ts +865 -290
- package/src/store/contract_class_store.ts +31 -103
- package/src/store/contract_instance_store.ts +51 -5
- package/src/store/data_stores.ts +104 -0
- package/src/store/function_names_cache.ts +37 -0
- package/src/store/l2_tips_cache.ts +16 -70
- package/src/store/log_store.ts +301 -559
- package/src/store/log_store_codec.ts +132 -0
- package/src/store/message_store.ts +59 -9
- package/src/structs/inbox_message.ts +1 -1
- package/src/test/fake_l1_state.ts +142 -29
- package/src/test/mock_l1_to_l2_message_source.ts +1 -0
- package/src/test/mock_l2_block_source.ts +303 -205
- package/src/test/noop_l1_archiver.ts +39 -9
- package/dest/store/kv_archiver_store.d.ts +0 -354
- package/dest/store/kv_archiver_store.d.ts.map +0 -1
- package/dest/store/kv_archiver_store.js +0 -464
- package/src/store/kv_archiver_store.ts +0 -671
package/src/archiver.ts
CHANGED
|
@@ -5,39 +5,43 @@ import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'
|
|
|
5
5
|
import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
|
|
6
6
|
import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
7
7
|
import { Buffer32 } from '@aztec/foundation/buffer';
|
|
8
|
-
import { merge } from '@aztec/foundation/collection';
|
|
8
|
+
import { merge, pick } from '@aztec/foundation/collection';
|
|
9
9
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
10
10
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
11
11
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
12
12
|
import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
13
13
|
import { RunningPromise, makeLoggingErrorHandler } from '@aztec/foundation/running-promise';
|
|
14
|
-
import { DateProvider } from '@aztec/foundation/timer';
|
|
14
|
+
import { DateProvider, elapsed } from '@aztec/foundation/timer';
|
|
15
15
|
import {
|
|
16
16
|
type ArchiverEmitter,
|
|
17
|
+
type BlockHash,
|
|
17
18
|
L2Block,
|
|
18
19
|
type L2BlockSink,
|
|
20
|
+
L2BlockSourceEvents,
|
|
19
21
|
type L2Tips,
|
|
20
22
|
type ValidateCheckpointResult,
|
|
21
23
|
} from '@aztec/stdlib/block';
|
|
22
|
-
import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
24
|
+
import { type ProposedCheckpointInput, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
23
25
|
import {
|
|
24
26
|
type L1RollupConstants,
|
|
25
|
-
|
|
27
|
+
getEpochAtSlot,
|
|
26
28
|
getSlotAtNextL1Block,
|
|
27
|
-
getSlotAtTimestamp,
|
|
28
29
|
getSlotRangeForEpoch,
|
|
30
|
+
getTimestampForSlot,
|
|
29
31
|
getTimestampRangeForEpoch,
|
|
30
32
|
} from '@aztec/stdlib/epoch-helpers';
|
|
33
|
+
import type { BlockHeader } from '@aztec/stdlib/tx';
|
|
31
34
|
import { type TelemetryClient, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
|
|
32
35
|
|
|
33
36
|
import { type ArchiverConfig, mapArchiverConfig } from './config.js';
|
|
34
|
-
import { NoBlobBodiesFoundError } from './errors.js';
|
|
37
|
+
import { BlockAlreadyCheckpointedError, BlockOrCheckpointSlotExpiredError, NoBlobBodiesFoundError } from './errors.js';
|
|
38
|
+
import { validateAndLogHistoricalLogsAvailability } from './l1/validate_historical_logs.js';
|
|
35
39
|
import { validateAndLogTraceAvailability } from './l1/validate_trace.js';
|
|
36
40
|
import { ArchiverDataSourceBase } from './modules/data_source_base.js';
|
|
37
41
|
import { ArchiverDataStoreUpdater } from './modules/data_store_updater.js';
|
|
38
42
|
import type { ArchiverInstrumentation } from './modules/instrumentation.js';
|
|
39
43
|
import type { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js';
|
|
40
|
-
import type
|
|
44
|
+
import { type ArchiverDataStores, backupArchiverDataStores, getArchiverSynchPoint } from './store/data_stores.js';
|
|
41
45
|
import { L2TipsCache } from './store/l2_tips_cache.js';
|
|
42
46
|
|
|
43
47
|
/** Export ArchiverEmitter for use in factory and tests. */
|
|
@@ -45,11 +49,20 @@ export type { ArchiverEmitter };
|
|
|
45
49
|
|
|
46
50
|
/** Request to add a block to the archiver, queued for processing by the sync loop. */
|
|
47
51
|
type AddBlockRequest = {
|
|
52
|
+
type: 'block';
|
|
48
53
|
block: L2Block;
|
|
49
54
|
resolve: () => void;
|
|
50
55
|
reject: (err: Error) => void;
|
|
51
56
|
};
|
|
52
57
|
|
|
58
|
+
/** Request to add a proposed checkpoint to the archiver, queued for processing by the sync loop. */
|
|
59
|
+
type AddProposedCheckpointRequest = {
|
|
60
|
+
type: 'checkpoint';
|
|
61
|
+
checkpoint: ProposedCheckpointInput;
|
|
62
|
+
resolve: () => void;
|
|
63
|
+
reject: (err: Error) => void;
|
|
64
|
+
};
|
|
65
|
+
|
|
53
66
|
export type ArchiverDeps = {
|
|
54
67
|
telemetry?: TelemetryClient;
|
|
55
68
|
blobClient: BlobClientInterface;
|
|
@@ -75,8 +88,8 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
75
88
|
private initialSyncComplete: boolean = false;
|
|
76
89
|
private initialSyncPromise: PromiseWithResolvers<void>;
|
|
77
90
|
|
|
78
|
-
/** Queue of blocks to be added to the store, processed by the sync loop. */
|
|
79
|
-
private
|
|
91
|
+
/** Queue of blocks and checkpoints to be added to the store, processed by the sync loop. */
|
|
92
|
+
private inboundQueue: (AddBlockRequest | AddProposedCheckpointRequest)[] = [];
|
|
80
93
|
|
|
81
94
|
/** Helper to handle updates to the store */
|
|
82
95
|
private readonly updater: ArchiverDataStoreUpdater;
|
|
@@ -86,20 +99,27 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
86
99
|
|
|
87
100
|
public readonly tracer: Tracer;
|
|
88
101
|
|
|
102
|
+
private readonly instrumentation: ArchiverInstrumentation;
|
|
103
|
+
|
|
89
104
|
/**
|
|
90
105
|
* Creates a new instance of the Archiver.
|
|
91
106
|
* @param publicClient - A client for interacting with the Ethereum node.
|
|
92
107
|
* @param debugClient - A client for interacting with the Ethereum node for debug/trace methods.
|
|
93
108
|
* @param rollup - Rollup contract instance.
|
|
94
109
|
* @param inbox - Inbox contract instance.
|
|
95
|
-
* @param l1Addresses - L1 contract addresses (registry, governance proposer,
|
|
96
|
-
* @param
|
|
110
|
+
* @param l1Addresses - L1 contract addresses (registry, governance proposer, slashing proposer).
|
|
111
|
+
* @param dataStores - Archiver substores for storage & retrieval of blocks, encrypted logs & contract data.
|
|
97
112
|
* @param config - Archiver configuration options.
|
|
98
113
|
* @param blobClient - Client for retrieving blob data.
|
|
99
|
-
* @param epochCache - Cache for epoch-related data.
|
|
100
|
-
* @param dateProvider - Provider for current date/time.
|
|
101
114
|
* @param instrumentation - Instrumentation for metrics and tracing.
|
|
102
115
|
* @param l1Constants - L1 rollup constants.
|
|
116
|
+
* @param synchronizer - L1 synchronizer that handles fetching checkpoints and messages from L1.
|
|
117
|
+
* @param events - Event emitter shared with the synchronizer.
|
|
118
|
+
* @param initialHeader - Genesis block header.
|
|
119
|
+
* @param initialBlockHash - Precomputed hash of the genesis block header.
|
|
120
|
+
* @param l2TipsCache - In-memory cache for L2 chain tips.
|
|
121
|
+
* @param epochCache - Cache used to compute the proposer pipelining offset.
|
|
122
|
+
* @param dateProvider - Provider for current date/time, used for wall-clock orphan-block pruning.
|
|
103
123
|
* @param log - A logger.
|
|
104
124
|
*/
|
|
105
125
|
constructor(
|
|
@@ -108,36 +128,44 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
108
128
|
private readonly rollup: RollupContract,
|
|
109
129
|
private readonly l1Addresses: Pick<
|
|
110
130
|
L1ContractAddresses,
|
|
111
|
-
'registryAddress' | '
|
|
112
|
-
> & {
|
|
113
|
-
|
|
131
|
+
'rollupAddress' | 'registryAddress' | 'inboxAddress' | 'governanceProposerAddress'
|
|
132
|
+
> & {
|
|
133
|
+
slashingProposerAddress: EthAddress;
|
|
134
|
+
},
|
|
135
|
+
readonly dataStores: ArchiverDataStores,
|
|
114
136
|
private config: {
|
|
115
137
|
pollingIntervalMs: number;
|
|
116
138
|
batchSize: number;
|
|
117
139
|
skipValidateCheckpointAttestations?: boolean;
|
|
118
140
|
maxAllowedEthClientDriftSeconds: number;
|
|
119
141
|
ethereumAllowNoDebugHosts?: boolean;
|
|
142
|
+
skipHistoricalLogsCheck?: boolean;
|
|
143
|
+
orphanProposedBlockPruneGraceSeconds: number;
|
|
120
144
|
},
|
|
121
145
|
private readonly blobClient: BlobClientInterface,
|
|
122
146
|
instrumentation: ArchiverInstrumentation,
|
|
123
147
|
protected override readonly l1Constants: L1RollupConstants & {
|
|
124
148
|
l1StartBlockHash: Buffer32;
|
|
125
149
|
genesisArchiveRoot: Fr;
|
|
126
|
-
rollupManaLimit?: number;
|
|
127
150
|
},
|
|
128
151
|
synchronizer: ArchiverL1Synchronizer,
|
|
129
152
|
events: ArchiverEmitter,
|
|
130
|
-
|
|
153
|
+
initialHeader: BlockHeader,
|
|
154
|
+
initialBlockHash: BlockHash,
|
|
155
|
+
l2TipsCache: L2TipsCache,
|
|
156
|
+
private readonly epochCache: EpochCache,
|
|
157
|
+
private readonly dateProvider: DateProvider,
|
|
131
158
|
private readonly log: Logger = createLogger('archiver'),
|
|
132
159
|
) {
|
|
133
|
-
super(
|
|
160
|
+
super(dataStores, l1Constants, initialHeader, initialBlockHash, l1Constants.genesisArchiveRoot);
|
|
134
161
|
|
|
135
162
|
this.tracer = instrumentation.tracer;
|
|
163
|
+
this.instrumentation = instrumentation;
|
|
136
164
|
this.initialSyncPromise = promiseWithResolvers();
|
|
137
165
|
this.synchronizer = synchronizer;
|
|
138
166
|
this.events = events;
|
|
139
|
-
this.l2TipsCache = l2TipsCache
|
|
140
|
-
this.updater = new ArchiverDataStoreUpdater(this.
|
|
167
|
+
this.l2TipsCache = l2TipsCache;
|
|
168
|
+
this.updater = new ArchiverDataStoreUpdater(this.dataStores, this.l2TipsCache, {
|
|
141
169
|
rollupManaLimit: l1Constants.rollupManaLimit,
|
|
142
170
|
});
|
|
143
171
|
|
|
@@ -173,11 +201,24 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
173
201
|
this.config.ethereumAllowNoDebugHosts ?? false,
|
|
174
202
|
this.log.getBindings(),
|
|
175
203
|
);
|
|
204
|
+
await validateAndLogHistoricalLogsAvailability(
|
|
205
|
+
this.publicClient,
|
|
206
|
+
{
|
|
207
|
+
rollupAddress: this.l1Addresses.rollupAddress,
|
|
208
|
+
inboxAddress: this.l1Addresses.inboxAddress,
|
|
209
|
+
registryAddress: this.l1Addresses.registryAddress,
|
|
210
|
+
governanceProposerAddress: this.l1Addresses.governanceProposerAddress,
|
|
211
|
+
},
|
|
212
|
+
this.config.skipHistoricalLogsCheck ?? false,
|
|
213
|
+
this.log.getBindings(),
|
|
214
|
+
);
|
|
176
215
|
|
|
177
216
|
// Log initial state for the archiver
|
|
178
217
|
const { l1StartBlock } = this.l1Constants;
|
|
179
|
-
const { blocksSynchedTo = l1StartBlock, messagesSynchedTo = l1StartBlock } = await
|
|
180
|
-
|
|
218
|
+
const { blocksSynchedTo = l1StartBlock, messagesSynchedTo = l1StartBlock } = await getArchiverSynchPoint(
|
|
219
|
+
this.stores,
|
|
220
|
+
);
|
|
221
|
+
const currentL2Checkpoint = await this.getCheckpointNumber();
|
|
181
222
|
this.log.info(
|
|
182
223
|
`Starting archiver sync to rollup contract ${this.rollup.address} from L1 block ${blocksSynchedTo} and L2 checkpoint ${currentL2Checkpoint}`,
|
|
183
224
|
{ blocksSynchedTo, messagesSynchedTo, currentL2Checkpoint },
|
|
@@ -194,6 +235,14 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
194
235
|
return this.runningPromise.trigger();
|
|
195
236
|
}
|
|
196
237
|
|
|
238
|
+
public trySyncImmediate() {
|
|
239
|
+
try {
|
|
240
|
+
return this.syncImmediate();
|
|
241
|
+
} catch (err) {
|
|
242
|
+
this.log.error(`Failed to trigger immediate archiver sync: ${err}`, err);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
197
246
|
/**
|
|
198
247
|
* Queues a block to be added to the archiver store and triggers processing.
|
|
199
248
|
* The block will be processed by the sync loop.
|
|
@@ -202,53 +251,85 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
202
251
|
* @returns A promise that resolves when the block has been added to the store, or rejects on error.
|
|
203
252
|
*/
|
|
204
253
|
public addBlock(block: L2Block): Promise<void> {
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
this.log.error(`Sync immediate call failed: ${err}`);
|
|
211
|
-
});
|
|
212
|
-
});
|
|
254
|
+
const promise = promiseWithResolvers<void>();
|
|
255
|
+
this.inboundQueue.push({ block, ...promise, type: 'block' });
|
|
256
|
+
this.log.debug(`Queued block ${block.number} for processing`);
|
|
257
|
+
void this.trySyncImmediate();
|
|
258
|
+
return promise.promise;
|
|
213
259
|
}
|
|
214
260
|
|
|
215
261
|
/**
|
|
216
|
-
*
|
|
262
|
+
* Queues a new proposed checkpoint into the archiver store.
|
|
263
|
+
* Checks that the checkpoint is not for an L2 slot already synced from L1.
|
|
264
|
+
* Resolves once the checkpoint has been processed.
|
|
265
|
+
*/
|
|
266
|
+
public addProposedCheckpoint(pending: ProposedCheckpointInput): Promise<void> {
|
|
267
|
+
const promise = promiseWithResolvers<void>();
|
|
268
|
+
this.inboundQueue.push({ checkpoint: pending, ...promise, type: 'checkpoint' });
|
|
269
|
+
this.log.debug(`Queued checkpoint ${pending.checkpointNumber} for processing`);
|
|
270
|
+
void this.trySyncImmediate();
|
|
271
|
+
return promise.promise;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Processes all queued blocks and checkpoints, adding them to the store.
|
|
217
276
|
* Called at the beginning of each sync iteration.
|
|
218
|
-
*
|
|
277
|
+
* Items are processed in the order they were queued.
|
|
219
278
|
*/
|
|
220
|
-
private async
|
|
221
|
-
if (this.
|
|
279
|
+
private async processInboundQueue(): Promise<void> {
|
|
280
|
+
if (this.inboundQueue.length === 0) {
|
|
222
281
|
return;
|
|
223
282
|
}
|
|
224
283
|
|
|
225
|
-
// Take all
|
|
226
|
-
const queuedItems = this.
|
|
227
|
-
this.log.debug(`Processing ${queuedItems.length} queued
|
|
284
|
+
// Take all items from the queue
|
|
285
|
+
const queuedItems = this.inboundQueue.splice(0, this.inboundQueue.length);
|
|
286
|
+
this.log.debug(`Processing ${queuedItems.length} queued inbound items`);
|
|
228
287
|
|
|
229
288
|
// Calculate slot threshold for validation
|
|
230
289
|
const l1Timestamp = this.synchronizer.getL1Timestamp();
|
|
231
290
|
const slotAtNextL1Block =
|
|
232
291
|
l1Timestamp === undefined ? undefined : getSlotAtNextL1Block(l1Timestamp, this.l1Constants);
|
|
233
292
|
|
|
234
|
-
//
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
293
|
+
// Helpers for manipulating blocks and checkpoints in the queue
|
|
294
|
+
const getSlot: (item: AddBlockRequest | AddProposedCheckpointRequest) => SlotNumber = item =>
|
|
295
|
+
item.type === 'block' ? item.block.header.globalVariables.slotNumber : item.checkpoint.header.slotNumber;
|
|
296
|
+
const getNumber: (item: AddBlockRequest | AddProposedCheckpointRequest) => number = item =>
|
|
297
|
+
item.type === 'block' ? item.block.number : item.checkpoint.checkpointNumber;
|
|
298
|
+
|
|
299
|
+
// Process each item individually to properly resolve/reject each promise
|
|
300
|
+
for (const item of queuedItems) {
|
|
301
|
+
const { resolve, reject, type } = item;
|
|
302
|
+
const itemSlot = getSlot(item);
|
|
303
|
+
const itemNumber = getNumber(item);
|
|
304
|
+
if (slotAtNextL1Block !== undefined && itemSlot < slotAtNextL1Block) {
|
|
305
|
+
const nextSlotTimestamp = getTimestampForSlot(slotAtNextL1Block, this.l1Constants);
|
|
238
306
|
this.log.warn(
|
|
239
|
-
`Rejecting proposed
|
|
240
|
-
{
|
|
307
|
+
`Rejecting proposed ${type} ${itemNumber} for past slot ${itemSlot} (current ${slotAtNextL1Block})`,
|
|
308
|
+
{ number: itemNumber, type, l1Timestamp, slotAtNextL1Block, nextSlotTimestamp },
|
|
241
309
|
);
|
|
242
|
-
reject(new
|
|
310
|
+
reject(new BlockOrCheckpointSlotExpiredError(itemSlot, nextSlotTimestamp, l1Timestamp));
|
|
243
311
|
continue;
|
|
244
312
|
}
|
|
245
313
|
|
|
246
314
|
try {
|
|
247
|
-
|
|
248
|
-
|
|
315
|
+
if (type === 'block') {
|
|
316
|
+
const [durationMs] = await elapsed(() => this.updater.addProposedBlock(item.block));
|
|
317
|
+
this.instrumentation.processNewProposedBlock(durationMs, item.block);
|
|
318
|
+
} else {
|
|
319
|
+
await this.updater.addProposedCheckpoint(item.checkpoint);
|
|
320
|
+
}
|
|
321
|
+
this.log.debug(`Added ${type} ${itemNumber} to store`);
|
|
249
322
|
resolve();
|
|
250
323
|
} catch (err: any) {
|
|
251
|
-
|
|
324
|
+
if (err instanceof BlockAlreadyCheckpointedError) {
|
|
325
|
+
this.log.debug(`Proposed block ${itemNumber} matches already checkpointed block, ignoring late proposal`);
|
|
326
|
+
resolve();
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
this.log.error(`Failed to add ${type} ${itemNumber} to store: ${err.message}`, err, {
|
|
330
|
+
number: itemNumber,
|
|
331
|
+
type,
|
|
332
|
+
});
|
|
252
333
|
reject(err);
|
|
253
334
|
}
|
|
254
335
|
}
|
|
@@ -264,9 +345,104 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
264
345
|
@trackSpan('Archiver.sync')
|
|
265
346
|
private async sync() {
|
|
266
347
|
// Process any queued blocks first, before doing L1 sync
|
|
267
|
-
await this.
|
|
348
|
+
await this.processInboundQueue();
|
|
268
349
|
// Now perform L1 sync
|
|
269
350
|
await this.syncFromL1();
|
|
351
|
+
// Prune proposed blocks with no corresponding proposed checkpoint by the end of their build slot.
|
|
352
|
+
await this.pruneOrphanProposedBlocks();
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* Prunes a block-only local tip that was built atop a checkpoint that was never itself proposed.
|
|
357
|
+
*
|
|
358
|
+
* Under pipelining, a proposer publishes the blocks for a checkpoint (block-only proposals) before
|
|
359
|
+
* assembling and publishing the enclosing proposed checkpoint at the end of the build slot. A node
|
|
360
|
+
* that received those blocks but never the proposed checkpoint is left with an orphan tip it must
|
|
361
|
+
* not build on. We prune it once enough wall-clock time has elapsed that the proposed checkpoint
|
|
362
|
+
* should have arrived. This runs on wall-clock time (not L1 block advancement) so it fires during
|
|
363
|
+
* quiet L1 periods, and is the liveness counterpart to the sequencer's checkSync guard.
|
|
364
|
+
*
|
|
365
|
+
* The uncheckpointed suffix is scanned in order. Blocks covered by proposed checkpoints are left in
|
|
366
|
+
* place; the first block not covered by a proposed checkpoint starts the orphan suffix to prune.
|
|
367
|
+
*/
|
|
368
|
+
private async pruneOrphanProposedBlocks(): Promise<void> {
|
|
369
|
+
const tips = await this.getL2Tips();
|
|
370
|
+
const now = BigInt(this.dateProvider.nowInSeconds());
|
|
371
|
+
const pipeliningOffset = this.epochCache.pipeliningOffset();
|
|
372
|
+
|
|
373
|
+
// This only applies under pipelining
|
|
374
|
+
if (pipeliningOffset === 0) {
|
|
375
|
+
this.log.trace(`Pipelining offset is 0, skipping orphan proposed block pruning`);
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// The proposed tip is a proposed-checkpointed block, so there are no orphan proposed blocks to prune
|
|
380
|
+
if (tips.proposedCheckpoint.block.number === tips.proposed.number) {
|
|
381
|
+
this.log.trace(
|
|
382
|
+
`No orphan proposed blocks to prune: proposed tip ${tips.proposed.number} is checkpointed`,
|
|
383
|
+
pick(tips, 'proposed', 'proposedCheckpoint'),
|
|
384
|
+
);
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// Load the blocks that are candidates for pruning (ie blocks without a proposed checkpoint covering them)
|
|
389
|
+
const blocksWithoutProposedCheckpoint = await this.stores.blocks.getBlocksData({
|
|
390
|
+
from: BlockNumber(tips.proposedCheckpoint.block.number + 1),
|
|
391
|
+
limit: tips.proposed.number - tips.proposedCheckpoint.block.number,
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
// Iterate through them in order, the first one with a slot that should have received a proposed checkpoint
|
|
395
|
+
// is the first orphan block, and all blocks after it are also orphaned and should be pruned.
|
|
396
|
+
let lastSlotChecked = undefined;
|
|
397
|
+
for (const blockData of blocksWithoutProposedCheckpoint) {
|
|
398
|
+
// No need to recheck if this block had the same slot as the previous one.
|
|
399
|
+
const blockSlot = blockData.header.getSlot();
|
|
400
|
+
const blockNumber = blockData.header.getBlockNumber();
|
|
401
|
+
if (lastSlotChecked !== undefined && blockSlot === lastSlotChecked) {
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
lastSlotChecked = blockSlot;
|
|
405
|
+
|
|
406
|
+
// The proposed checkpoint should have landed by the start of the slot after the block's build slot
|
|
407
|
+
// (build slot = blockSlot - pipeliningOffset). Wait a grace period beyond that to tolerate propagation.
|
|
408
|
+
const expectedCheckpointedBySlot = SlotNumber(Number(blockSlot) - pipeliningOffset + 1);
|
|
409
|
+
const expectedCheckpointedByTime =
|
|
410
|
+
getTimestampForSlot(expectedCheckpointedBySlot, this.l1Constants) +
|
|
411
|
+
BigInt(this.config.orphanProposedBlockPruneGraceSeconds);
|
|
412
|
+
|
|
413
|
+
// If it's not checkpointed by the expected time, prune it along with all blocks after it.
|
|
414
|
+
if (now >= expectedCheckpointedByTime) {
|
|
415
|
+
const pruneAfterBlockNumber = BlockNumber(blockNumber - 1);
|
|
416
|
+
this.log.warn(
|
|
417
|
+
`Pruning orphan blocks after block ${pruneAfterBlockNumber}: block at slot ${blockSlot} belongs to ` +
|
|
418
|
+
`checkpoint ${blockData.checkpointNumber} which has no matching proposed checkpoint`,
|
|
419
|
+
{
|
|
420
|
+
firstUncheckpointedBlockHeader: blockData.header.toInspect(),
|
|
421
|
+
blockCheckpointNumber: blockData.checkpointNumber,
|
|
422
|
+
blockNumber,
|
|
423
|
+
blockSlot,
|
|
424
|
+
pipeliningOffset,
|
|
425
|
+
expectedCheckpointedBySlot,
|
|
426
|
+
expectedCheckpointedByTime,
|
|
427
|
+
now,
|
|
428
|
+
},
|
|
429
|
+
);
|
|
430
|
+
|
|
431
|
+
const prunedBlocks = await this.updater.removeBlocksWithoutProposedCheckpointAfter(pruneAfterBlockNumber);
|
|
432
|
+
if (prunedBlocks.length > 0) {
|
|
433
|
+
this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
|
|
434
|
+
type: L2BlockSourceEvents.L2PruneUncheckpointed,
|
|
435
|
+
slotNumber: blockSlot,
|
|
436
|
+
blocks: prunedBlocks,
|
|
437
|
+
});
|
|
438
|
+
}
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
this.log.trace('No orphan proposed blocks to prune: all uncheckpointed blocks are still within the grace period', {
|
|
444
|
+
blocksWithoutProposedCheckpoint: blocksWithoutProposedCheckpoint.map(b => b.header.toInspect()),
|
|
445
|
+
});
|
|
270
446
|
}
|
|
271
447
|
|
|
272
448
|
private async syncFromL1() {
|
|
@@ -280,7 +456,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
280
456
|
if (currentL1BlockNumber + 1n >= l1BlockNumberAtEnd) {
|
|
281
457
|
this.log.info(`Initial archiver sync to L1 block ${currentL1BlockNumber} complete`, {
|
|
282
458
|
l1BlockNumber: currentL1BlockNumber,
|
|
283
|
-
syncPoint: await this.
|
|
459
|
+
syncPoint: await getArchiverSynchPoint(this.stores),
|
|
284
460
|
...(await this.getL2Tips()),
|
|
285
461
|
});
|
|
286
462
|
this.runningPromise.setPollingIntervalMS(this.config.pollingIntervalMs);
|
|
@@ -312,7 +488,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
312
488
|
}
|
|
313
489
|
|
|
314
490
|
public backupTo(destPath: string): Promise<string> {
|
|
315
|
-
return this.
|
|
491
|
+
return backupArchiverDataStores(this.dataStores, destPath);
|
|
316
492
|
}
|
|
317
493
|
|
|
318
494
|
public getL1Constants(): Promise<L1RollupConstants> {
|
|
@@ -339,24 +515,56 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
339
515
|
return Promise.resolve(this.synchronizer.getL1Timestamp());
|
|
340
516
|
}
|
|
341
517
|
|
|
342
|
-
public
|
|
518
|
+
public async getSyncedL2SlotNumber(): Promise<SlotNumber | undefined> {
|
|
519
|
+
// The synced L2 slot is the latest slot for which we have all L1 data,
|
|
520
|
+
// either because we have seen all L1 blocks for that slot, or because
|
|
521
|
+
// we have seen the corresponding checkpoint.
|
|
522
|
+
|
|
523
|
+
let slotFromL1Sync: SlotNumber | undefined;
|
|
343
524
|
const l1Timestamp = this.synchronizer.getL1Timestamp();
|
|
344
|
-
|
|
525
|
+
if (l1Timestamp !== undefined) {
|
|
526
|
+
const nextL1BlockSlot = getSlotAtNextL1Block(l1Timestamp, this.l1Constants);
|
|
527
|
+
if (Number(nextL1BlockSlot) > 0) {
|
|
528
|
+
slotFromL1Sync = SlotNumber.add(nextL1BlockSlot, -1);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
let slotFromCheckpoint: SlotNumber | undefined;
|
|
533
|
+
const latestCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
|
|
534
|
+
if (latestCheckpointNumber > 0) {
|
|
535
|
+
const checkpointData = await this.stores.blocks.getCheckpointData(latestCheckpointNumber);
|
|
536
|
+
if (checkpointData) {
|
|
537
|
+
slotFromCheckpoint = checkpointData.header.slotNumber;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
if (slotFromL1Sync === undefined && slotFromCheckpoint === undefined) {
|
|
542
|
+
return undefined;
|
|
543
|
+
}
|
|
544
|
+
return SlotNumber(Math.max(slotFromL1Sync ?? 0, slotFromCheckpoint ?? 0));
|
|
345
545
|
}
|
|
346
546
|
|
|
347
|
-
public
|
|
348
|
-
const
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
547
|
+
public async getSyncedL2EpochNumber(): Promise<EpochNumber | undefined> {
|
|
548
|
+
const syncedSlot = await this.getSyncedL2SlotNumber();
|
|
549
|
+
if (syncedSlot === undefined) {
|
|
550
|
+
return undefined;
|
|
551
|
+
}
|
|
552
|
+
// An epoch is fully synced when all its slots are synced.
|
|
553
|
+
// We check if syncedSlot is the last slot of its epoch; if so, that epoch is fully synced.
|
|
554
|
+
// Otherwise, only the previous epoch is fully synced.
|
|
555
|
+
const epoch = getEpochAtSlot(syncedSlot, this.l1Constants);
|
|
556
|
+
const [, endSlot] = getSlotRangeForEpoch(epoch, this.l1Constants);
|
|
557
|
+
if (syncedSlot >= endSlot) {
|
|
558
|
+
return epoch;
|
|
559
|
+
}
|
|
560
|
+
return Number(epoch) > 0 ? EpochNumber(Number(epoch) - 1) : undefined;
|
|
352
561
|
}
|
|
353
562
|
|
|
354
563
|
public async isEpochComplete(epochNumber: EpochNumber): Promise<boolean> {
|
|
355
564
|
// The epoch is complete if the current checkpointed L2 block is the last one in the epoch (or later).
|
|
356
565
|
// We use the checkpointed block number (synced from L1) instead of 'latest' to avoid returning true
|
|
357
566
|
// prematurely when proposed blocks have been pushed to the archiver but not yet checkpointed on L1.
|
|
358
|
-
const
|
|
359
|
-
const header = checkpointedBlockNumber > 0 ? await this.getBlockHeader(checkpointedBlockNumber) : undefined;
|
|
567
|
+
const header = (await this.getBlockData({ tag: 'checkpointed' }))?.header;
|
|
360
568
|
const slot = header ? header.globalVariables.slotNumber : undefined;
|
|
361
569
|
const [_startSlot, endSlot] = getSlotRangeForEpoch(epochNumber, this.l1Constants);
|
|
362
570
|
if (slot && slot >= endSlot) {
|
|
@@ -412,29 +620,34 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
412
620
|
if (targetL2BlockNumber >= currentL2Block) {
|
|
413
621
|
throw new Error(`Target L2 block ${targetL2BlockNumber} must be less than current L2 block ${currentL2Block}`);
|
|
414
622
|
}
|
|
415
|
-
const
|
|
416
|
-
if (
|
|
623
|
+
const checkpointedTip = await this.stores.blocks.getCheckpointedL2BlockNumber();
|
|
624
|
+
if (targetL2BlockNumber > checkpointedTip) {
|
|
625
|
+
throw new Error(`Target L2 block ${targetL2BlockNumber} is not checkpointed yet`);
|
|
626
|
+
}
|
|
627
|
+
const targetBlockData = await this.stores.blocks.getBlockData({ number: targetL2BlockNumber });
|
|
628
|
+
if (!targetBlockData) {
|
|
417
629
|
throw new Error(`Target L2 block ${targetL2BlockNumber} not found`);
|
|
418
630
|
}
|
|
419
|
-
const targetCheckpointNumber =
|
|
631
|
+
const targetCheckpointNumber = targetBlockData.checkpointNumber;
|
|
420
632
|
|
|
421
633
|
// Rollback operates at checkpoint granularity: the target block must be the last block of its checkpoint.
|
|
422
|
-
const checkpointData = await this.
|
|
423
|
-
if (checkpointData) {
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
634
|
+
const checkpointData = await this.stores.blocks.getCheckpointData(targetCheckpointNumber);
|
|
635
|
+
if (!checkpointData) {
|
|
636
|
+
throw new Error(`Checkpoint ${targetCheckpointNumber} not found for block ${targetL2BlockNumber}`);
|
|
637
|
+
}
|
|
638
|
+
const lastBlockInCheckpoint = BlockNumber(checkpointData.startBlock + checkpointData.blockCount - 1);
|
|
639
|
+
if (targetL2BlockNumber !== lastBlockInCheckpoint) {
|
|
640
|
+
const previousCheckpointBoundary =
|
|
641
|
+
checkpointData.startBlock > 1 ? BlockNumber(checkpointData.startBlock - 1) : BlockNumber(0);
|
|
642
|
+
throw new Error(
|
|
643
|
+
`Target L2 block ${targetL2BlockNumber} is not at a checkpoint boundary. ` +
|
|
644
|
+
`Checkpoint ${targetCheckpointNumber} spans blocks ${checkpointData.startBlock} to ${lastBlockInCheckpoint}. ` +
|
|
645
|
+
`Use block ${lastBlockInCheckpoint} to roll back to this checkpoint, ` +
|
|
646
|
+
`or block ${previousCheckpointBoundary} to roll back to the previous one.`,
|
|
647
|
+
);
|
|
435
648
|
}
|
|
436
649
|
|
|
437
|
-
const targetL1BlockNumber =
|
|
650
|
+
const targetL1BlockNumber = checkpointData.l1.blockNumber;
|
|
438
651
|
const targetL1Block = await this.publicClient.getBlock({
|
|
439
652
|
blockNumber: targetL1BlockNumber,
|
|
440
653
|
includeTransactions: false,
|
|
@@ -448,19 +661,21 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra
|
|
|
448
661
|
);
|
|
449
662
|
await this.updater.removeCheckpointsAfter(targetCheckpointNumber);
|
|
450
663
|
this.log.info(`Rolling back L1 to L2 messages to checkpoint ${targetCheckpointNumber}`);
|
|
451
|
-
await this.
|
|
664
|
+
await this.stores.messages.rollbackL1ToL2MessagesToCheckpoint(targetCheckpointNumber);
|
|
452
665
|
this.log.info(`Setting L1 syncpoints to ${targetL1BlockNumber}`);
|
|
453
|
-
await this.
|
|
454
|
-
await this.
|
|
666
|
+
await this.stores.blocks.setSynchedL1BlockNumber(targetL1BlockNumber);
|
|
667
|
+
await this.stores.messages.setMessageSyncState(
|
|
668
|
+
{ l1BlockNumber: targetL1BlockNumber, l1BlockHash: targetL1BlockHash },
|
|
669
|
+
undefined,
|
|
670
|
+
);
|
|
455
671
|
if (targetL2BlockNumber < currentProvenBlock) {
|
|
456
672
|
this.log.info(`Rolling back proven L2 checkpoint to ${targetCheckpointNumber}`);
|
|
457
673
|
await this.updater.setProvenCheckpointNumber(targetCheckpointNumber);
|
|
458
674
|
}
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
// }
|
|
675
|
+
const currentFinalizedBlock = currentBlocks.finalized.block.number;
|
|
676
|
+
if (targetL2BlockNumber < currentFinalizedBlock) {
|
|
677
|
+
this.log.info(`Rolling back finalized L2 checkpoint to ${targetCheckpointNumber}`);
|
|
678
|
+
await this.updater.setFinalizedCheckpointNumber(targetCheckpointNumber);
|
|
679
|
+
}
|
|
465
680
|
}
|
|
466
681
|
}
|