@aztec/archiver 0.0.1-commit.3fd054f6 → 0.0.1-commit.431c48d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -13
- package/dest/archiver.d.ts +71 -15
- package/dest/archiver.d.ts.map +1 -1
- package/dest/archiver.js +275 -70
- package/dest/config.d.ts +7 -3
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +44 -16
- package/dest/errors.d.ts +44 -10
- package/dest/errors.d.ts.map +1 -1
- package/dest/errors.js +66 -12
- package/dest/factory.d.ts +18 -6
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +66 -21
- package/dest/index.d.ts +11 -3
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +10 -2
- package/dest/l1/calldata_retriever.d.ts +9 -1
- package/dest/l1/calldata_retriever.d.ts.map +1 -1
- package/dest/l1/calldata_retriever.js +22 -11
- package/dest/l1/data_retrieval.d.ts +25 -11
- package/dest/l1/data_retrieval.d.ts.map +1 -1
- package/dest/l1/data_retrieval.js +25 -32
- package/dest/l1/trace_tx.d.ts +12 -66
- package/dest/l1/trace_tx.d.ts.map +1 -1
- package/dest/l1/validate_historical_logs.d.ts +23 -0
- package/dest/l1/validate_historical_logs.d.ts.map +1 -0
- package/dest/l1/validate_historical_logs.js +108 -0
- package/dest/modules/contract_data_source_adapter.d.ts +25 -0
- package/dest/modules/contract_data_source_adapter.d.ts.map +1 -0
- package/dest/modules/contract_data_source_adapter.js +32 -0
- package/dest/modules/data_source_base.d.ts +71 -48
- package/dest/modules/data_source_base.d.ts.map +1 -1
- package/dest/modules/data_source_base.js +250 -144
- package/dest/modules/data_store_updater.d.ts +28 -8
- package/dest/modules/data_store_updater.d.ts.map +1 -1
- package/dest/modules/data_store_updater.js +132 -58
- package/dest/modules/instrumentation.d.ts +15 -2
- package/dest/modules/instrumentation.d.ts.map +1 -1
- package/dest/modules/instrumentation.js +48 -9
- package/dest/modules/l1_synchronizer.d.ts +13 -8
- package/dest/modules/l1_synchronizer.d.ts.map +1 -1
- package/dest/modules/l1_synchronizer.js +453 -219
- package/dest/modules/outbox_trees_resolver.d.ts +64 -0
- package/dest/modules/outbox_trees_resolver.d.ts.map +1 -0
- package/dest/modules/outbox_trees_resolver.js +184 -0
- package/dest/modules/validation.d.ts +30 -7
- package/dest/modules/validation.d.ts.map +1 -1
- package/dest/modules/validation.js +35 -17
- package/dest/store/block_store.d.ts +177 -68
- package/dest/store/block_store.d.ts.map +1 -1
- package/dest/store/block_store.js +719 -246
- package/dest/store/contract_class_store.d.ts +17 -3
- package/dest/store/contract_class_store.d.ts.map +1 -1
- package/dest/store/contract_class_store.js +36 -2
- package/dest/store/contract_instance_store.d.ts +28 -1
- package/dest/store/contract_instance_store.d.ts.map +1 -1
- package/dest/store/contract_instance_store.js +53 -5
- package/dest/store/data_stores.d.ts +68 -0
- package/dest/store/data_stores.d.ts.map +1 -0
- package/dest/store/data_stores.js +54 -0
- package/dest/store/function_names_cache.d.ts +17 -0
- package/dest/store/function_names_cache.d.ts.map +1 -0
- package/dest/store/function_names_cache.js +30 -0
- package/dest/store/l2_tips_cache.d.ts +13 -7
- package/dest/store/l2_tips_cache.d.ts.map +1 -1
- package/dest/store/l2_tips_cache.js +13 -76
- package/dest/store/log_store.d.ts +42 -40
- package/dest/store/log_store.d.ts.map +1 -1
- package/dest/store/log_store.js +257 -480
- package/dest/store/log_store_codec.d.ts +78 -0
- package/dest/store/log_store_codec.d.ts.map +1 -0
- package/dest/store/log_store_codec.js +110 -0
- package/dest/store/message_store.d.ts +9 -3
- package/dest/store/message_store.d.ts.map +1 -1
- package/dest/store/message_store.js +39 -10
- package/dest/test/fake_l1_state.d.ts +15 -3
- package/dest/test/fake_l1_state.d.ts.map +1 -1
- package/dest/test/fake_l1_state.js +83 -21
- package/dest/test/mock_l2_block_source.d.ts +48 -50
- package/dest/test/mock_l2_block_source.d.ts.map +1 -1
- package/dest/test/mock_l2_block_source.js +228 -179
- package/dest/test/mock_structs.d.ts +1 -1
- package/dest/test/mock_structs.d.ts.map +1 -1
- package/dest/test/mock_structs.js +2 -2
- package/dest/test/noop_l1_archiver.d.ts +14 -6
- package/dest/test/noop_l1_archiver.d.ts.map +1 -1
- package/dest/test/noop_l1_archiver.js +26 -10
- package/package.json +15 -14
- package/src/archiver.ts +361 -76
- package/src/config.ts +51 -20
- package/src/errors.ts +103 -18
- package/src/factory.ts +80 -19
- package/src/index.ts +18 -2
- package/src/l1/calldata_retriever.ts +29 -11
- package/src/l1/data_retrieval.ts +44 -46
- package/src/l1/validate_historical_logs.ts +140 -0
- package/src/modules/contract_data_source_adapter.ts +46 -0
- package/src/modules/data_source_base.ts +325 -192
- package/src/modules/data_store_updater.ts +162 -58
- package/src/modules/instrumentation.ts +43 -10
- package/src/modules/l1_synchronizer.ts +612 -259
- package/src/modules/outbox_trees_resolver.ts +220 -0
- package/src/modules/validation.ts +80 -23
- package/src/store/block_store.ts +879 -300
- package/src/store/contract_class_store.ts +47 -3
- package/src/store/contract_instance_store.ts +67 -7
- package/src/store/data_stores.ts +104 -0
- package/src/store/function_names_cache.ts +37 -0
- package/src/store/l2_tips_cache.ts +16 -70
- package/src/store/log_store.ts +296 -652
- package/src/store/log_store_codec.ts +143 -0
- package/src/store/message_store.ts +43 -12
- package/src/structs/inbox_message.ts +1 -1
- package/src/test/fake_l1_state.ts +102 -30
- package/src/test/mock_l2_block_source.ts +285 -232
- package/src/test/mock_structs.ts +5 -2
- package/src/test/noop_l1_archiver.ts +54 -11
- package/dest/store/kv_archiver_store.d.ts +0 -364
- package/dest/store/kv_archiver_store.d.ts.map +0 -1
- package/dest/store/kv_archiver_store.js +0 -477
- package/src/store/kv_archiver_store.ts +0 -682
|
@@ -6,17 +6,23 @@ import {
|
|
|
6
6
|
ContractInstancePublishedEvent,
|
|
7
7
|
ContractInstanceUpdatedEvent,
|
|
8
8
|
} from '@aztec/protocol-contracts/instance-registry';
|
|
9
|
-
import type { L2Block, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
10
|
-
import {
|
|
9
|
+
import type { CommitteeAttestation, L2Block, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
10
|
+
import {
|
|
11
|
+
type L1PublishedData,
|
|
12
|
+
type ProposedCheckpointInput,
|
|
13
|
+
type PublishedCheckpoint,
|
|
14
|
+
validateCheckpoint,
|
|
15
|
+
} from '@aztec/stdlib/checkpoint';
|
|
11
16
|
import {
|
|
12
17
|
type ContractClassPublicWithCommitment,
|
|
13
18
|
computeContractAddressFromInstance,
|
|
14
19
|
computeContractClassId,
|
|
15
20
|
} from '@aztec/stdlib/contract';
|
|
21
|
+
import { MAX_CAPACITY_BLOCKS_PER_CHECKPOINT } from '@aztec/stdlib/deserialization';
|
|
16
22
|
import type { ContractClassLog, PrivateLog, PublicLog } from '@aztec/stdlib/logs';
|
|
17
23
|
import type { UInt64 } from '@aztec/stdlib/types';
|
|
18
24
|
|
|
19
|
-
import type {
|
|
25
|
+
import type { ArchiverDataStores } from '../store/data_stores.js';
|
|
20
26
|
import type { L2TipsCache } from '../store/l2_tips_cache.js';
|
|
21
27
|
|
|
22
28
|
/** Operation type for contract data updates. */
|
|
@@ -38,7 +44,7 @@ export class ArchiverDataStoreUpdater {
|
|
|
38
44
|
private readonly log = createLogger('archiver:store_updater');
|
|
39
45
|
|
|
40
46
|
constructor(
|
|
41
|
-
private
|
|
47
|
+
private stores: ArchiverDataStores,
|
|
42
48
|
private l2TipsCache?: L2TipsCache,
|
|
43
49
|
private opts: { rollupManaLimit?: number } = {},
|
|
44
50
|
) {}
|
|
@@ -56,21 +62,22 @@ export class ArchiverDataStoreUpdater {
|
|
|
56
62
|
block: L2Block,
|
|
57
63
|
pendingChainValidationStatus?: ValidateCheckpointResult,
|
|
58
64
|
): Promise<boolean> {
|
|
59
|
-
const result = await this.
|
|
60
|
-
await this.
|
|
65
|
+
const result = await this.stores.db.transactionAsync(async () => {
|
|
66
|
+
await this.stores.blocks.addProposedBlock(block);
|
|
61
67
|
|
|
62
68
|
const opResults = await Promise.all([
|
|
63
69
|
// Update the pending chain validation status if provided
|
|
64
|
-
pendingChainValidationStatus &&
|
|
70
|
+
pendingChainValidationStatus &&
|
|
71
|
+
this.stores.blocks.setPendingChainValidationStatus(pendingChainValidationStatus),
|
|
65
72
|
// Add any logs emitted during the retrieved block
|
|
66
|
-
this.
|
|
73
|
+
this.stores.logs.addLogs([block]),
|
|
67
74
|
// Unroll all logs emitted during the retrieved block and extract any contract classes and instances from it
|
|
68
75
|
this.addContractDataToDb(block),
|
|
69
76
|
]);
|
|
70
77
|
|
|
71
|
-
await this.l2TipsCache?.refresh();
|
|
72
78
|
return opResults.every(Boolean);
|
|
73
79
|
});
|
|
80
|
+
await this.l2TipsCache?.refresh();
|
|
74
81
|
return result;
|
|
75
82
|
}
|
|
76
83
|
|
|
@@ -79,55 +86,106 @@ export class ArchiverDataStoreUpdater {
|
|
|
79
86
|
* Adds new checkpoints to the store with contract class/instance extraction from logs.
|
|
80
87
|
* Prunes any local blocks that conflict with checkpoint data (by comparing archive roots).
|
|
81
88
|
* Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events from the checkpoint block logs.
|
|
89
|
+
* If `promoteProposed` is supplied, the proposed-checkpoint promotion runs inside the same transaction
|
|
90
|
+
* as the added checkpoints so both updates are applied atomically.
|
|
82
91
|
*
|
|
83
|
-
* @param checkpoints - The published checkpoints to add.
|
|
92
|
+
* @param checkpoints - The published checkpoints to add (excluding any being promoted from proposed).
|
|
84
93
|
* @param pendingChainValidationStatus - Optional validation status to set.
|
|
94
|
+
* @param promoteProposed - Optional promotion of the current proposed checkpoint (fast path when blocks are already local).
|
|
85
95
|
* @returns Result with information about any pruned blocks.
|
|
86
96
|
*/
|
|
87
97
|
public async addCheckpoints(
|
|
88
98
|
checkpoints: PublishedCheckpoint[],
|
|
89
99
|
pendingChainValidationStatus?: ValidateCheckpointResult,
|
|
100
|
+
promoteProposed?: {
|
|
101
|
+
l1: L1PublishedData;
|
|
102
|
+
attestations: CommitteeAttestation[];
|
|
103
|
+
checkpoint: PublishedCheckpoint;
|
|
104
|
+
},
|
|
105
|
+
evictProposedFrom?: CheckpointNumber,
|
|
90
106
|
): Promise<ReconcileCheckpointsResult> {
|
|
107
|
+
// These checkpoints are already on L1, so ingest validates against the physical blob capacity rather than
|
|
108
|
+
// the conservative build/attest policy, and tolerates an empty non-first block. Both rules are enforced by
|
|
109
|
+
// proposers and attesters; rejecting an ingest would stall sync rather than undo the checkpoint.
|
|
110
|
+
const validateOpts = {
|
|
111
|
+
rollupManaLimit: this.opts?.rollupManaLimit,
|
|
112
|
+
maxBlocksPerCheckpoint: MAX_CAPACITY_BLOCKS_PER_CHECKPOINT,
|
|
113
|
+
allowEmptyNonFirstBlocks: true,
|
|
114
|
+
};
|
|
91
115
|
for (const checkpoint of checkpoints) {
|
|
92
|
-
validateCheckpoint(checkpoint.checkpoint,
|
|
116
|
+
validateCheckpoint(checkpoint.checkpoint, validateOpts);
|
|
117
|
+
}
|
|
118
|
+
if (promoteProposed) {
|
|
119
|
+
validateCheckpoint(promoteProposed.checkpoint.checkpoint, validateOpts);
|
|
93
120
|
}
|
|
94
121
|
|
|
95
|
-
const result = await this.
|
|
122
|
+
const result = await this.stores.db.transactionAsync(async () => {
|
|
96
123
|
// Before adding checkpoints, check for conflicts with local blocks if any
|
|
97
124
|
const { prunedBlocks, lastAlreadyInsertedBlockNumber } = await this.pruneMismatchingLocalBlocks(checkpoints);
|
|
98
125
|
|
|
99
|
-
await this.
|
|
126
|
+
const insertedCheckpoints = await this.stores.blocks.addCheckpoints(checkpoints);
|
|
100
127
|
|
|
101
|
-
//
|
|
102
|
-
|
|
128
|
+
// Skip blocks already inserted via addProposedBlock() and blocks of already-stored checkpoints
|
|
129
|
+
// re-included by an L1 reorg: their logs/contract data were extracted when first inserted.
|
|
130
|
+
const newBlocks = insertedCheckpoints
|
|
103
131
|
.flatMap((ch: PublishedCheckpoint) => ch.checkpoint.blocks)
|
|
104
132
|
.filter(b => lastAlreadyInsertedBlockNumber === undefined || b.number > lastAlreadyInsertedBlockNumber);
|
|
105
133
|
|
|
106
134
|
await Promise.all([
|
|
107
135
|
// Update the pending chain validation status if provided
|
|
108
|
-
pendingChainValidationStatus &&
|
|
136
|
+
pendingChainValidationStatus &&
|
|
137
|
+
this.stores.blocks.setPendingChainValidationStatus(pendingChainValidationStatus),
|
|
109
138
|
// Add any logs emitted during the retrieved blocks
|
|
110
|
-
this.
|
|
139
|
+
this.stores.logs.addLogs(newBlocks),
|
|
111
140
|
// Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
|
|
112
141
|
...newBlocks.map(block => this.addContractDataToDb(block)),
|
|
142
|
+
// Promote the proposed checkpoint if requested (uses explicit checkpoint number)
|
|
143
|
+
promoteProposed
|
|
144
|
+
? this.stores.blocks.promoteProposedToCheckpointed(
|
|
145
|
+
promoteProposed.checkpoint.checkpoint.number,
|
|
146
|
+
promoteProposed.l1,
|
|
147
|
+
promoteProposed.attestations,
|
|
148
|
+
promoteProposed.checkpoint.checkpoint.archive.root,
|
|
149
|
+
)
|
|
150
|
+
: undefined,
|
|
151
|
+
// Evict pending checkpoints that diverged from what L1 mined
|
|
152
|
+
evictProposedFrom !== undefined
|
|
153
|
+
? this.stores.blocks.evictProposedCheckpointsFrom(evictProposedFrom)
|
|
154
|
+
: undefined,
|
|
113
155
|
]);
|
|
114
156
|
|
|
115
|
-
await this.l2TipsCache?.refresh();
|
|
116
157
|
return { prunedBlocks, lastAlreadyInsertedBlockNumber };
|
|
117
158
|
});
|
|
159
|
+
await this.l2TipsCache?.refresh();
|
|
160
|
+
return result;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
public async addProposedCheckpoint(proposedCheckpoint: ProposedCheckpointInput) {
|
|
164
|
+
const result = await this.stores.db.transactionAsync(async () => {
|
|
165
|
+
await this.stores.blocks.addProposedCheckpoint(proposedCheckpoint);
|
|
166
|
+
});
|
|
167
|
+
await this.l2TipsCache?.refresh();
|
|
118
168
|
return result;
|
|
119
169
|
}
|
|
120
170
|
|
|
121
171
|
/**
|
|
122
172
|
* Checks for local proposed blocks that do not match the ones to be checkpointed and prunes them.
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
173
|
+
* Conflict detection is keyed on `blockNumber`: when a local proposed block and an L1
|
|
174
|
+
* checkpointed block share a block number but live at different slots (e.g. a different proposer
|
|
175
|
+
* mined the same block number one slot earlier), we still treat them as a conflict and prune.
|
|
176
|
+
* The trailing per-checkpoint prune that handles "local has extra trailing blocks within the
|
|
177
|
+
* same slot as the published checkpoint" remains scoped by slot to preserve pipelining: local
|
|
178
|
+
* blocks that live at a later slot than the checkpoint being processed represent speculation
|
|
179
|
+
* atop the just-confirmed tip (and may be referenced by a pending proposed checkpoint), so we
|
|
180
|
+
* leave them in place. This method handles multiple checkpoints but returns after pruning the
|
|
181
|
+
* first conflict found. This is correct because pruning from the first conflict point removes
|
|
182
|
+
* all subsequent blocks, and when checkpoints are added afterward, they include all the correct
|
|
183
|
+
* blocks.
|
|
126
184
|
*/
|
|
127
185
|
private async pruneMismatchingLocalBlocks(checkpoints: PublishedCheckpoint[]): Promise<ReconcileCheckpointsResult> {
|
|
128
186
|
const [lastCheckpointedBlockNumber, lastBlockNumber] = await Promise.all([
|
|
129
|
-
this.
|
|
130
|
-
this.
|
|
187
|
+
this.stores.blocks.getCheckpointedL2BlockNumber(),
|
|
188
|
+
this.stores.blocks.getLatestL2BlockNumber(),
|
|
131
189
|
]);
|
|
132
190
|
|
|
133
191
|
// Exit early if there are no local uncheckpointed blocks
|
|
@@ -136,29 +194,29 @@ export class ArchiverDataStoreUpdater {
|
|
|
136
194
|
}
|
|
137
195
|
|
|
138
196
|
// Get all uncheckpointed local blocks
|
|
139
|
-
const uncheckpointedLocalBlocks = await this.
|
|
140
|
-
BlockNumber.add(lastCheckpointedBlockNumber, 1),
|
|
141
|
-
lastBlockNumber - lastCheckpointedBlockNumber,
|
|
142
|
-
);
|
|
197
|
+
const uncheckpointedLocalBlocks = await this.stores.blocks.getBlocksData({
|
|
198
|
+
from: BlockNumber.add(lastCheckpointedBlockNumber, 1),
|
|
199
|
+
limit: lastBlockNumber - lastCheckpointedBlockNumber,
|
|
200
|
+
});
|
|
143
201
|
|
|
144
202
|
let lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
|
|
145
203
|
|
|
146
204
|
for (const publishedCheckpoint of checkpoints) {
|
|
147
205
|
const checkpointBlocks = publishedCheckpoint.checkpoint.blocks;
|
|
148
206
|
const slot = publishedCheckpoint.checkpoint.slot;
|
|
149
|
-
const localBlocksInSlot = uncheckpointedLocalBlocks.filter(b => b.slot === slot);
|
|
150
207
|
|
|
151
208
|
if (checkpointBlocks.length === 0) {
|
|
152
209
|
this.log.warn(`Checkpoint ${publishedCheckpoint.checkpoint.number} for slot ${slot} has no blocks`);
|
|
153
210
|
continue;
|
|
154
211
|
}
|
|
155
212
|
|
|
156
|
-
// Find the first checkpoint block that conflicts with an existing local block and prune local afterwards
|
|
213
|
+
// Find the first checkpoint block that conflicts with an existing local block and prune local afterwards.
|
|
214
|
+
// Conflict detection joins on block number only — same block number at a different slot is still a conflict.
|
|
157
215
|
for (const checkpointBlock of checkpointBlocks) {
|
|
158
216
|
const blockNumber = checkpointBlock.number;
|
|
159
|
-
const existingBlock =
|
|
217
|
+
const existingBlock = uncheckpointedLocalBlocks.find(b => b.header.getBlockNumber() === blockNumber);
|
|
160
218
|
const blockInfos = {
|
|
161
|
-
existingBlock: existingBlock?.
|
|
219
|
+
existingBlock: existingBlock?.header.toInspect(),
|
|
162
220
|
checkpointBlock: checkpointBlock.toBlockInfo(),
|
|
163
221
|
};
|
|
164
222
|
|
|
@@ -170,20 +228,24 @@ export class ArchiverDataStoreUpdater {
|
|
|
170
228
|
} else {
|
|
171
229
|
this.log.info(`Conflict detected at block ${blockNumber} between checkpointed and local block`, blockInfos);
|
|
172
230
|
const prunedBlocks = await this.removeBlocksAfter(BlockNumber(blockNumber - 1));
|
|
231
|
+
await this.evictProposedCheckpointsForPrunedBlocks(prunedBlocks);
|
|
173
232
|
return { prunedBlocks, lastAlreadyInsertedBlockNumber };
|
|
174
233
|
}
|
|
175
234
|
}
|
|
176
235
|
|
|
177
|
-
// If
|
|
178
|
-
//
|
|
236
|
+
// If the sequencer locally proposed extra blocks within this checkpoint's slot (e.g. local has
|
|
237
|
+
// [N, N+1] but L1 confirmed just [N]), prune the extras. Scoped to the checkpoint's slot so we
|
|
238
|
+
// do not throw away speculative blocks at later slots that belong to a pending proposed checkpoint.
|
|
179
239
|
const lastCheckpointBlockNumber = checkpointBlocks.at(-1)!.number;
|
|
180
|
-
const
|
|
240
|
+
const localBlocksInSlot = uncheckpointedLocalBlocks.filter(b => b.header.getSlot() === slot);
|
|
241
|
+
const lastLocalBlockNumber = localBlocksInSlot.at(-1)?.header.getBlockNumber();
|
|
181
242
|
|
|
182
243
|
if (lastLocalBlockNumber !== undefined && lastLocalBlockNumber > lastCheckpointBlockNumber) {
|
|
183
244
|
this.log.warn(
|
|
184
245
|
`Local chain for slot ${slot} ends at block ${lastLocalBlockNumber} but checkpoint ends at ${lastCheckpointBlockNumber}. Pruning blocks after block ${lastCheckpointBlockNumber}.`,
|
|
185
246
|
);
|
|
186
247
|
const prunedBlocks = await this.removeBlocksAfter(lastCheckpointBlockNumber);
|
|
248
|
+
await this.evictProposedCheckpointsForPrunedBlocks(prunedBlocks);
|
|
187
249
|
return { prunedBlocks, lastAlreadyInsertedBlockNumber };
|
|
188
250
|
}
|
|
189
251
|
}
|
|
@@ -201,33 +263,74 @@ export class ArchiverDataStoreUpdater {
|
|
|
201
263
|
* @throws Error if any block to be removed is checkpointed.
|
|
202
264
|
*/
|
|
203
265
|
public async removeUncheckpointedBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
|
|
204
|
-
const result = await this.
|
|
266
|
+
const result = await this.stores.db.transactionAsync(async () => {
|
|
205
267
|
// Verify we're only removing uncheckpointed blocks
|
|
206
|
-
const lastCheckpointedBlockNumber = await this.
|
|
268
|
+
const lastCheckpointedBlockNumber = await this.stores.blocks.getCheckpointedL2BlockNumber();
|
|
207
269
|
if (blockNumber < lastCheckpointedBlockNumber) {
|
|
208
270
|
throw new Error(
|
|
209
271
|
`Cannot remove blocks after ${blockNumber} because checkpointed blocks exist up to ${lastCheckpointedBlockNumber}`,
|
|
210
272
|
);
|
|
211
273
|
}
|
|
212
274
|
|
|
213
|
-
const
|
|
214
|
-
await this.
|
|
215
|
-
|
|
275
|
+
const prunedBlocks = await this.removeBlocksAfter(blockNumber);
|
|
276
|
+
await this.evictProposedCheckpointsForPrunedBlocks(prunedBlocks);
|
|
277
|
+
|
|
278
|
+
return prunedBlocks;
|
|
216
279
|
});
|
|
280
|
+
await this.l2TipsCache?.refresh();
|
|
217
281
|
return result;
|
|
218
282
|
}
|
|
219
283
|
|
|
284
|
+
/**
|
|
285
|
+
* Removes all blocks without a proposed checkpoint strictly after the specified block number and cleans up associated contract data.
|
|
286
|
+
* This handles removal of provisionally added blocks along with their contract classes/instances.
|
|
287
|
+
* Verifies that each block being removed is not part of a stored checkpoint (proposed or not).
|
|
288
|
+
* This differs from `removeUncheckpointedBlocksAfter` in that it also checks proposed checkpoints.
|
|
289
|
+
*
|
|
290
|
+
* @param blockNumber - Remove all blocks with number greater than this.
|
|
291
|
+
* @returns The removed blocks.
|
|
292
|
+
* @throws Error if any block to be removed is checkpointed.
|
|
293
|
+
*/
|
|
294
|
+
public async removeBlocksWithoutProposedCheckpointAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
|
|
295
|
+
const result = await this.stores.db.transactionAsync(async () => {
|
|
296
|
+
// Verify we're only removing uncheckpointed blocks
|
|
297
|
+
const lastCheckpointedBlockNumber = await this.stores.blocks.getProposedCheckpointL2BlockNumber();
|
|
298
|
+
if (blockNumber < lastCheckpointedBlockNumber) {
|
|
299
|
+
throw new Error(
|
|
300
|
+
`Cannot remove blocks after ${blockNumber} because proposed checkpointed blocks exist up to ${lastCheckpointedBlockNumber}`,
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
return await this.removeBlocksAfter(blockNumber);
|
|
305
|
+
});
|
|
306
|
+
await this.l2TipsCache?.refresh();
|
|
307
|
+
return result;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Evicts pending proposed checkpoints that referenced any of the just-pruned blocks. Pruned
|
|
312
|
+
* blocks invalidate all proposed checkpoints from the lowest pruned block's checkpoint number
|
|
313
|
+
* onwards: those checkpoints either reference the pruned blocks directly or chain off them.
|
|
314
|
+
*/
|
|
315
|
+
private async evictProposedCheckpointsForPrunedBlocks(prunedBlocks: L2Block[]): Promise<void> {
|
|
316
|
+
if (prunedBlocks.length === 0) {
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
const fromCheckpointNumber = prunedBlocks[0].checkpointNumber;
|
|
320
|
+
await this.stores.blocks.evictProposedCheckpointsFrom(fromCheckpointNumber);
|
|
321
|
+
}
|
|
322
|
+
|
|
220
323
|
/**
|
|
221
324
|
* Removes all blocks strictly after the given block number along with any logs and contract data.
|
|
222
325
|
* Does not remove their checkpoints.
|
|
223
326
|
*/
|
|
224
327
|
private async removeBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
|
|
225
328
|
// First get the blocks to be removed so we can clean up contract data
|
|
226
|
-
const removedBlocks = await this.
|
|
329
|
+
const removedBlocks = await this.stores.blocks.removeBlocksAfter(blockNumber);
|
|
227
330
|
|
|
228
331
|
// Clean up contract data and logs for the removed blocks
|
|
229
332
|
await Promise.all([
|
|
230
|
-
this.
|
|
333
|
+
this.stores.logs.deleteLogs(removedBlocks),
|
|
231
334
|
...removedBlocks.map(block => this.removeContractDataFromDb(block)),
|
|
232
335
|
]);
|
|
233
336
|
|
|
@@ -244,20 +347,21 @@ export class ArchiverDataStoreUpdater {
|
|
|
244
347
|
* @returns True if the operation is successful.
|
|
245
348
|
*/
|
|
246
349
|
public async removeCheckpointsAfter(checkpointNumber: CheckpointNumber): Promise<boolean> {
|
|
247
|
-
|
|
248
|
-
const { blocksRemoved = [] } = await this.
|
|
350
|
+
const result = await this.stores.db.transactionAsync(async () => {
|
|
351
|
+
const { blocksRemoved = [] } = await this.stores.blocks.removeCheckpointsAfter(checkpointNumber);
|
|
249
352
|
|
|
250
353
|
const opResults = await Promise.all([
|
|
251
354
|
// Prune rolls back to the last proven block, which is by definition valid
|
|
252
|
-
this.
|
|
355
|
+
this.stores.blocks.setPendingChainValidationStatus({ valid: true }),
|
|
253
356
|
// Remove contract data for all blocks being removed
|
|
254
357
|
...blocksRemoved.map(block => this.removeContractDataFromDb(block)),
|
|
255
|
-
this.
|
|
358
|
+
this.stores.logs.deleteLogs(blocksRemoved),
|
|
256
359
|
]);
|
|
257
360
|
|
|
258
|
-
await this.l2TipsCache?.refresh();
|
|
259
361
|
return opResults.every(Boolean);
|
|
260
362
|
});
|
|
363
|
+
await this.l2TipsCache?.refresh();
|
|
364
|
+
return result;
|
|
261
365
|
}
|
|
262
366
|
|
|
263
367
|
/**
|
|
@@ -265,10 +369,10 @@ export class ArchiverDataStoreUpdater {
|
|
|
265
369
|
* @param checkpointNumber - The checkpoint number to set as proven.
|
|
266
370
|
*/
|
|
267
371
|
public async setProvenCheckpointNumber(checkpointNumber: CheckpointNumber): Promise<void> {
|
|
268
|
-
await this.
|
|
269
|
-
await this.
|
|
270
|
-
await this.l2TipsCache?.refresh();
|
|
372
|
+
await this.stores.db.transactionAsync(async () => {
|
|
373
|
+
await this.stores.blocks.setProvenCheckpointNumber(checkpointNumber);
|
|
271
374
|
});
|
|
375
|
+
await this.l2TipsCache?.refresh();
|
|
272
376
|
}
|
|
273
377
|
|
|
274
378
|
/**
|
|
@@ -276,10 +380,10 @@ export class ArchiverDataStoreUpdater {
|
|
|
276
380
|
* @param checkpointNumber - The checkpoint number to set as finalized.
|
|
277
381
|
*/
|
|
278
382
|
public async setFinalizedCheckpointNumber(checkpointNumber: CheckpointNumber): Promise<void> {
|
|
279
|
-
await this.
|
|
280
|
-
await this.
|
|
281
|
-
await this.l2TipsCache?.refresh();
|
|
383
|
+
await this.stores.db.transactionAsync(async () => {
|
|
384
|
+
await this.stores.blocks.setFinalizedCheckpointNumber(checkpointNumber);
|
|
282
385
|
});
|
|
386
|
+
await this.l2TipsCache?.refresh();
|
|
283
387
|
}
|
|
284
388
|
|
|
285
389
|
/** Extracts and stores contract data from a single block. */
|
|
@@ -323,7 +427,7 @@ export class ArchiverDataStoreUpdater {
|
|
|
323
427
|
const contractClasses = contractClassPublishedEvents.map(e => e.toContractClassPublic());
|
|
324
428
|
if (contractClasses.length > 0) {
|
|
325
429
|
contractClasses.forEach(c => this.log.verbose(`${Operation[operation]} contract class ${c.id.toString()}`));
|
|
326
|
-
return await this.
|
|
430
|
+
return await this.stores.contractClasses.deleteContractClasses(contractClasses, blockNum);
|
|
327
431
|
}
|
|
328
432
|
return true;
|
|
329
433
|
}
|
|
@@ -349,7 +453,7 @@ export class ArchiverDataStoreUpdater {
|
|
|
349
453
|
|
|
350
454
|
if (contractClasses.length > 0) {
|
|
351
455
|
contractClasses.forEach(c => this.log.verbose(`${Operation[operation]} contract class ${c.id.toString()}`));
|
|
352
|
-
return await this.
|
|
456
|
+
return await this.stores.contractClasses.addContractClasses(contractClasses, blockNum);
|
|
353
457
|
}
|
|
354
458
|
return true;
|
|
355
459
|
}
|
|
@@ -388,9 +492,9 @@ export class ArchiverDataStoreUpdater {
|
|
|
388
492
|
this.log.verbose(`${Operation[operation]} contract instance at ${c.address.toString()}`),
|
|
389
493
|
);
|
|
390
494
|
if (operation == Operation.Store) {
|
|
391
|
-
return await this.
|
|
495
|
+
return await this.stores.contractInstances.addContractInstances(contractInstances, blockNum);
|
|
392
496
|
} else if (operation == Operation.Delete) {
|
|
393
|
-
return await this.
|
|
497
|
+
return await this.stores.contractInstances.deleteContractInstances(contractInstances);
|
|
394
498
|
}
|
|
395
499
|
}
|
|
396
500
|
return true;
|
|
@@ -414,9 +518,9 @@ export class ArchiverDataStoreUpdater {
|
|
|
414
518
|
this.log.verbose(`${Operation[operation]} contract instance update at ${c.address.toString()}`),
|
|
415
519
|
);
|
|
416
520
|
if (operation == Operation.Store) {
|
|
417
|
-
return await this.
|
|
521
|
+
return await this.stores.contractInstances.addContractInstanceUpdates(contractUpdates, timestamp);
|
|
418
522
|
} else if (operation == Operation.Delete) {
|
|
419
|
-
return await this.
|
|
523
|
+
return await this.stores.contractInstances.deleteContractInstanceUpdates(contractUpdates, timestamp);
|
|
420
524
|
}
|
|
421
525
|
}
|
|
422
526
|
return true;
|
|
@@ -32,6 +32,7 @@ export class ArchiverInstrumentation {
|
|
|
32
32
|
private pruneCount: UpDownCounter;
|
|
33
33
|
|
|
34
34
|
private syncDurationPerBlock: Histogram;
|
|
35
|
+
private syncDurationPerCheckpoint: Histogram;
|
|
35
36
|
private syncBlockCount: UpDownCounter;
|
|
36
37
|
private manaPerBlock: Histogram;
|
|
37
38
|
private txsPerBlock: Histogram;
|
|
@@ -42,6 +43,7 @@ export class ArchiverInstrumentation {
|
|
|
42
43
|
private blockProposalTxTargetCount: UpDownCounter;
|
|
43
44
|
|
|
44
45
|
private checkpointL1InclusionDelay: Histogram;
|
|
46
|
+
private checkpointPromotedCount: UpDownCounter;
|
|
45
47
|
|
|
46
48
|
private log = createLogger('archiver:instrumentation');
|
|
47
49
|
|
|
@@ -68,6 +70,8 @@ export class ArchiverInstrumentation {
|
|
|
68
70
|
|
|
69
71
|
this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK);
|
|
70
72
|
|
|
73
|
+
this.syncDurationPerCheckpoint = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_CHECKPOINT);
|
|
74
|
+
|
|
71
75
|
this.syncBlockCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
|
|
72
76
|
|
|
73
77
|
this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK);
|
|
@@ -80,7 +84,9 @@ export class ArchiverInstrumentation {
|
|
|
80
84
|
|
|
81
85
|
this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION);
|
|
82
86
|
|
|
83
|
-
this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT
|
|
87
|
+
this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT, {
|
|
88
|
+
[Attributes.PRUNE_TYPE]: ['unproven', 'uncheckpointed', 'l1_conflict', 'orphan', 'l1_mismatch'],
|
|
89
|
+
});
|
|
84
90
|
|
|
85
91
|
this.blockProposalTxTargetCount = createUpDownCounterWithDefault(
|
|
86
92
|
meter,
|
|
@@ -92,6 +98,8 @@ export class ArchiverInstrumentation {
|
|
|
92
98
|
|
|
93
99
|
this.checkpointL1InclusionDelay = meter.createHistogram(Metrics.ARCHIVER_CHECKPOINT_L1_INCLUSION_DELAY);
|
|
94
100
|
|
|
101
|
+
this.checkpointPromotedCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_CHECKPOINT_PROMOTED_COUNT);
|
|
102
|
+
|
|
95
103
|
this.dbMetrics = new LmdbMetrics(
|
|
96
104
|
meter,
|
|
97
105
|
{
|
|
@@ -113,17 +121,26 @@ export class ArchiverInstrumentation {
|
|
|
113
121
|
return this.telemetry.isEnabled();
|
|
114
122
|
}
|
|
115
123
|
|
|
116
|
-
public
|
|
124
|
+
public processNewProposedBlock(syncTimePerBlock: number, block: L2Block) {
|
|
125
|
+
const attrs = { [Attributes.STATUS]: 'proposed' };
|
|
126
|
+
this.blockHeight.record(block.number, attrs);
|
|
117
127
|
this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock));
|
|
118
|
-
this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
|
|
119
|
-
this.checkpointHeight.record(Math.max(...blocks.map(b => b.checkpointNumber)));
|
|
120
|
-
this.syncBlockCount.add(blocks.length);
|
|
121
128
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
129
|
+
// Per block metrics
|
|
130
|
+
this.txCount.add(block.body.txEffects.length);
|
|
131
|
+
this.txsPerBlock.record(block.body.txEffects.length);
|
|
132
|
+
this.manaPerBlock.record(block.header.totalManaUsed.toNumber() / 1e6);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
public processNewCheckpointedBlocks(syncTimePerCheckpoint: number, blocks: L2Block[]) {
|
|
136
|
+
if (blocks.length === 0) {
|
|
137
|
+
return;
|
|
126
138
|
}
|
|
139
|
+
|
|
140
|
+
this.syncDurationPerCheckpoint.record(Math.ceil(syncTimePerCheckpoint));
|
|
141
|
+
this.blockHeight.record(Math.max(...blocks.map(b => b.number)), { [Attributes.STATUS]: 'checkpointed' });
|
|
142
|
+
this.checkpointHeight.record(Math.max(...blocks.map(b => b.checkpointNumber)));
|
|
143
|
+
this.syncBlockCount.add(blocks.length);
|
|
127
144
|
}
|
|
128
145
|
|
|
129
146
|
public processNewMessages(count: number, syncPerMessageMs: number) {
|
|
@@ -135,10 +152,21 @@ export class ArchiverInstrumentation {
|
|
|
135
152
|
}
|
|
136
153
|
|
|
137
154
|
public processPrune(duration: number) {
|
|
138
|
-
this.pruneCount.add(1);
|
|
155
|
+
this.pruneCount.add(1, { [Attributes.PRUNE_TYPE]: 'unproven' });
|
|
139
156
|
this.pruneDuration.record(Math.ceil(duration));
|
|
140
157
|
}
|
|
141
158
|
|
|
159
|
+
/**
|
|
160
|
+
* Records a pending-chain reorg, where the archiver dropped proposed blocks (and world-state follows by pruning). The
|
|
161
|
+
* type distinguishes the cause: 'uncheckpointed' (slot ended without a checkpoint), 'l1_conflict' (proposed blocks
|
|
162
|
+
* conflicting with an L1 checkpoint), 'orphan' (no matching proposed checkpoint arrived before the deadline), or
|
|
163
|
+
* 'l1_mismatch' (the local checkpointed tip diverged from L1 — an L1 reorg or a pruned/missed-proof checkpoint — so
|
|
164
|
+
* already-checkpointed blocks were rewound).
|
|
165
|
+
*/
|
|
166
|
+
public recordPrune(pruneType: 'uncheckpointed' | 'l1_conflict' | 'orphan' | 'l1_mismatch') {
|
|
167
|
+
this.pruneCount.add(1, { [Attributes.PRUNE_TYPE]: pruneType });
|
|
168
|
+
}
|
|
169
|
+
|
|
142
170
|
public updateLastProvenCheckpoint(checkpoint: CheckpointData) {
|
|
143
171
|
const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.blockCount - 1;
|
|
144
172
|
this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' });
|
|
@@ -169,6 +197,11 @@ export class ArchiverInstrumentation {
|
|
|
169
197
|
});
|
|
170
198
|
}
|
|
171
199
|
|
|
200
|
+
/** Records a checkpoint promoted from proposed (blob fetch skipped). */
|
|
201
|
+
public processCheckpointPromoted() {
|
|
202
|
+
this.checkpointPromotedCount.add(1);
|
|
203
|
+
}
|
|
204
|
+
|
|
172
205
|
/**
|
|
173
206
|
* Records L1 inclusion timing for a checkpoint observed on L1 (seconds into the L2 slot).
|
|
174
207
|
*/
|