@aztec/archiver 0.0.1-commit.d3ec352c → 0.0.1-commit.f295ac2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +147 -22
- package/dest/archiver.d.ts +135 -0
- package/dest/archiver.d.ts.map +1 -0
- package/dest/archiver.js +769 -0
- package/dest/config.d.ts +30 -0
- package/dest/config.d.ts.map +1 -0
- package/dest/{archiver/config.js → config.js} +21 -5
- package/dest/errors.d.ts +36 -0
- package/dest/errors.d.ts.map +1 -0
- package/dest/errors.js +54 -0
- package/dest/factory.d.ts +5 -6
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +82 -5
- package/dest/index.d.ts +10 -4
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +8 -3
- package/dest/interfaces.d.ts +9 -0
- package/dest/interfaces.d.ts.map +1 -0
- package/dest/interfaces.js +3 -0
- package/dest/l1/bin/retrieve-calldata.d.ts +3 -0
- package/dest/l1/bin/retrieve-calldata.d.ts.map +1 -0
- package/dest/l1/bin/retrieve-calldata.js +149 -0
- package/dest/l1/calldata_retriever.d.ts +112 -0
- package/dest/l1/calldata_retriever.d.ts.map +1 -0
- package/dest/l1/calldata_retriever.js +471 -0
- package/dest/l1/data_retrieval.d.ts +88 -0
- package/dest/l1/data_retrieval.d.ts.map +1 -0
- package/dest/{archiver → l1}/data_retrieval.js +75 -150
- package/dest/l1/debug_tx.d.ts +19 -0
- package/dest/l1/debug_tx.d.ts.map +1 -0
- package/dest/l1/debug_tx.js +73 -0
- package/dest/l1/spire_proposer.d.ts +70 -0
- package/dest/l1/spire_proposer.d.ts.map +1 -0
- package/dest/l1/spire_proposer.js +157 -0
- package/dest/l1/trace_tx.d.ts +97 -0
- package/dest/l1/trace_tx.d.ts.map +1 -0
- package/dest/l1/trace_tx.js +91 -0
- package/dest/l1/types.d.ts +12 -0
- package/dest/l1/types.d.ts.map +1 -0
- package/dest/l1/types.js +3 -0
- package/dest/l1/validate_trace.d.ts +29 -0
- package/dest/l1/validate_trace.d.ts.map +1 -0
- package/dest/l1/validate_trace.js +150 -0
- package/dest/modules/data_source_base.d.ts +85 -0
- package/dest/modules/data_source_base.d.ts.map +1 -0
- package/dest/modules/data_source_base.js +291 -0
- package/dest/modules/data_store_updater.d.ts +69 -0
- package/dest/modules/data_store_updater.d.ts.map +1 -0
- package/dest/modules/data_store_updater.js +304 -0
- package/dest/modules/instrumentation.d.ts +37 -0
- package/dest/modules/instrumentation.d.ts.map +1 -0
- package/dest/{archiver → modules}/instrumentation.js +22 -59
- package/dest/modules/l1_synchronizer.d.ts +75 -0
- package/dest/modules/l1_synchronizer.d.ts.map +1 -0
- package/dest/modules/l1_synchronizer.js +1113 -0
- package/dest/modules/validation.d.ts +17 -0
- package/dest/modules/validation.d.ts.map +1 -0
- package/dest/{archiver → modules}/validation.js +7 -1
- package/dest/store/block_store.d.ts +178 -0
- package/dest/store/block_store.d.ts.map +1 -0
- package/dest/store/block_store.js +680 -0
- package/dest/store/contract_class_store.d.ts +18 -0
- package/dest/store/contract_class_store.d.ts.map +1 -0
- package/dest/{archiver/kv_archiver_store → store}/contract_class_store.js +2 -2
- package/dest/store/contract_instance_store.d.ts +24 -0
- package/dest/store/contract_instance_store.d.ts.map +1 -0
- package/dest/{archiver/kv_archiver_store → store}/contract_instance_store.js +1 -1
- package/dest/store/kv_archiver_store.d.ts +331 -0
- package/dest/store/kv_archiver_store.d.ts.map +1 -0
- package/dest/store/kv_archiver_store.js +438 -0
- package/dest/store/log_store.d.ts +45 -0
- package/dest/store/log_store.d.ts.map +1 -0
- package/dest/store/log_store.js +422 -0
- package/dest/store/message_store.d.ts +40 -0
- package/dest/store/message_store.d.ts.map +1 -0
- package/dest/{archiver/kv_archiver_store → store}/message_store.js +15 -14
- package/dest/{archiver/structs → structs}/data_retrieval.d.ts +1 -1
- package/dest/structs/data_retrieval.d.ts.map +1 -0
- package/dest/structs/inbox_message.d.ts +15 -0
- package/dest/structs/inbox_message.d.ts.map +1 -0
- package/dest/{archiver/structs → structs}/inbox_message.js +6 -6
- package/dest/structs/published.d.ts +2 -0
- package/dest/structs/published.d.ts.map +1 -0
- package/dest/test/fake_l1_state.d.ts +190 -0
- package/dest/test/fake_l1_state.d.ts.map +1 -0
- package/dest/test/fake_l1_state.js +383 -0
- package/dest/test/index.d.ts +2 -1
- package/dest/test/index.d.ts.map +1 -1
- package/dest/test/index.js +1 -0
- package/dest/test/mock_archiver.d.ts +5 -6
- package/dest/test/mock_archiver.d.ts.map +1 -1
- package/dest/test/mock_archiver.js +6 -11
- package/dest/test/mock_l1_to_l2_message_source.d.ts +6 -7
- package/dest/test/mock_l1_to_l2_message_source.d.ts.map +1 -1
- package/dest/test/mock_l1_to_l2_message_source.js +19 -14
- package/dest/test/mock_l2_block_source.d.ts +28 -14
- package/dest/test/mock_l2_block_source.d.ts.map +1 -1
- package/dest/test/mock_l2_block_source.js +160 -52
- package/dest/test/mock_structs.d.ts +78 -3
- package/dest/test/mock_structs.d.ts.map +1 -1
- package/dest/test/mock_structs.js +141 -10
- package/package.json +17 -18
- package/src/archiver.ts +525 -0
- package/src/{archiver/config.ts → config.ts} +28 -12
- package/src/errors.ts +90 -0
- package/src/factory.ts +118 -6
- package/src/index.ts +10 -3
- package/src/interfaces.ts +9 -0
- package/src/l1/README.md +98 -0
- package/src/l1/bin/retrieve-calldata.ts +182 -0
- package/src/l1/calldata_retriever.ts +641 -0
- package/src/{archiver → l1}/data_retrieval.ts +136 -218
- package/src/l1/debug_tx.ts +99 -0
- package/src/l1/spire_proposer.ts +160 -0
- package/src/l1/trace_tx.ts +128 -0
- package/src/l1/types.ts +13 -0
- package/src/l1/validate_trace.ts +211 -0
- package/src/modules/data_source_base.ts +414 -0
- package/src/modules/data_store_updater.ts +419 -0
- package/src/{archiver → modules}/instrumentation.ts +24 -59
- package/src/modules/l1_synchronizer.ts +931 -0
- package/src/{archiver → modules}/validation.ts +11 -6
- package/src/store/block_store.ts +919 -0
- package/src/{archiver/kv_archiver_store → store}/contract_class_store.ts +2 -2
- package/src/{archiver/kv_archiver_store → store}/contract_instance_store.ts +2 -2
- package/src/store/kv_archiver_store.ts +622 -0
- package/src/store/log_store.ts +552 -0
- package/src/{archiver/kv_archiver_store → store}/message_store.ts +21 -18
- package/src/{archiver/structs → structs}/inbox_message.ts +7 -8
- package/src/{archiver/structs → structs}/published.ts +0 -1
- package/src/test/fake_l1_state.ts +599 -0
- package/src/test/fixtures/debug_traceTransaction-multicall3.json +88 -0
- package/src/test/fixtures/debug_traceTransaction-multiplePropose.json +153 -0
- package/src/test/fixtures/debug_traceTransaction-proxied.json +122 -0
- package/src/test/fixtures/trace_transaction-multicall3.json +65 -0
- package/src/test/fixtures/trace_transaction-multiplePropose.json +319 -0
- package/src/test/fixtures/trace_transaction-proxied.json +128 -0
- package/src/test/fixtures/trace_transaction-randomRevert.json +216 -0
- package/src/test/index.ts +1 -0
- package/src/test/mock_archiver.ts +8 -13
- package/src/test/mock_l1_to_l2_message_source.ts +16 -15
- package/src/test/mock_l2_block_source.ts +184 -64
- package/src/test/mock_structs.ts +256 -11
- package/dest/archiver/archiver.d.ts +0 -290
- package/dest/archiver/archiver.d.ts.map +0 -1
- package/dest/archiver/archiver.js +0 -1434
- package/dest/archiver/archiver_store.d.ts +0 -256
- package/dest/archiver/archiver_store.d.ts.map +0 -1
- package/dest/archiver/archiver_store.js +0 -4
- package/dest/archiver/archiver_store_test_suite.d.ts +0 -8
- package/dest/archiver/archiver_store_test_suite.d.ts.map +0 -1
- package/dest/archiver/archiver_store_test_suite.js +0 -1289
- package/dest/archiver/config.d.ts +0 -21
- package/dest/archiver/config.d.ts.map +0 -1
- package/dest/archiver/data_retrieval.d.ts +0 -80
- package/dest/archiver/data_retrieval.d.ts.map +0 -1
- package/dest/archiver/errors.d.ts +0 -12
- package/dest/archiver/errors.d.ts.map +0 -1
- package/dest/archiver/errors.js +0 -17
- package/dest/archiver/index.d.ts +0 -7
- package/dest/archiver/index.d.ts.map +0 -1
- package/dest/archiver/index.js +0 -4
- package/dest/archiver/instrumentation.d.ts +0 -35
- package/dest/archiver/instrumentation.d.ts.map +0 -1
- package/dest/archiver/kv_archiver_store/block_store.d.ts +0 -125
- package/dest/archiver/kv_archiver_store/block_store.d.ts.map +0 -1
- package/dest/archiver/kv_archiver_store/block_store.js +0 -371
- package/dest/archiver/kv_archiver_store/contract_class_store.d.ts +0 -18
- package/dest/archiver/kv_archiver_store/contract_class_store.d.ts.map +0 -1
- package/dest/archiver/kv_archiver_store/contract_instance_store.d.ts +0 -24
- package/dest/archiver/kv_archiver_store/contract_instance_store.d.ts.map +0 -1
- package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts +0 -169
- package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts.map +0 -1
- package/dest/archiver/kv_archiver_store/kv_archiver_store.js +0 -296
- package/dest/archiver/kv_archiver_store/log_store.d.ts +0 -49
- package/dest/archiver/kv_archiver_store/log_store.d.ts.map +0 -1
- package/dest/archiver/kv_archiver_store/log_store.js +0 -337
- package/dest/archiver/kv_archiver_store/message_store.d.ts +0 -39
- package/dest/archiver/kv_archiver_store/message_store.d.ts.map +0 -1
- package/dest/archiver/structs/data_retrieval.d.ts.map +0 -1
- package/dest/archiver/structs/inbox_message.d.ts +0 -15
- package/dest/archiver/structs/inbox_message.d.ts.map +0 -1
- package/dest/archiver/structs/published.d.ts +0 -3
- package/dest/archiver/structs/published.d.ts.map +0 -1
- package/dest/archiver/validation.d.ts +0 -17
- package/dest/archiver/validation.d.ts.map +0 -1
- package/dest/rpc/index.d.ts +0 -9
- package/dest/rpc/index.d.ts.map +0 -1
- package/dest/rpc/index.js +0 -15
- package/src/archiver/archiver.ts +0 -1880
- package/src/archiver/archiver_store.ts +0 -310
- package/src/archiver/archiver_store_test_suite.ts +0 -1295
- package/src/archiver/errors.ts +0 -26
- package/src/archiver/index.ts +0 -6
- package/src/archiver/kv_archiver_store/block_store.ts +0 -482
- package/src/archiver/kv_archiver_store/kv_archiver_store.ts +0 -423
- package/src/archiver/kv_archiver_store/log_store.ts +0 -407
- package/src/rpc/index.ts +0 -16
- /package/dest/{archiver/structs → structs}/data_retrieval.js +0 -0
- /package/dest/{archiver/structs → structs}/published.js +0 -0
- /package/src/{archiver/structs → structs}/data_retrieval.ts +0 -0
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
import { BlockNumber, type CheckpointNumber } from '@aztec/foundation/branded-types';
|
|
2
|
+
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
4
|
+
import {
|
|
5
|
+
ContractClassPublishedEvent,
|
|
6
|
+
PrivateFunctionBroadcastedEvent,
|
|
7
|
+
UtilityFunctionBroadcastedEvent,
|
|
8
|
+
} from '@aztec/protocol-contracts/class-registry';
|
|
9
|
+
import {
|
|
10
|
+
ContractInstancePublishedEvent,
|
|
11
|
+
ContractInstanceUpdatedEvent,
|
|
12
|
+
} from '@aztec/protocol-contracts/instance-registry';
|
|
13
|
+
import type { L2BlockNew, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
14
|
+
import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
15
|
+
import {
|
|
16
|
+
type ExecutablePrivateFunctionWithMembershipProof,
|
|
17
|
+
type UtilityFunctionWithMembershipProof,
|
|
18
|
+
computePublicBytecodeCommitment,
|
|
19
|
+
isValidPrivateFunctionMembershipProof,
|
|
20
|
+
isValidUtilityFunctionMembershipProof,
|
|
21
|
+
} from '@aztec/stdlib/contract';
|
|
22
|
+
import type { ContractClassLog, PrivateLog, PublicLog } from '@aztec/stdlib/logs';
|
|
23
|
+
import type { UInt64 } from '@aztec/stdlib/types';
|
|
24
|
+
|
|
25
|
+
import groupBy from 'lodash.groupby';
|
|
26
|
+
|
|
27
|
+
import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
|
|
28
|
+
|
|
29
|
+
/** Operation type for contract data updates. */
|
|
30
|
+
enum Operation {
|
|
31
|
+
Store,
|
|
32
|
+
Delete,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Result of adding checkpoints with information about any pruned blocks. */
|
|
36
|
+
type ReconcileCheckpointsResult = {
|
|
37
|
+
/** Blocks that were pruned due to conflict with L1 checkpoints. */
|
|
38
|
+
prunedBlocks: L2BlockNew[] | undefined;
|
|
39
|
+
/** Last block number that was already inserted locally, or undefined if none. */
|
|
40
|
+
lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/** Archiver helper module to handle updates to the data store. */
|
|
44
|
+
export class ArchiverDataStoreUpdater {
|
|
45
|
+
private readonly log = createLogger('archiver:store_updater');
|
|
46
|
+
|
|
47
|
+
constructor(private store: KVArchiverDataStore) {}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Adds blocks to the store with contract class/instance extraction from logs.
|
|
51
|
+
* Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
|
|
52
|
+
* and individually broadcasted functions from the block logs.
|
|
53
|
+
*
|
|
54
|
+
* @param blocks - The L2 blocks to add.
|
|
55
|
+
* @param pendingChainValidationStatus - Optional validation status to set.
|
|
56
|
+
* @returns True if the operation is successful.
|
|
57
|
+
*/
|
|
58
|
+
public addBlocks(blocks: L2BlockNew[], pendingChainValidationStatus?: ValidateCheckpointResult): Promise<boolean> {
|
|
59
|
+
return this.store.transactionAsync(async () => {
|
|
60
|
+
await this.store.addBlocks(blocks);
|
|
61
|
+
|
|
62
|
+
const opResults = await Promise.all([
|
|
63
|
+
// Update the pending chain validation status if provided
|
|
64
|
+
pendingChainValidationStatus && this.store.setPendingChainValidationStatus(pendingChainValidationStatus),
|
|
65
|
+
// Add any logs emitted during the retrieved blocks
|
|
66
|
+
this.store.addLogs(blocks),
|
|
67
|
+
// Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
|
|
68
|
+
...blocks.map(block => this.addBlockDataToDB(block)),
|
|
69
|
+
]);
|
|
70
|
+
|
|
71
|
+
return opResults.every(Boolean);
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Reconciles local blocks with incoming checkpoints from L1.
|
|
77
|
+
* Adds checkpoints to the store with contract class/instance extraction from logs.
|
|
78
|
+
* Prunes any local blocks that conflict with checkpoint data (by comparing archive roots).
|
|
79
|
+
* Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
|
|
80
|
+
* and individually broadcasted functions from the checkpoint block logs.
|
|
81
|
+
*
|
|
82
|
+
* @param checkpoints - The published checkpoints to add.
|
|
83
|
+
* @param pendingChainValidationStatus - Optional validation status to set.
|
|
84
|
+
* @returns Result with information about any pruned blocks.
|
|
85
|
+
*/
|
|
86
|
+
public setNewCheckpointData(
|
|
87
|
+
checkpoints: PublishedCheckpoint[],
|
|
88
|
+
pendingChainValidationStatus?: ValidateCheckpointResult,
|
|
89
|
+
): Promise<ReconcileCheckpointsResult> {
|
|
90
|
+
return this.store.transactionAsync(async () => {
|
|
91
|
+
// Before adding checkpoints, check for conflicts with local blocks if any
|
|
92
|
+
const { prunedBlocks, lastAlreadyInsertedBlockNumber } = await this.pruneMismatchingLocalBlocks(checkpoints);
|
|
93
|
+
|
|
94
|
+
await this.store.addCheckpoints(checkpoints);
|
|
95
|
+
|
|
96
|
+
// Filter out blocks that were already inserted via addBlocks() to avoid duplicating logs/contract data
|
|
97
|
+
const newBlocks = checkpoints
|
|
98
|
+
.flatMap((ch: PublishedCheckpoint) => ch.checkpoint.blocks)
|
|
99
|
+
.filter(b => lastAlreadyInsertedBlockNumber === undefined || b.number > lastAlreadyInsertedBlockNumber);
|
|
100
|
+
|
|
101
|
+
await Promise.all([
|
|
102
|
+
// Update the pending chain validation status if provided
|
|
103
|
+
pendingChainValidationStatus && this.store.setPendingChainValidationStatus(pendingChainValidationStatus),
|
|
104
|
+
// Add any logs emitted during the retrieved blocks
|
|
105
|
+
this.store.addLogs(newBlocks),
|
|
106
|
+
// Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
|
|
107
|
+
...newBlocks.map(block => this.addBlockDataToDB(block)),
|
|
108
|
+
]);
|
|
109
|
+
|
|
110
|
+
return { prunedBlocks, lastAlreadyInsertedBlockNumber };
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Checks for local proposed blocks that do not match the ones to be checkpointed and prunes them.
|
|
116
|
+
* This method handles multiple checkpoints but returns after pruning the first conflict found.
|
|
117
|
+
* This is correct because pruning from the first conflict point removes all subsequent blocks,
|
|
118
|
+
* and when checkpoints are added afterward, they include all the correct blocks.
|
|
119
|
+
*/
|
|
120
|
+
private async pruneMismatchingLocalBlocks(checkpoints: PublishedCheckpoint[]): Promise<ReconcileCheckpointsResult> {
|
|
121
|
+
const [lastCheckpointedBlockNumber, lastBlockNumber] = await Promise.all([
|
|
122
|
+
this.store.getCheckpointedL2BlockNumber(),
|
|
123
|
+
this.store.getLatestBlockNumber(),
|
|
124
|
+
]);
|
|
125
|
+
|
|
126
|
+
// Exit early if there are no local uncheckpointed blocks
|
|
127
|
+
if (lastBlockNumber === lastCheckpointedBlockNumber) {
|
|
128
|
+
return { prunedBlocks: undefined, lastAlreadyInsertedBlockNumber: undefined };
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Get all uncheckpointed local blocks
|
|
132
|
+
const uncheckpointedLocalBlocks = await this.store.getBlocks(
|
|
133
|
+
BlockNumber.add(lastCheckpointedBlockNumber, 1),
|
|
134
|
+
lastBlockNumber - lastCheckpointedBlockNumber,
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
let lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
|
|
138
|
+
|
|
139
|
+
for (const publishedCheckpoint of checkpoints) {
|
|
140
|
+
const checkpointBlocks = publishedCheckpoint.checkpoint.blocks;
|
|
141
|
+
const slot = publishedCheckpoint.checkpoint.slot;
|
|
142
|
+
const localBlocksInSlot = uncheckpointedLocalBlocks.filter(b => b.slot === slot);
|
|
143
|
+
|
|
144
|
+
if (checkpointBlocks.length === 0) {
|
|
145
|
+
this.log.warn(`Checkpoint ${publishedCheckpoint.checkpoint.number} for slot ${slot} has no blocks`);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Find the first checkpoint block that conflicts with an existing local block and prune local afterwards
|
|
150
|
+
for (const checkpointBlock of checkpointBlocks) {
|
|
151
|
+
const blockNumber = checkpointBlock.number;
|
|
152
|
+
const existingBlock = localBlocksInSlot.find(b => b.number === blockNumber);
|
|
153
|
+
const blockInfos = {
|
|
154
|
+
existingBlock: existingBlock?.toBlockInfo(),
|
|
155
|
+
checkpointBlock: checkpointBlock.toBlockInfo(),
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
if (!existingBlock) {
|
|
159
|
+
this.log.verbose(`No local block found for checkpointed block number ${blockNumber}`, blockInfos);
|
|
160
|
+
} else if (existingBlock.archive.root.equals(checkpointBlock.archive.root)) {
|
|
161
|
+
this.log.verbose(`Block number ${blockNumber} already inserted and matches checkpoint`, blockInfos);
|
|
162
|
+
lastAlreadyInsertedBlockNumber = blockNumber;
|
|
163
|
+
} else {
|
|
164
|
+
this.log.warn(`Conflict detected at block ${blockNumber} between checkpointed and local block`, blockInfos);
|
|
165
|
+
const prunedBlocks = await this.removeBlocksAfter(BlockNumber(blockNumber - 1));
|
|
166
|
+
return { prunedBlocks, lastAlreadyInsertedBlockNumber };
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// If local has more blocks than the checkpoint (e.g., local has [2,3,4] but checkpoint has [2,3]),
|
|
171
|
+
// we need to prune the extra local blocks so they match what was checkpointed
|
|
172
|
+
const lastCheckpointBlockNumber = checkpointBlocks.at(-1)!.number;
|
|
173
|
+
const lastLocalBlockNumber = localBlocksInSlot.at(-1)?.number;
|
|
174
|
+
|
|
175
|
+
if (lastLocalBlockNumber !== undefined && lastLocalBlockNumber > lastCheckpointBlockNumber) {
|
|
176
|
+
this.log.warn(
|
|
177
|
+
`Local chain for slot ${slot} ends at block ${lastLocalBlockNumber} but checkpoint ends at ${lastCheckpointBlockNumber}. Pruning blocks after block ${lastCheckpointBlockNumber}.`,
|
|
178
|
+
);
|
|
179
|
+
const prunedBlocks = await this.removeBlocksAfter(lastCheckpointBlockNumber);
|
|
180
|
+
return { prunedBlocks, lastAlreadyInsertedBlockNumber };
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
return { prunedBlocks: undefined, lastAlreadyInsertedBlockNumber };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Removes all blocks strictly after the specified block number and cleans up associated contract data.
|
|
189
|
+
* This handles removal of provisionally added blocks along with their contract classes/instances.
|
|
190
|
+
*
|
|
191
|
+
* @param blockNumber - Remove all blocks with number greater than this.
|
|
192
|
+
* @returns The removed blocks.
|
|
193
|
+
*/
|
|
194
|
+
public removeBlocksAfter(blockNumber: BlockNumber): Promise<L2BlockNew[]> {
|
|
195
|
+
return this.store.transactionAsync(async () => {
|
|
196
|
+
// First get the blocks to be removed so we can clean up contract data
|
|
197
|
+
const removedBlocks = await this.store.removeBlocksAfter(blockNumber);
|
|
198
|
+
|
|
199
|
+
// Clean up contract data and logs for the removed blocks
|
|
200
|
+
await Promise.all([
|
|
201
|
+
this.store.deleteLogs(removedBlocks),
|
|
202
|
+
...removedBlocks.map(block => this.removeBlockDataFromDB(block)),
|
|
203
|
+
]);
|
|
204
|
+
|
|
205
|
+
return removedBlocks;
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Unwinds checkpoints from the store with reverse contract extraction.
|
|
211
|
+
* Deletes ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated data
|
|
212
|
+
* that was stored for the unwound checkpoints.
|
|
213
|
+
*
|
|
214
|
+
* @param from - The checkpoint number to unwind from (must be the current tip).
|
|
215
|
+
* @param checkpointsToUnwind - The number of checkpoints to unwind.
|
|
216
|
+
* @returns True if the operation is successful.
|
|
217
|
+
*/
|
|
218
|
+
public async unwindCheckpoints(from: CheckpointNumber, checkpointsToUnwind: number): Promise<boolean> {
|
|
219
|
+
if (checkpointsToUnwind <= 0) {
|
|
220
|
+
throw new Error(`Cannot unwind ${checkpointsToUnwind} blocks`);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const last = await this.store.getSynchedCheckpointNumber();
|
|
224
|
+
if (from != last) {
|
|
225
|
+
throw new Error(`Cannot unwind checkpoints from checkpoint ${from} when the last checkpoint is ${last}`);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const blocks = [];
|
|
229
|
+
const lastCheckpointNumber = from + checkpointsToUnwind - 1;
|
|
230
|
+
for (let checkpointNumber = from; checkpointNumber <= lastCheckpointNumber; checkpointNumber++) {
|
|
231
|
+
const blocksForCheckpoint = await this.store.getBlocksForCheckpoint(checkpointNumber);
|
|
232
|
+
if (!blocksForCheckpoint) {
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
blocks.push(...blocksForCheckpoint);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const opResults = await Promise.all([
|
|
239
|
+
// Prune rolls back to the last proven block, which is by definition valid
|
|
240
|
+
this.store.setPendingChainValidationStatus({ valid: true }),
|
|
241
|
+
// Remove contract data for all blocks being unwound
|
|
242
|
+
...blocks.map(block => this.removeBlockDataFromDB(block)),
|
|
243
|
+
this.store.deleteLogs(blocks),
|
|
244
|
+
this.store.unwindCheckpoints(from, checkpointsToUnwind),
|
|
245
|
+
]);
|
|
246
|
+
|
|
247
|
+
return opResults.every(Boolean);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** Extracts and stores contract data from a single block. */
|
|
251
|
+
private addBlockDataToDB(block: L2BlockNew): Promise<boolean> {
|
|
252
|
+
return this.editContractBlockData(block, Operation.Store);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** Removes contract data associated with a block. */
|
|
256
|
+
private removeBlockDataFromDB(block: L2BlockNew): Promise<boolean> {
|
|
257
|
+
return this.editContractBlockData(block, Operation.Delete);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** Adds or remove contract data associated with a block. */
|
|
261
|
+
private async editContractBlockData(block: L2BlockNew, operation: Operation): Promise<boolean> {
|
|
262
|
+
const contractClassLogs = block.body.txEffects.flatMap(txEffect => txEffect.contractClassLogs);
|
|
263
|
+
const privateLogs = block.body.txEffects.flatMap(txEffect => txEffect.privateLogs);
|
|
264
|
+
const publicLogs = block.body.txEffects.flatMap(txEffect => txEffect.publicLogs);
|
|
265
|
+
|
|
266
|
+
return (
|
|
267
|
+
await Promise.all([
|
|
268
|
+
this.updatePublishedContractClasses(contractClassLogs, block.number, operation),
|
|
269
|
+
this.updateDeployedContractInstances(privateLogs, block.number, operation),
|
|
270
|
+
this.updateUpdatedContractInstances(publicLogs, block.header.globalVariables.timestamp, operation),
|
|
271
|
+
operation === Operation.Store
|
|
272
|
+
? this.storeBroadcastedIndividualFunctions(contractClassLogs, block.number)
|
|
273
|
+
: Promise.resolve(true),
|
|
274
|
+
])
|
|
275
|
+
).every(Boolean);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Extracts and stores contract classes out of ContractClassPublished events emitted by the class registry contract.
|
|
280
|
+
*/
|
|
281
|
+
private async updatePublishedContractClasses(
|
|
282
|
+
allLogs: ContractClassLog[],
|
|
283
|
+
blockNum: BlockNumber,
|
|
284
|
+
operation: Operation,
|
|
285
|
+
): Promise<boolean> {
|
|
286
|
+
const contractClassPublishedEvents = allLogs
|
|
287
|
+
.filter(log => ContractClassPublishedEvent.isContractClassPublishedEvent(log))
|
|
288
|
+
.map(log => ContractClassPublishedEvent.fromLog(log));
|
|
289
|
+
|
|
290
|
+
const contractClasses = await Promise.all(contractClassPublishedEvents.map(e => e.toContractClassPublic()));
|
|
291
|
+
if (contractClasses.length > 0) {
|
|
292
|
+
contractClasses.forEach(c => this.log.verbose(`${Operation[operation]} contract class ${c.id.toString()}`));
|
|
293
|
+
if (operation == Operation.Store) {
|
|
294
|
+
// TODO: Will probably want to create some worker threads to compute these bytecode commitments as they are expensive
|
|
295
|
+
const commitments = await Promise.all(
|
|
296
|
+
contractClasses.map(c => computePublicBytecodeCommitment(c.packedBytecode)),
|
|
297
|
+
);
|
|
298
|
+
return await this.store.addContractClasses(contractClasses, commitments, blockNum);
|
|
299
|
+
} else if (operation == Operation.Delete) {
|
|
300
|
+
return await this.store.deleteContractClasses(contractClasses, blockNum);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return true;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Extracts and stores contract instances out of ContractInstancePublished events emitted by the canonical deployer contract.
|
|
308
|
+
*/
|
|
309
|
+
private async updateDeployedContractInstances(
|
|
310
|
+
allLogs: PrivateLog[],
|
|
311
|
+
blockNum: BlockNumber,
|
|
312
|
+
operation: Operation,
|
|
313
|
+
): Promise<boolean> {
|
|
314
|
+
const contractInstances = allLogs
|
|
315
|
+
.filter(log => ContractInstancePublishedEvent.isContractInstancePublishedEvent(log))
|
|
316
|
+
.map(log => ContractInstancePublishedEvent.fromLog(log))
|
|
317
|
+
.map(e => e.toContractInstance());
|
|
318
|
+
if (contractInstances.length > 0) {
|
|
319
|
+
contractInstances.forEach(c =>
|
|
320
|
+
this.log.verbose(`${Operation[operation]} contract instance at ${c.address.toString()}`),
|
|
321
|
+
);
|
|
322
|
+
if (operation == Operation.Store) {
|
|
323
|
+
return await this.store.addContractInstances(contractInstances, blockNum);
|
|
324
|
+
} else if (operation == Operation.Delete) {
|
|
325
|
+
return await this.store.deleteContractInstances(contractInstances, blockNum);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
return true;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Extracts and stores contract instance updates out of ContractInstanceUpdated events.
|
|
333
|
+
*/
|
|
334
|
+
private async updateUpdatedContractInstances(
|
|
335
|
+
allLogs: PublicLog[],
|
|
336
|
+
timestamp: UInt64,
|
|
337
|
+
operation: Operation,
|
|
338
|
+
): Promise<boolean> {
|
|
339
|
+
const contractUpdates = allLogs
|
|
340
|
+
.filter(log => ContractInstanceUpdatedEvent.isContractInstanceUpdatedEvent(log))
|
|
341
|
+
.map(log => ContractInstanceUpdatedEvent.fromLog(log))
|
|
342
|
+
.map(e => e.toContractInstanceUpdate());
|
|
343
|
+
|
|
344
|
+
if (contractUpdates.length > 0) {
|
|
345
|
+
contractUpdates.forEach(c =>
|
|
346
|
+
this.log.verbose(`${Operation[operation]} contract instance update at ${c.address.toString()}`),
|
|
347
|
+
);
|
|
348
|
+
if (operation == Operation.Store) {
|
|
349
|
+
return await this.store.addContractInstanceUpdates(contractUpdates, timestamp);
|
|
350
|
+
} else if (operation == Operation.Delete) {
|
|
351
|
+
return await this.store.deleteContractInstanceUpdates(contractUpdates, timestamp);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
return true;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Stores the functions that were broadcasted individually.
|
|
359
|
+
*
|
|
360
|
+
* @dev Beware that there is not a delete variant of this, since they are added to contract classes
|
|
361
|
+
* and will be deleted as part of the class if needed.
|
|
362
|
+
*/
|
|
363
|
+
private async storeBroadcastedIndividualFunctions(
|
|
364
|
+
allLogs: ContractClassLog[],
|
|
365
|
+
_blockNum: BlockNumber,
|
|
366
|
+
): Promise<boolean> {
|
|
367
|
+
// Filter out private and utility function broadcast events
|
|
368
|
+
const privateFnEvents = allLogs
|
|
369
|
+
.filter(log => PrivateFunctionBroadcastedEvent.isPrivateFunctionBroadcastedEvent(log))
|
|
370
|
+
.map(log => PrivateFunctionBroadcastedEvent.fromLog(log));
|
|
371
|
+
const utilityFnEvents = allLogs
|
|
372
|
+
.filter(log => UtilityFunctionBroadcastedEvent.isUtilityFunctionBroadcastedEvent(log))
|
|
373
|
+
.map(log => UtilityFunctionBroadcastedEvent.fromLog(log));
|
|
374
|
+
|
|
375
|
+
// Group all events by contract class id
|
|
376
|
+
for (const [classIdString, classEvents] of Object.entries(
|
|
377
|
+
groupBy([...privateFnEvents, ...utilityFnEvents], e => e.contractClassId.toString()),
|
|
378
|
+
)) {
|
|
379
|
+
const contractClassId = Fr.fromHexString(classIdString);
|
|
380
|
+
const contractClass = await this.store.getContractClass(contractClassId);
|
|
381
|
+
if (!contractClass) {
|
|
382
|
+
this.log.warn(`Skipping broadcasted functions as contract class ${contractClassId.toString()} was not found`);
|
|
383
|
+
continue;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// Split private and utility functions, and filter out invalid ones
|
|
387
|
+
const allFns = classEvents.map(e => e.toFunctionWithMembershipProof());
|
|
388
|
+
const privateFns = allFns.filter(
|
|
389
|
+
(fn): fn is ExecutablePrivateFunctionWithMembershipProof => 'utilityFunctionsTreeRoot' in fn,
|
|
390
|
+
);
|
|
391
|
+
const utilityFns = allFns.filter(
|
|
392
|
+
(fn): fn is UtilityFunctionWithMembershipProof => 'privateFunctionsArtifactTreeRoot' in fn,
|
|
393
|
+
);
|
|
394
|
+
|
|
395
|
+
const privateFunctionsWithValidity = await Promise.all(
|
|
396
|
+
privateFns.map(async fn => ({ fn, valid: await isValidPrivateFunctionMembershipProof(fn, contractClass) })),
|
|
397
|
+
);
|
|
398
|
+
const validPrivateFns = privateFunctionsWithValidity.filter(({ valid }) => valid).map(({ fn }) => fn);
|
|
399
|
+
const utilityFunctionsWithValidity = await Promise.all(
|
|
400
|
+
utilityFns.map(async fn => ({
|
|
401
|
+
fn,
|
|
402
|
+
valid: await isValidUtilityFunctionMembershipProof(fn, contractClass),
|
|
403
|
+
})),
|
|
404
|
+
);
|
|
405
|
+
const validUtilityFns = utilityFunctionsWithValidity.filter(({ valid }) => valid).map(({ fn }) => fn);
|
|
406
|
+
const validFnCount = validPrivateFns.length + validUtilityFns.length;
|
|
407
|
+
if (validFnCount !== allFns.length) {
|
|
408
|
+
this.log.warn(`Skipping ${allFns.length - validFnCount} invalid functions`);
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// Store the functions in the contract class in a single operation
|
|
412
|
+
if (validFnCount > 0) {
|
|
413
|
+
this.log.verbose(`Storing ${validFnCount} functions for contract class ${contractClassId.toString()}`);
|
|
414
|
+
}
|
|
415
|
+
return await this.store.addFunctions(contractClassId, validPrivateFns, validUtilityFns);
|
|
416
|
+
}
|
|
417
|
+
return true;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
@@ -10,7 +10,6 @@ import {
|
|
|
10
10
|
type TelemetryClient,
|
|
11
11
|
type Tracer,
|
|
12
12
|
type UpDownCounter,
|
|
13
|
-
ValueType,
|
|
14
13
|
} from '@aztec/telemetry-client';
|
|
15
14
|
|
|
16
15
|
export class ArchiverInstrumentation {
|
|
@@ -34,6 +33,8 @@ export class ArchiverInstrumentation {
|
|
|
34
33
|
private syncDurationPerMessage: Histogram;
|
|
35
34
|
private syncMessageCount: UpDownCounter;
|
|
36
35
|
|
|
36
|
+
private blockProposalTxTargetCount: UpDownCounter;
|
|
37
|
+
|
|
37
38
|
private log = createLogger('archiver:instrumentation');
|
|
38
39
|
|
|
39
40
|
private constructor(
|
|
@@ -43,76 +44,33 @@ export class ArchiverInstrumentation {
|
|
|
43
44
|
this.tracer = telemetry.getTracer('Archiver');
|
|
44
45
|
const meter = telemetry.getMeter('Archiver');
|
|
45
46
|
|
|
46
|
-
this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT
|
|
47
|
-
description: 'The height of the latest block processed by the archiver',
|
|
48
|
-
valueType: ValueType.INT,
|
|
49
|
-
});
|
|
47
|
+
this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT);
|
|
50
48
|
|
|
51
|
-
this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT
|
|
52
|
-
description: 'The height of the latest L1 block processed by the archiver',
|
|
53
|
-
valueType: ValueType.INT,
|
|
54
|
-
});
|
|
49
|
+
this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT);
|
|
55
50
|
|
|
56
|
-
this.txCount = meter.createUpDownCounter(Metrics.ARCHIVER_TOTAL_TXS
|
|
57
|
-
description: 'The total number of transactions',
|
|
58
|
-
valueType: ValueType.INT,
|
|
59
|
-
});
|
|
51
|
+
this.txCount = meter.createUpDownCounter(Metrics.ARCHIVER_TOTAL_TXS);
|
|
60
52
|
|
|
61
|
-
this.proofsSubmittedCount = meter.createUpDownCounter(Metrics.ARCHIVER_ROLLUP_PROOF_COUNT
|
|
62
|
-
description: 'Number of proofs submitted',
|
|
63
|
-
valueType: ValueType.INT,
|
|
64
|
-
});
|
|
53
|
+
this.proofsSubmittedCount = meter.createUpDownCounter(Metrics.ARCHIVER_ROLLUP_PROOF_COUNT);
|
|
65
54
|
|
|
66
|
-
this.proofsSubmittedDelay = meter.createHistogram(Metrics.ARCHIVER_ROLLUP_PROOF_DELAY
|
|
67
|
-
unit: 'ms',
|
|
68
|
-
description: 'Time after a block is submitted until its proof is published',
|
|
69
|
-
valueType: ValueType.INT,
|
|
70
|
-
});
|
|
55
|
+
this.proofsSubmittedDelay = meter.createHistogram(Metrics.ARCHIVER_ROLLUP_PROOF_DELAY);
|
|
71
56
|
|
|
72
|
-
this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK
|
|
73
|
-
unit: 'ms',
|
|
74
|
-
description: 'Duration to sync a block',
|
|
75
|
-
valueType: ValueType.INT,
|
|
76
|
-
});
|
|
57
|
+
this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK);
|
|
77
58
|
|
|
78
|
-
this.syncBlockCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_BLOCK_COUNT
|
|
79
|
-
description: 'Number of blocks synced from L1',
|
|
80
|
-
valueType: ValueType.INT,
|
|
81
|
-
});
|
|
59
|
+
this.syncBlockCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
|
|
82
60
|
|
|
83
|
-
this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK
|
|
84
|
-
description: 'The mana consumed by blocks',
|
|
85
|
-
valueType: ValueType.DOUBLE,
|
|
86
|
-
unit: 'Mmana',
|
|
87
|
-
});
|
|
61
|
+
this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK);
|
|
88
62
|
|
|
89
|
-
this.txsPerBlock = meter.createHistogram(Metrics.ARCHIVER_TXS_PER_BLOCK
|
|
90
|
-
description: 'The block tx count',
|
|
91
|
-
valueType: ValueType.INT,
|
|
92
|
-
unit: 'tx',
|
|
93
|
-
});
|
|
63
|
+
this.txsPerBlock = meter.createHistogram(Metrics.ARCHIVER_TXS_PER_BLOCK);
|
|
94
64
|
|
|
95
|
-
this.syncDurationPerMessage = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_MESSAGE
|
|
96
|
-
unit: 'ms',
|
|
97
|
-
description: 'Duration to sync a message',
|
|
98
|
-
valueType: ValueType.INT,
|
|
99
|
-
});
|
|
65
|
+
this.syncDurationPerMessage = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_MESSAGE);
|
|
100
66
|
|
|
101
|
-
this.syncMessageCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_MESSAGE_COUNT
|
|
102
|
-
description: 'Number of L1 to L2 messages synced',
|
|
103
|
-
valueType: ValueType.INT,
|
|
104
|
-
});
|
|
67
|
+
this.syncMessageCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_MESSAGE_COUNT);
|
|
105
68
|
|
|
106
|
-
this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION
|
|
107
|
-
unit: 'ms',
|
|
108
|
-
description: 'Duration to sync a message',
|
|
109
|
-
valueType: ValueType.INT,
|
|
110
|
-
});
|
|
69
|
+
this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION);
|
|
111
70
|
|
|
112
|
-
this.pruneCount = meter.createUpDownCounter(Metrics.ARCHIVER_PRUNE_COUNT
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
});
|
|
71
|
+
this.pruneCount = meter.createUpDownCounter(Metrics.ARCHIVER_PRUNE_COUNT);
|
|
72
|
+
|
|
73
|
+
this.blockProposalTxTargetCount = meter.createUpDownCounter(Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT);
|
|
116
74
|
|
|
117
75
|
this.dbMetrics = new LmdbMetrics(
|
|
118
76
|
meter,
|
|
@@ -184,4 +142,11 @@ export class ArchiverInstrumentation {
|
|
|
184
142
|
public updateL1BlockHeight(blockNumber: bigint) {
|
|
185
143
|
this.l1BlockHeight.record(Number(blockNumber));
|
|
186
144
|
}
|
|
145
|
+
|
|
146
|
+
public recordBlockProposalTxTarget(target: string, usedTrace: boolean) {
|
|
147
|
+
this.blockProposalTxTargetCount.add(1, {
|
|
148
|
+
[Attributes.L1_BLOCK_PROPOSAL_TX_TARGET]: target.toLowerCase(),
|
|
149
|
+
[Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: usedTrace,
|
|
150
|
+
});
|
|
151
|
+
}
|
|
187
152
|
}
|