@aztec/archiver 0.65.2 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/archiver/archiver.d.ts +20 -24
- package/dest/archiver/archiver.d.ts.map +1 -1
- package/dest/archiver/archiver.js +148 -102
- package/dest/archiver/archiver_store.d.ts +8 -9
- package/dest/archiver/archiver_store.d.ts.map +1 -1
- package/dest/archiver/archiver_store_test_suite.d.ts.map +1 -1
- package/dest/archiver/archiver_store_test_suite.js +126 -150
- package/dest/archiver/config.d.ts +6 -12
- package/dest/archiver/config.d.ts.map +1 -1
- package/dest/archiver/config.js +6 -1
- package/dest/archiver/data_retrieval.d.ts +4 -5
- package/dest/archiver/data_retrieval.d.ts.map +1 -1
- package/dest/archiver/data_retrieval.js +21 -22
- package/dest/archiver/instrumentation.d.ts +4 -7
- package/dest/archiver/instrumentation.d.ts.map +1 -1
- package/dest/archiver/instrumentation.js +17 -22
- package/dest/archiver/kv_archiver_store/block_store.d.ts +2 -2
- package/dest/archiver/kv_archiver_store/block_store.d.ts.map +1 -1
- package/dest/archiver/kv_archiver_store/block_store.js +6 -6
- package/dest/archiver/kv_archiver_store/contract_class_store.js +2 -2
- package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts +8 -9
- package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts.map +1 -1
- package/dest/archiver/kv_archiver_store/kv_archiver_store.js +9 -10
- package/dest/archiver/kv_archiver_store/log_store.d.ts +7 -8
- package/dest/archiver/kv_archiver_store/log_store.d.ts.map +1 -1
- package/dest/archiver/kv_archiver_store/log_store.js +57 -97
- package/dest/archiver/kv_archiver_store/message_store.js +3 -3
- package/dest/archiver/kv_archiver_store/nullifier_store.js +3 -3
- package/dest/archiver/memory_archiver_store/memory_archiver_store.d.ts +9 -11
- package/dest/archiver/memory_archiver_store/memory_archiver_store.d.ts.map +1 -1
- package/dest/archiver/memory_archiver_store/memory_archiver_store.js +53 -60
- package/dest/factory.js +4 -4
- package/dest/index.d.ts +2 -2
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +3 -42
- package/dest/test/mock_l2_block_source.d.ts +2 -2
- package/dest/test/mock_l2_block_source.d.ts.map +1 -1
- package/dest/test/mock_l2_block_source.js +4 -4
- package/package.json +15 -13
- package/src/archiver/archiver.ts +208 -198
- package/src/archiver/archiver_store.ts +8 -15
- package/src/archiver/archiver_store_test_suite.ts +160 -186
- package/src/archiver/config.ts +12 -12
- package/src/archiver/data_retrieval.ts +21 -26
- package/src/archiver/instrumentation.ts +21 -23
- package/src/archiver/kv_archiver_store/block_store.ts +6 -6
- package/src/archiver/kv_archiver_store/contract_class_store.ts +1 -1
- package/src/archiver/kv_archiver_store/kv_archiver_store.ts +11 -18
- package/src/archiver/kv_archiver_store/log_store.ts +70 -120
- package/src/archiver/kv_archiver_store/message_store.ts +2 -2
- package/src/archiver/kv_archiver_store/nullifier_store.ts +2 -2
- package/src/archiver/memory_archiver_store/memory_archiver_store.ts +56 -70
- package/src/factory.ts +3 -3
- package/src/index.ts +2 -59
- package/src/test/mock_l2_block_source.ts +5 -6
- package/dest/archiver/epoch_helpers.d.ts +0 -20
- package/dest/archiver/epoch_helpers.d.ts.map +0 -1
- package/dest/archiver/epoch_helpers.js +0 -34
- package/src/archiver/epoch_helpers.ts +0 -54
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { Body, InboxLeaf, L2Block } from '@aztec/circuit-types';
|
|
2
|
-
import { AppendOnlyTreeSnapshot,
|
|
2
|
+
import { AppendOnlyTreeSnapshot, BlockHeader, Fr, Proof } from '@aztec/circuits.js';
|
|
3
|
+
import { asyncPool } from '@aztec/foundation/async-pool';
|
|
3
4
|
import { type EthAddress } from '@aztec/foundation/eth-address';
|
|
4
5
|
import { type ViemSignature } from '@aztec/foundation/eth-signature';
|
|
5
|
-
import { type
|
|
6
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
6
7
|
import { numToUInt32BE } from '@aztec/foundation/serialize';
|
|
7
8
|
import { type InboxAbi, RollupAbi } from '@aztec/l1-artifacts';
|
|
8
9
|
|
|
@@ -25,19 +26,17 @@ import { type L1Published, type L1PublishedData } from './structs/published.js';
|
|
|
25
26
|
* Fetches new L2 blocks.
|
|
26
27
|
* @param publicClient - The viem public client to use for transaction retrieval.
|
|
27
28
|
* @param rollupAddress - The address of the rollup contract.
|
|
28
|
-
* @param blockUntilSynced - If true, blocks until the archiver has fully synced.
|
|
29
29
|
* @param searchStartBlock - The block number to use for starting the search.
|
|
30
30
|
* @param searchEndBlock - The highest block number that we should search up to.
|
|
31
31
|
* @param expectedNextL2BlockNum - The next L2 block number that we expect to find.
|
|
32
32
|
* @returns An array of block; as well as the next eth block to search from.
|
|
33
33
|
*/
|
|
34
|
-
export async function
|
|
34
|
+
export async function retrieveBlocksFromRollup(
|
|
35
35
|
rollup: GetContractReturnType<typeof RollupAbi, PublicClient<HttpTransport, Chain>>,
|
|
36
36
|
publicClient: PublicClient,
|
|
37
|
-
blockUntilSynced: boolean,
|
|
38
37
|
searchStartBlock: bigint,
|
|
39
38
|
searchEndBlock: bigint,
|
|
40
|
-
logger:
|
|
39
|
+
logger: Logger = createLogger('archiver'),
|
|
41
40
|
): Promise<L1Published<L2Block>[]> {
|
|
42
41
|
const retrievedBlocks: L1Published<L2Block>[] = [];
|
|
43
42
|
do {
|
|
@@ -58,13 +57,13 @@ export async function retrieveBlockFromRollup(
|
|
|
58
57
|
|
|
59
58
|
const lastLog = l2BlockProposedLogs[l2BlockProposedLogs.length - 1];
|
|
60
59
|
logger.debug(
|
|
61
|
-
`Got L2 block processed logs for ${l2BlockProposedLogs[0].blockNumber}-${lastLog.blockNumber} between ${searchStartBlock}-${searchEndBlock}
|
|
60
|
+
`Got ${l2BlockProposedLogs.length} L2 block processed logs for L2 blocks ${l2BlockProposedLogs[0].args.blockNumber}-${lastLog.args.blockNumber} between L1 blocks ${searchStartBlock}-${searchEndBlock}`,
|
|
62
61
|
);
|
|
63
62
|
|
|
64
63
|
const newBlocks = await processL2BlockProposedLogs(rollup, publicClient, l2BlockProposedLogs, logger);
|
|
65
64
|
retrievedBlocks.push(...newBlocks);
|
|
66
65
|
searchStartBlock = lastLog.blockNumber! + 1n;
|
|
67
|
-
} while (
|
|
66
|
+
} while (searchStartBlock <= searchEndBlock);
|
|
68
67
|
return retrievedBlocks;
|
|
69
68
|
}
|
|
70
69
|
|
|
@@ -79,17 +78,16 @@ export async function processL2BlockProposedLogs(
|
|
|
79
78
|
rollup: GetContractReturnType<typeof RollupAbi, PublicClient<HttpTransport, Chain>>,
|
|
80
79
|
publicClient: PublicClient,
|
|
81
80
|
logs: GetContractEventsReturnType<typeof RollupAbi, 'L2BlockProposed'>,
|
|
82
|
-
logger:
|
|
81
|
+
logger: Logger,
|
|
83
82
|
): Promise<L1Published<L2Block>[]> {
|
|
84
83
|
const retrievedBlocks: L1Published<L2Block>[] = [];
|
|
85
|
-
|
|
84
|
+
await asyncPool(10, logs, async log => {
|
|
86
85
|
const l2BlockNumber = log.args.blockNumber!;
|
|
87
86
|
const archive = log.args.archive!;
|
|
88
87
|
const archiveFromChain = await rollup.read.archiveAt([l2BlockNumber]);
|
|
89
88
|
|
|
90
89
|
// The value from the event and contract will match only if the block is in the chain.
|
|
91
90
|
if (archive === archiveFromChain) {
|
|
92
|
-
// TODO: Fetch blocks from calldata in parallel
|
|
93
91
|
const block = await getBlockFromRollupTx(publicClient, log.transactionHash!, l2BlockNumber);
|
|
94
92
|
|
|
95
93
|
const l1: L1PublishedData = {
|
|
@@ -100,11 +98,12 @@ export async function processL2BlockProposedLogs(
|
|
|
100
98
|
|
|
101
99
|
retrievedBlocks.push({ data: block, l1 });
|
|
102
100
|
} else {
|
|
103
|
-
logger.warn(
|
|
104
|
-
|
|
105
|
-
|
|
101
|
+
logger.warn(`Ignoring L2 block ${l2BlockNumber} due to archive root mismatch`, {
|
|
102
|
+
actual: archive,
|
|
103
|
+
expected: archiveFromChain,
|
|
104
|
+
});
|
|
106
105
|
}
|
|
107
|
-
}
|
|
106
|
+
});
|
|
108
107
|
|
|
109
108
|
return retrievedBlocks;
|
|
110
109
|
}
|
|
@@ -129,10 +128,7 @@ async function getBlockFromRollupTx(
|
|
|
129
128
|
l2BlockNum: bigint,
|
|
130
129
|
): Promise<L2Block> {
|
|
131
130
|
const { input: data } = await publicClient.getTransaction({ hash: txHash });
|
|
132
|
-
const { functionName, args } = decodeFunctionData({
|
|
133
|
-
abi: RollupAbi,
|
|
134
|
-
data,
|
|
135
|
-
});
|
|
131
|
+
const { functionName, args } = decodeFunctionData({ abi: RollupAbi, data });
|
|
136
132
|
|
|
137
133
|
const allowedMethods = ['propose', 'proposeAndClaim'];
|
|
138
134
|
|
|
@@ -154,7 +150,7 @@ async function getBlockFromRollupTx(
|
|
|
154
150
|
Hex,
|
|
155
151
|
];
|
|
156
152
|
|
|
157
|
-
const header =
|
|
153
|
+
const header = BlockHeader.fromBuffer(Buffer.from(hexToBytes(decodedArgs.header)));
|
|
158
154
|
const blockBody = Body.fromBuffer(Buffer.from(hexToBytes(bodyHex)));
|
|
159
155
|
|
|
160
156
|
const blockNumberFromHeader = header.globalVariables.blockNumber.toBigInt();
|
|
@@ -184,7 +180,6 @@ async function getBlockFromRollupTx(
|
|
|
184
180
|
*/
|
|
185
181
|
export async function retrieveL1ToL2Messages(
|
|
186
182
|
inbox: GetContractReturnType<typeof InboxAbi, PublicClient<HttpTransport, Chain>>,
|
|
187
|
-
blockUntilSynced: boolean,
|
|
188
183
|
searchStartBlock: bigint,
|
|
189
184
|
searchEndBlock: bigint,
|
|
190
185
|
): Promise<DataRetrieval<InboxLeaf>> {
|
|
@@ -208,12 +203,12 @@ export async function retrieveL1ToL2Messages(
|
|
|
208
203
|
|
|
209
204
|
for (const log of messageSentLogs) {
|
|
210
205
|
const { index, hash } = log.args;
|
|
211
|
-
retrievedL1ToL2Messages.push(new InboxLeaf(index!, Fr.
|
|
206
|
+
retrievedL1ToL2Messages.push(new InboxLeaf(index!, Fr.fromHexString(hash!)));
|
|
212
207
|
}
|
|
213
208
|
|
|
214
209
|
// handles the case when there are no new messages:
|
|
215
210
|
searchStartBlock = (messageSentLogs.findLast(msgLog => !!msgLog)?.blockNumber || searchStartBlock) + 1n;
|
|
216
|
-
} while (
|
|
211
|
+
} while (searchStartBlock <= searchEndBlock);
|
|
217
212
|
return { lastProcessedL1BlockNumber: searchStartBlock - 1n, retrievedData: retrievedL1ToL2Messages };
|
|
218
213
|
}
|
|
219
214
|
|
|
@@ -235,7 +230,7 @@ export async function retrieveL2ProofVerifiedEvents(
|
|
|
235
230
|
return logs.map(log => ({
|
|
236
231
|
l1BlockNumber: log.blockNumber,
|
|
237
232
|
l2BlockNumber: log.args.blockNumber,
|
|
238
|
-
proverId: Fr.
|
|
233
|
+
proverId: Fr.fromHexString(log.args.proverId),
|
|
239
234
|
txHash: log.transactionHash,
|
|
240
235
|
}));
|
|
241
236
|
}
|
|
@@ -302,8 +297,8 @@ export async function getProofFromSubmitProofTx(
|
|
|
302
297
|
];
|
|
303
298
|
|
|
304
299
|
aggregationObject = Buffer.from(hexToBytes(decodedArgs.aggregationObject));
|
|
305
|
-
proverId = Fr.
|
|
306
|
-
archiveRoot = Fr.
|
|
300
|
+
proverId = Fr.fromHexString(decodedArgs.args[6]);
|
|
301
|
+
archiveRoot = Fr.fromHexString(decodedArgs.args[1]);
|
|
307
302
|
proof = Proof.fromBuffer(Buffer.from(hexToBytes(decodedArgs.proof)));
|
|
308
303
|
} else {
|
|
309
304
|
throw new Error(`Unexpected proof method called ${functionName}`);
|
|
@@ -1,29 +1,29 @@
|
|
|
1
1
|
import { type L2Block } from '@aztec/circuit-types';
|
|
2
|
-
import {
|
|
2
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
3
3
|
import {
|
|
4
4
|
Attributes,
|
|
5
5
|
type Gauge,
|
|
6
6
|
type Histogram,
|
|
7
7
|
LmdbMetrics,
|
|
8
|
+
type LmdbStatsCallback,
|
|
8
9
|
Metrics,
|
|
9
10
|
type TelemetryClient,
|
|
10
11
|
type UpDownCounter,
|
|
11
12
|
ValueType,
|
|
12
|
-
exponentialBuckets,
|
|
13
|
-
millisecondBuckets,
|
|
14
13
|
} from '@aztec/telemetry-client';
|
|
15
14
|
|
|
16
15
|
export class ArchiverInstrumentation {
|
|
17
16
|
private blockHeight: Gauge;
|
|
18
17
|
private blockSize: Gauge;
|
|
19
18
|
private syncDuration: Histogram;
|
|
19
|
+
private l1BlocksSynced: UpDownCounter;
|
|
20
20
|
private proofsSubmittedDelay: Histogram;
|
|
21
21
|
private proofsSubmittedCount: UpDownCounter;
|
|
22
22
|
private dbMetrics: LmdbMetrics;
|
|
23
23
|
|
|
24
|
-
private log =
|
|
24
|
+
private log = createLogger('archiver:instrumentation');
|
|
25
25
|
|
|
26
|
-
constructor(private telemetry: TelemetryClient) {
|
|
26
|
+
private constructor(private telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
|
|
27
27
|
const meter = telemetry.getMeter('Archiver');
|
|
28
28
|
this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT, {
|
|
29
29
|
description: 'The height of the latest block processed by the archiver',
|
|
@@ -39,9 +39,6 @@ export class ArchiverInstrumentation {
|
|
|
39
39
|
unit: 'ms',
|
|
40
40
|
description: 'Duration to sync a block',
|
|
41
41
|
valueType: ValueType.INT,
|
|
42
|
-
advice: {
|
|
43
|
-
explicitBucketBoundaries: exponentialBuckets(1, 16),
|
|
44
|
-
},
|
|
45
42
|
});
|
|
46
43
|
|
|
47
44
|
this.proofsSubmittedCount = meter.createUpDownCounter(Metrics.ARCHIVER_ROLLUP_PROOF_COUNT, {
|
|
@@ -53,30 +50,30 @@ export class ArchiverInstrumentation {
|
|
|
53
50
|
unit: 'ms',
|
|
54
51
|
description: 'Time after a block is submitted until its proof is published',
|
|
55
52
|
valueType: ValueType.INT,
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
this.l1BlocksSynced = meter.createUpDownCounter(Metrics.ARCHIVER_L1_BLOCKS_SYNCED, {
|
|
56
|
+
description: 'Number of blocks synced from L1',
|
|
57
|
+
valueType: ValueType.INT,
|
|
59
58
|
});
|
|
60
59
|
|
|
61
60
|
this.dbMetrics = new LmdbMetrics(
|
|
62
61
|
meter,
|
|
63
62
|
{
|
|
64
|
-
|
|
65
|
-
description: 'Database map size for the archiver',
|
|
66
|
-
},
|
|
67
|
-
{
|
|
68
|
-
name: Metrics.ARCHIVER_DB_USED_SIZE,
|
|
69
|
-
description: 'Database used size for the archiver',
|
|
70
|
-
},
|
|
71
|
-
{
|
|
72
|
-
name: Metrics.ARCHIVER_DB_NUM_ITEMS,
|
|
73
|
-
description: 'Num items in the archiver database',
|
|
63
|
+
[Attributes.DB_DATA_TYPE]: 'archiver',
|
|
74
64
|
},
|
|
65
|
+
lmdbStats,
|
|
75
66
|
);
|
|
76
67
|
}
|
|
77
68
|
|
|
78
|
-
public
|
|
79
|
-
|
|
69
|
+
public static async new(telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
|
|
70
|
+
const instance = new ArchiverInstrumentation(telemetry, lmdbStats);
|
|
71
|
+
|
|
72
|
+
instance.l1BlocksSynced.add(0);
|
|
73
|
+
|
|
74
|
+
await instance.telemetry.flush();
|
|
75
|
+
|
|
76
|
+
return instance;
|
|
80
77
|
}
|
|
81
78
|
|
|
82
79
|
public isEnabled(): boolean {
|
|
@@ -86,6 +83,7 @@ export class ArchiverInstrumentation {
|
|
|
86
83
|
public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) {
|
|
87
84
|
this.syncDuration.record(Math.ceil(syncTimePerBlock));
|
|
88
85
|
this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
|
|
86
|
+
this.l1BlocksSynced.add(blocks.length);
|
|
89
87
|
for (const block of blocks) {
|
|
90
88
|
this.blockSize.record(block.body.txEffects.length);
|
|
91
89
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Body, type InBlock, L2Block, L2BlockHash, type TxEffect, type TxHash, TxReceipt } from '@aztec/circuit-types';
|
|
2
|
-
import { AppendOnlyTreeSnapshot, type AztecAddress,
|
|
3
|
-
import {
|
|
2
|
+
import { AppendOnlyTreeSnapshot, type AztecAddress, BlockHeader, INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js';
|
|
3
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
4
4
|
import { type AztecKVStore, type AztecMap, type AztecSingleton, type Range } from '@aztec/kv-store';
|
|
5
5
|
|
|
6
6
|
import { type L1Published, type L1PublishedData } from '../structs/published.js';
|
|
@@ -38,7 +38,7 @@ export class BlockStore {
|
|
|
38
38
|
/** Index mapping a contract's address (as a string) to its location in a block */
|
|
39
39
|
#contractIndex: AztecMap<string, BlockIndexValue>;
|
|
40
40
|
|
|
41
|
-
#log =
|
|
41
|
+
#log = createLogger('archiver:block_store');
|
|
42
42
|
|
|
43
43
|
constructor(private db: AztecKVStore) {
|
|
44
44
|
this.#blocks = db.openMap('archiver_blocks');
|
|
@@ -147,14 +147,14 @@ export class BlockStore {
|
|
|
147
147
|
* @param limit - The number of blocks to return.
|
|
148
148
|
* @returns The requested L2 block headers
|
|
149
149
|
*/
|
|
150
|
-
*getBlockHeaders(start: number, limit: number): IterableIterator<
|
|
150
|
+
*getBlockHeaders(start: number, limit: number): IterableIterator<BlockHeader> {
|
|
151
151
|
for (const blockStorage of this.#blocks.values(this.#computeBlockRange(start, limit))) {
|
|
152
|
-
yield
|
|
152
|
+
yield BlockHeader.fromBuffer(blockStorage.header);
|
|
153
153
|
}
|
|
154
154
|
}
|
|
155
155
|
|
|
156
156
|
private getBlockFromBlockStorage(blockStorage: BlockStorage) {
|
|
157
|
-
const header =
|
|
157
|
+
const header = BlockHeader.fromBuffer(blockStorage.header);
|
|
158
158
|
const archive = AppendOnlyTreeSnapshot.fromBuffer(blockStorage.archive);
|
|
159
159
|
const blockHash = header.hash().toString();
|
|
160
160
|
const blockBodyBuffer = this.#blockBodies.get(blockHash);
|
|
@@ -53,7 +53,7 @@ export class ContractClassStore {
|
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
getContractClassIds(): Fr[] {
|
|
56
|
-
return Array.from(this.#contractClasses.keys()).map(key => Fr.
|
|
56
|
+
return Array.from(this.#contractClasses.keys()).map(key => Fr.fromHexString(key));
|
|
57
57
|
}
|
|
58
58
|
|
|
59
59
|
async addFunctions(
|
|
@@ -1,27 +1,25 @@
|
|
|
1
1
|
import {
|
|
2
|
-
type FromLogType,
|
|
3
2
|
type GetUnencryptedLogsResponse,
|
|
4
3
|
type InBlock,
|
|
5
4
|
type InboxLeaf,
|
|
6
5
|
type L2Block,
|
|
7
|
-
type L2BlockL2Logs,
|
|
8
6
|
type LogFilter,
|
|
9
|
-
type LogType,
|
|
10
7
|
type TxHash,
|
|
11
8
|
type TxReceipt,
|
|
12
9
|
type TxScopedL2Log,
|
|
13
10
|
} from '@aztec/circuit-types';
|
|
14
11
|
import {
|
|
12
|
+
type BlockHeader,
|
|
15
13
|
type ContractClassPublic,
|
|
16
14
|
type ContractInstanceWithAddress,
|
|
17
15
|
type ExecutablePrivateFunctionWithMembershipProof,
|
|
18
16
|
type Fr,
|
|
19
|
-
type
|
|
17
|
+
type PrivateLog,
|
|
20
18
|
type UnconstrainedFunctionWithMembershipProof,
|
|
21
19
|
} from '@aztec/circuits.js';
|
|
22
20
|
import { type ContractArtifact, FunctionSelector } from '@aztec/foundation/abi';
|
|
23
21
|
import { type AztecAddress } from '@aztec/foundation/aztec-address';
|
|
24
|
-
import {
|
|
22
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
25
23
|
import { type AztecKVStore } from '@aztec/kv-store';
|
|
26
24
|
|
|
27
25
|
import { type ArchiverDataStore, type ArchiverL1SynchPoint } from '../archiver_store.js';
|
|
@@ -48,7 +46,7 @@ export class KVArchiverDataStore implements ArchiverDataStore {
|
|
|
48
46
|
#contractArtifactStore: ContractArtifactsStore;
|
|
49
47
|
private functionNames = new Map<string, string>();
|
|
50
48
|
|
|
51
|
-
#log =
|
|
49
|
+
#log = createLogger('archiver:data-store');
|
|
52
50
|
|
|
53
51
|
constructor(private db: AztecKVStore, logsMaxPageSize: number = 1000) {
|
|
54
52
|
this.#blockStore = new BlockStore(db);
|
|
@@ -173,7 +171,7 @@ export class KVArchiverDataStore implements ArchiverDataStore {
|
|
|
173
171
|
* @param limit - The number of blocks to return.
|
|
174
172
|
* @returns The requested L2 blocks
|
|
175
173
|
*/
|
|
176
|
-
getBlockHeaders(start: number, limit: number): Promise<
|
|
174
|
+
getBlockHeaders(start: number, limit: number): Promise<BlockHeader[]> {
|
|
177
175
|
try {
|
|
178
176
|
return Promise.resolve(Array.from(this.#blockStore.getBlockHeaders(start, limit)));
|
|
179
177
|
} catch (err) {
|
|
@@ -266,19 +264,14 @@ export class KVArchiverDataStore implements ArchiverDataStore {
|
|
|
266
264
|
}
|
|
267
265
|
|
|
268
266
|
/**
|
|
269
|
-
*
|
|
270
|
-
* @param
|
|
271
|
-
* @param limit - The number of
|
|
272
|
-
* @
|
|
273
|
-
* @returns The requested logs.
|
|
267
|
+
* Retrieves all private logs from up to `limit` blocks, starting from the block number `from`.
|
|
268
|
+
* @param from - The block number from which to begin retrieving logs.
|
|
269
|
+
* @param limit - The maximum number of blocks to retrieve logs from.
|
|
270
|
+
* @returns An array of private logs from the specified range of blocks.
|
|
274
271
|
*/
|
|
275
|
-
|
|
276
|
-
start: number,
|
|
277
|
-
limit: number,
|
|
278
|
-
logType: TLogType,
|
|
279
|
-
): Promise<L2BlockL2Logs<FromLogType<TLogType>>[]> {
|
|
272
|
+
getPrivateLogs(from: number, limit: number): Promise<PrivateLog[]> {
|
|
280
273
|
try {
|
|
281
|
-
return Promise.resolve(Array.from(this.#logStore.
|
|
274
|
+
return Promise.resolve(Array.from(this.#logStore.getPrivateLogs(from, limit)));
|
|
282
275
|
} catch (err) {
|
|
283
276
|
return Promise.reject(err);
|
|
284
277
|
}
|
|
@@ -1,23 +1,18 @@
|
|
|
1
1
|
import {
|
|
2
|
-
type Body,
|
|
3
2
|
ContractClass2BlockL2Logs,
|
|
4
|
-
EncryptedL2BlockL2Logs,
|
|
5
|
-
EncryptedNoteL2BlockL2Logs,
|
|
6
3
|
ExtendedUnencryptedL2Log,
|
|
7
|
-
type FromLogType,
|
|
8
4
|
type GetUnencryptedLogsResponse,
|
|
9
5
|
type L2Block,
|
|
10
|
-
type L2BlockL2Logs,
|
|
11
6
|
type LogFilter,
|
|
12
7
|
LogId,
|
|
13
|
-
LogType,
|
|
14
8
|
TxScopedL2Log,
|
|
15
9
|
UnencryptedL2BlockL2Logs,
|
|
16
10
|
type UnencryptedL2Log,
|
|
17
11
|
} from '@aztec/circuit-types';
|
|
18
|
-
import { Fr } from '@aztec/circuits.js';
|
|
12
|
+
import { Fr, PrivateLog } from '@aztec/circuits.js';
|
|
19
13
|
import { INITIAL_L2_BLOCK_NUM, MAX_NOTE_HASHES_PER_TX } from '@aztec/circuits.js/constants';
|
|
20
|
-
import {
|
|
14
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
15
|
+
import { BufferReader } from '@aztec/foundation/serialize';
|
|
21
16
|
import { type AztecKVStore, type AztecMap } from '@aztec/kv-store';
|
|
22
17
|
|
|
23
18
|
import { type BlockStore } from './block_store.js';
|
|
@@ -26,72 +21,83 @@ import { type BlockStore } from './block_store.js';
|
|
|
26
21
|
* A store for logs
|
|
27
22
|
*/
|
|
28
23
|
export class LogStore {
|
|
29
|
-
#noteEncryptedLogsByBlock: AztecMap<number, Buffer>;
|
|
30
24
|
#logsByTag: AztecMap<string, Buffer[]>;
|
|
31
25
|
#logTagsByBlock: AztecMap<number, string[]>;
|
|
32
|
-
#
|
|
26
|
+
#privateLogsByBlock: AztecMap<number, Buffer>;
|
|
33
27
|
#unencryptedLogsByBlock: AztecMap<number, Buffer>;
|
|
34
28
|
#contractClassLogsByBlock: AztecMap<number, Buffer>;
|
|
35
29
|
#logsMaxPageSize: number;
|
|
36
|
-
#log =
|
|
30
|
+
#log = createLogger('archiver:log_store');
|
|
37
31
|
|
|
38
32
|
constructor(private db: AztecKVStore, private blockStore: BlockStore, logsMaxPageSize: number = 1000) {
|
|
39
|
-
this.#noteEncryptedLogsByBlock = db.openMap('archiver_note_encrypted_logs_by_block');
|
|
40
33
|
this.#logsByTag = db.openMap('archiver_tagged_logs_by_tag');
|
|
41
34
|
this.#logTagsByBlock = db.openMap('archiver_log_tags_by_block');
|
|
42
|
-
this.#
|
|
35
|
+
this.#privateLogsByBlock = db.openMap('archiver_private_logs_by_block');
|
|
43
36
|
this.#unencryptedLogsByBlock = db.openMap('archiver_unencrypted_logs_by_block');
|
|
44
37
|
this.#contractClassLogsByBlock = db.openMap('archiver_contract_class_logs_by_block');
|
|
45
38
|
|
|
46
39
|
this.#logsMaxPageSize = logsMaxPageSize;
|
|
47
40
|
}
|
|
48
41
|
|
|
49
|
-
#
|
|
42
|
+
#extractTaggedLogsFromPrivate(block: L2Block) {
|
|
50
43
|
const taggedLogs = new Map<string, Buffer[]>();
|
|
51
44
|
const dataStartIndexForBlock =
|
|
52
45
|
block.header.state.partial.noteHashTree.nextAvailableLeafIndex -
|
|
53
46
|
block.body.numberOfTxsIncludingPadded * MAX_NOTE_HASHES_PER_TX;
|
|
54
|
-
block.body
|
|
47
|
+
block.body.txEffects.forEach((txEffect, txIndex) => {
|
|
48
|
+
const txHash = txEffect.txHash;
|
|
49
|
+
const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NOTE_HASHES_PER_TX;
|
|
50
|
+
txEffect.privateLogs.forEach(log => {
|
|
51
|
+
const tag = log.fields[0];
|
|
52
|
+
const currentLogs = taggedLogs.get(tag.toString()) ?? [];
|
|
53
|
+
currentLogs.push(
|
|
54
|
+
new TxScopedL2Log(
|
|
55
|
+
txHash,
|
|
56
|
+
dataStartIndexForTx,
|
|
57
|
+
block.number,
|
|
58
|
+
/* isFromPublic */ false,
|
|
59
|
+
log.toBuffer(),
|
|
60
|
+
).toBuffer(),
|
|
61
|
+
);
|
|
62
|
+
taggedLogs.set(tag.toString(), currentLogs);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
return taggedLogs;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
#extractTaggedLogsFromPublic(block: L2Block) {
|
|
69
|
+
const taggedLogs = new Map<string, Buffer[]>();
|
|
70
|
+
const dataStartIndexForBlock =
|
|
71
|
+
block.header.state.partial.noteHashTree.nextAvailableLeafIndex -
|
|
72
|
+
block.body.numberOfTxsIncludingPadded * MAX_NOTE_HASHES_PER_TX;
|
|
73
|
+
block.body.unencryptedLogs.txLogs.forEach((txLogs, txIndex) => {
|
|
55
74
|
const txHash = block.body.txEffects[txIndex].txHash;
|
|
56
75
|
const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NOTE_HASHES_PER_TX;
|
|
57
76
|
const logs = txLogs.unrollLogs();
|
|
58
77
|
logs.forEach(log => {
|
|
59
|
-
if (
|
|
60
|
-
(logType == 'noteEncryptedLogs' && log.data.length < 32) ||
|
|
78
|
+
if (log.data.length < 32 * 33) {
|
|
61
79
|
// TODO remove when #9835 and #9836 are fixed
|
|
62
|
-
(
|
|
63
|
-
) {
|
|
64
|
-
this.#log.warn(`Skipping log (${logType}) with invalid data length: ${log.data.length}`);
|
|
80
|
+
this.#log.warn(`Skipping unencrypted log with insufficient data length: ${log.data.length}`);
|
|
65
81
|
return;
|
|
66
82
|
}
|
|
67
83
|
try {
|
|
68
|
-
let tag = Fr.ZERO;
|
|
69
84
|
// TODO remove when #9835 and #9836 are fixed. The partial note logs are emitted as bytes, but encoded as Fields.
|
|
70
85
|
// This means that for every 32 bytes of payload, we only have 1 byte of data.
|
|
71
86
|
// Also, the tag is not stored in the first 32 bytes of the log, (that's the length of public fields now) but in the next 32.
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
correctedBuffer.writeUInt8(byte, i);
|
|
80
|
-
}
|
|
81
|
-
tag = new Fr(correctedBuffer);
|
|
82
|
-
} else {
|
|
83
|
-
tag = new Fr(log.data.subarray(0, 32));
|
|
87
|
+
const correctedBuffer = Buffer.alloc(32);
|
|
88
|
+
const initialOffset = 32;
|
|
89
|
+
for (let i = 0; i < 32; i++) {
|
|
90
|
+
const byte = Fr.fromBuffer(
|
|
91
|
+
log.data.subarray(i * 32 + initialOffset, i * 32 + 32 + initialOffset),
|
|
92
|
+
).toNumber();
|
|
93
|
+
correctedBuffer.writeUInt8(byte, i);
|
|
84
94
|
}
|
|
85
|
-
|
|
95
|
+
const tag = new Fr(correctedBuffer);
|
|
96
|
+
|
|
97
|
+
this.#log.debug(`Found tagged unencrypted log with tag ${tag.toString()} in block ${block.number}`);
|
|
86
98
|
const currentLogs = taggedLogs.get(tag.toString()) ?? [];
|
|
87
99
|
currentLogs.push(
|
|
88
|
-
new TxScopedL2Log(
|
|
89
|
-
txHash,
|
|
90
|
-
dataStartIndexForTx,
|
|
91
|
-
block.number,
|
|
92
|
-
logType === 'unencryptedLogs',
|
|
93
|
-
log.data,
|
|
94
|
-
).toBuffer(),
|
|
100
|
+
new TxScopedL2Log(txHash, dataStartIndexForTx, block.number, /* isFromPublic */ true, log.data).toBuffer(),
|
|
95
101
|
);
|
|
96
102
|
taggedLogs.set(tag.toString(), currentLogs);
|
|
97
103
|
} catch (err) {
|
|
@@ -109,10 +115,7 @@ export class LogStore {
|
|
|
109
115
|
*/
|
|
110
116
|
async addLogs(blocks: L2Block[]): Promise<boolean> {
|
|
111
117
|
const taggedLogsToAdd = blocks
|
|
112
|
-
.flatMap(block => [
|
|
113
|
-
this.#extractTaggedLogs(block, 'noteEncryptedLogs'),
|
|
114
|
-
this.#extractTaggedLogs(block, 'unencryptedLogs'),
|
|
115
|
-
])
|
|
118
|
+
.flatMap(block => [this.#extractTaggedLogsFromPrivate(block), this.#extractTaggedLogsFromPublic(block)])
|
|
116
119
|
.reduce((acc, val) => {
|
|
117
120
|
for (const [tag, logs] of val.entries()) {
|
|
118
121
|
const currentLogs = acc.get(tag) ?? [];
|
|
@@ -140,8 +143,13 @@ export class LogStore {
|
|
|
140
143
|
tagsInBlock.push(tag);
|
|
141
144
|
}
|
|
142
145
|
void this.#logTagsByBlock.set(block.number, tagsInBlock);
|
|
143
|
-
|
|
144
|
-
|
|
146
|
+
|
|
147
|
+
const privateLogsInBlock = block.body.txEffects
|
|
148
|
+
.map(txEffect => txEffect.privateLogs)
|
|
149
|
+
.flat()
|
|
150
|
+
.map(log => log.toBuffer());
|
|
151
|
+
void this.#privateLogsByBlock.set(block.number, Buffer.concat(privateLogsInBlock));
|
|
152
|
+
|
|
145
153
|
void this.#unencryptedLogsByBlock.set(block.number, block.body.unencryptedLogs.toBuffer());
|
|
146
154
|
void this.#contractClassLogsByBlock.set(block.number, block.body.contractClassLogs.toBuffer());
|
|
147
155
|
});
|
|
@@ -156,8 +164,7 @@ export class LogStore {
|
|
|
156
164
|
});
|
|
157
165
|
return this.db.transaction(() => {
|
|
158
166
|
blocks.forEach(block => {
|
|
159
|
-
void this.#
|
|
160
|
-
void this.#encryptedLogsByBlock.delete(block.number);
|
|
167
|
+
void this.#privateLogsByBlock.delete(block.number);
|
|
161
168
|
void this.#unencryptedLogsByBlock.delete(block.number);
|
|
162
169
|
void this.#logTagsByBlock.delete(block.number);
|
|
163
170
|
});
|
|
@@ -171,43 +178,20 @@ export class LogStore {
|
|
|
171
178
|
}
|
|
172
179
|
|
|
173
180
|
/**
|
|
174
|
-
*
|
|
175
|
-
* @param start -
|
|
176
|
-
* @param limit - The number of
|
|
177
|
-
* @
|
|
178
|
-
* @returns The requested logs.
|
|
181
|
+
* Retrieves all private logs from up to `limit` blocks, starting from the block number `start`.
|
|
182
|
+
* @param start - The block number from which to begin retrieving logs.
|
|
183
|
+
* @param limit - The maximum number of blocks to retrieve logs from.
|
|
184
|
+
* @returns An array of private logs from the specified range of blocks.
|
|
179
185
|
*/
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
limit
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
switch (logType) {
|
|
187
|
-
case LogType.ENCRYPTED:
|
|
188
|
-
return this.#encryptedLogsByBlock;
|
|
189
|
-
case LogType.NOTEENCRYPTED:
|
|
190
|
-
return this.#noteEncryptedLogsByBlock;
|
|
191
|
-
case LogType.UNENCRYPTED:
|
|
192
|
-
default:
|
|
193
|
-
return this.#unencryptedLogsByBlock;
|
|
194
|
-
}
|
|
195
|
-
})();
|
|
196
|
-
const logTypeMap = (() => {
|
|
197
|
-
switch (logType) {
|
|
198
|
-
case LogType.ENCRYPTED:
|
|
199
|
-
return EncryptedL2BlockL2Logs;
|
|
200
|
-
case LogType.NOTEENCRYPTED:
|
|
201
|
-
return EncryptedNoteL2BlockL2Logs;
|
|
202
|
-
case LogType.UNENCRYPTED:
|
|
203
|
-
default:
|
|
204
|
-
return UnencryptedL2BlockL2Logs;
|
|
186
|
+
getPrivateLogs(start: number, limit: number) {
|
|
187
|
+
const logs = [];
|
|
188
|
+
for (const buffer of this.#privateLogsByBlock.values({ start, limit })) {
|
|
189
|
+
const reader = new BufferReader(buffer);
|
|
190
|
+
while (reader.remainingBytes() > 0) {
|
|
191
|
+
logs.push(reader.readObject(PrivateLog));
|
|
205
192
|
}
|
|
206
|
-
})();
|
|
207
|
-
const L2BlockL2Logs = logTypeMap;
|
|
208
|
-
for (const buffer of logMap.values({ start, limit })) {
|
|
209
|
-
yield L2BlockL2Logs.fromBuffer(buffer) as L2BlockL2Logs<FromLogType<TLogType>>;
|
|
210
193
|
}
|
|
194
|
+
return logs;
|
|
211
195
|
}
|
|
212
196
|
|
|
213
197
|
/**
|
|
@@ -249,7 +233,9 @@ export class LogStore {
|
|
|
249
233
|
return { logs: [], maxLogsHit: false };
|
|
250
234
|
}
|
|
251
235
|
|
|
252
|
-
const
|
|
236
|
+
const buffer = this.#unencryptedLogsByBlock.get(blockNumber) ?? Buffer.alloc(0);
|
|
237
|
+
const unencryptedLogsInBlock = UnencryptedL2BlockL2Logs.fromBuffer(buffer);
|
|
238
|
+
|
|
253
239
|
const txLogs = unencryptedLogsInBlock.txLogs[txIndex].unrollLogs();
|
|
254
240
|
|
|
255
241
|
const logs: ExtendedUnencryptedL2Log[] = [];
|
|
@@ -376,40 +362,4 @@ export class LogStore {
|
|
|
376
362
|
|
|
377
363
|
return maxLogsHit;
|
|
378
364
|
}
|
|
379
|
-
|
|
380
|
-
#getBlockLogs<TLogType extends LogType>(
|
|
381
|
-
blockNumber: number,
|
|
382
|
-
logType: TLogType,
|
|
383
|
-
): L2BlockL2Logs<FromLogType<TLogType>> {
|
|
384
|
-
const logMap = (() => {
|
|
385
|
-
switch (logType) {
|
|
386
|
-
case LogType.ENCRYPTED:
|
|
387
|
-
return this.#encryptedLogsByBlock;
|
|
388
|
-
case LogType.NOTEENCRYPTED:
|
|
389
|
-
return this.#noteEncryptedLogsByBlock;
|
|
390
|
-
case LogType.UNENCRYPTED:
|
|
391
|
-
default:
|
|
392
|
-
return this.#unencryptedLogsByBlock;
|
|
393
|
-
}
|
|
394
|
-
})();
|
|
395
|
-
const logTypeMap = (() => {
|
|
396
|
-
switch (logType) {
|
|
397
|
-
case LogType.ENCRYPTED:
|
|
398
|
-
return EncryptedL2BlockL2Logs;
|
|
399
|
-
case LogType.NOTEENCRYPTED:
|
|
400
|
-
return EncryptedNoteL2BlockL2Logs;
|
|
401
|
-
case LogType.UNENCRYPTED:
|
|
402
|
-
default:
|
|
403
|
-
return UnencryptedL2BlockL2Logs;
|
|
404
|
-
}
|
|
405
|
-
})();
|
|
406
|
-
const L2BlockL2Logs = logTypeMap;
|
|
407
|
-
const buffer = logMap.get(blockNumber);
|
|
408
|
-
|
|
409
|
-
if (!buffer) {
|
|
410
|
-
return new L2BlockL2Logs([]) as L2BlockL2Logs<FromLogType<TLogType>>;
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
return L2BlockL2Logs.fromBuffer(buffer) as L2BlockL2Logs<FromLogType<TLogType>>;
|
|
414
|
-
}
|
|
415
365
|
}
|