@aztec/prover-node 0.0.1-fake-ceab37513c → 0.0.6-commit.a2d1860fe9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/actions/download-epoch-proving-job.d.ts +4 -4
- package/dest/actions/index.d.ts +1 -1
- package/dest/actions/rerun-epoch-proving-job.d.ts +3 -2
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +5 -3
- package/dest/actions/upload-epoch-proof-failure.d.ts +1 -1
- package/dest/bin/run-failed-epoch.d.ts +1 -1
- package/dest/bin/run-failed-epoch.js +5 -2
- package/dest/config.d.ts +8 -10
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +17 -19
- package/dest/factory.d.ts +20 -15
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +30 -62
- package/dest/index.d.ts +2 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/job/epoch-proving-job-data.d.ts +8 -6
- package/dest/job/epoch-proving-job-data.d.ts.map +1 -1
- package/dest/job/epoch-proving-job-data.js +25 -18
- package/dest/job/epoch-proving-job.d.ts +7 -13
- package/dest/job/epoch-proving-job.d.ts.map +1 -1
- package/dest/job/epoch-proving-job.js +513 -89
- package/dest/metrics.d.ts +4 -3
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +30 -98
- package/dest/monitors/epoch-monitor.d.ts +3 -2
- package/dest/monitors/epoch-monitor.d.ts.map +1 -1
- package/dest/monitors/epoch-monitor.js +3 -11
- package/dest/monitors/index.d.ts +1 -1
- package/dest/prover-node-publisher.d.ts +14 -11
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +53 -46
- package/dest/prover-node.d.ts +26 -15
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +441 -57
- package/dest/prover-publisher-factory.d.ts +9 -5
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +4 -2
- package/dest/test/index.d.ts +1 -1
- package/dest/test/index.d.ts.map +1 -1
- package/package.json +26 -25
- package/src/actions/rerun-epoch-proving-job.ts +5 -3
- package/src/bin/run-failed-epoch.ts +5 -2
- package/src/config.ts +26 -32
- package/src/factory.ts +68 -99
- package/src/index.ts +1 -0
- package/src/job/epoch-proving-job-data.ts +31 -25
- package/src/job/epoch-proving-job.ts +155 -92
- package/src/metrics.ts +31 -82
- package/src/monitors/epoch-monitor.ts +5 -11
- package/src/prover-node-publisher.ts +76 -61
- package/src/prover-node.ts +60 -50
- package/src/prover-publisher-factory.ts +16 -8
|
@@ -1,49 +1,55 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
|
|
2
|
+
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
2
3
|
import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
|
|
3
|
-
import { CommitteeAttestation
|
|
4
|
+
import { CommitteeAttestation } from '@aztec/stdlib/block';
|
|
5
|
+
import { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
4
6
|
import { BlockHeader, Tx } from '@aztec/stdlib/tx';
|
|
5
7
|
|
|
6
8
|
/** All data from an epoch used in proving. */
|
|
7
9
|
export type EpochProvingJobData = {
|
|
8
|
-
epochNumber:
|
|
9
|
-
|
|
10
|
+
epochNumber: EpochNumber;
|
|
11
|
+
checkpoints: Checkpoint[];
|
|
10
12
|
txs: Map<string, Tx>;
|
|
11
|
-
l1ToL2Messages: Record<
|
|
13
|
+
l1ToL2Messages: Record<CheckpointNumber, Fr[]>;
|
|
12
14
|
previousBlockHeader: BlockHeader;
|
|
13
15
|
attestations: CommitteeAttestation[];
|
|
14
16
|
};
|
|
15
17
|
|
|
16
18
|
export function validateEpochProvingJobData(data: EpochProvingJobData) {
|
|
17
|
-
if (data.
|
|
19
|
+
if (data.checkpoints.length === 0) {
|
|
20
|
+
throw new Error('No checkpoints to prove');
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const firstBlockNumber = data.checkpoints[0].blocks[0].number;
|
|
24
|
+
const previousBlockNumber = data.previousBlockHeader.getBlockNumber();
|
|
25
|
+
if (previousBlockNumber + 1 !== firstBlockNumber) {
|
|
18
26
|
throw new Error(
|
|
19
|
-
`Initial block number ${
|
|
20
|
-
data.blocks[0].number
|
|
21
|
-
} does not match previous block header ${data.previousBlockHeader.getBlockNumber()}`,
|
|
27
|
+
`Initial block number ${firstBlockNumber} does not match previous block header ${previousBlockNumber}`,
|
|
22
28
|
);
|
|
23
29
|
}
|
|
24
30
|
|
|
25
|
-
for (const
|
|
26
|
-
if (!(
|
|
27
|
-
throw new Error(`Missing L1 to L2 messages for
|
|
31
|
+
for (const checkpoint of data.checkpoints) {
|
|
32
|
+
if (!(checkpoint.number in data.l1ToL2Messages)) {
|
|
33
|
+
throw new Error(`Missing L1 to L2 messages for checkpoint number ${checkpoint.number}`);
|
|
28
34
|
}
|
|
29
35
|
}
|
|
30
36
|
}
|
|
31
37
|
|
|
32
38
|
export function serializeEpochProvingJobData(data: EpochProvingJobData): Buffer {
|
|
33
|
-
const
|
|
39
|
+
const checkpoints = data.checkpoints.map(checkpoint => checkpoint.toBuffer());
|
|
34
40
|
const txs = Array.from(data.txs.values()).map(tx => tx.toBuffer());
|
|
35
|
-
const l1ToL2Messages = Object.entries(data.l1ToL2Messages).map(([
|
|
36
|
-
Number(
|
|
41
|
+
const l1ToL2Messages = Object.entries(data.l1ToL2Messages).map(([checkpointNumber, messages]) => [
|
|
42
|
+
Number(checkpointNumber),
|
|
37
43
|
messages.length,
|
|
38
44
|
...messages,
|
|
39
45
|
]);
|
|
40
46
|
const attestations = data.attestations.map(attestation => attestation.toBuffer());
|
|
41
47
|
|
|
42
48
|
return serializeToBuffer(
|
|
43
|
-
|
|
49
|
+
data.epochNumber,
|
|
44
50
|
data.previousBlockHeader,
|
|
45
|
-
|
|
46
|
-
...
|
|
51
|
+
checkpoints.length,
|
|
52
|
+
...checkpoints,
|
|
47
53
|
txs.length,
|
|
48
54
|
...txs,
|
|
49
55
|
l1ToL2Messages.length,
|
|
@@ -55,22 +61,22 @@ export function serializeEpochProvingJobData(data: EpochProvingJobData): Buffer
|
|
|
55
61
|
|
|
56
62
|
export function deserializeEpochProvingJobData(buf: Buffer): EpochProvingJobData {
|
|
57
63
|
const reader = BufferReader.asReader(buf);
|
|
58
|
-
const epochNumber =
|
|
64
|
+
const epochNumber = EpochNumber(reader.readNumber());
|
|
59
65
|
const previousBlockHeader = reader.readObject(BlockHeader);
|
|
60
|
-
const
|
|
66
|
+
const checkpoints = reader.readVector(Checkpoint);
|
|
61
67
|
const txArray = reader.readVector(Tx);
|
|
62
68
|
|
|
63
|
-
const
|
|
69
|
+
const l1ToL2MessageCheckpointCount = reader.readNumber();
|
|
64
70
|
const l1ToL2Messages: Record<number, Fr[]> = {};
|
|
65
|
-
for (let i = 0; i <
|
|
66
|
-
const
|
|
71
|
+
for (let i = 0; i < l1ToL2MessageCheckpointCount; i++) {
|
|
72
|
+
const checkpointNumber = CheckpointNumber(reader.readNumber());
|
|
67
73
|
const messages = reader.readVector(Fr);
|
|
68
|
-
l1ToL2Messages[
|
|
74
|
+
l1ToL2Messages[checkpointNumber] = messages;
|
|
69
75
|
}
|
|
70
76
|
|
|
71
77
|
const attestations = reader.readVector(CommitteeAttestation);
|
|
72
78
|
|
|
73
79
|
const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
|
|
74
80
|
|
|
75
|
-
return { epochNumber, previousBlockHeader,
|
|
81
|
+
return { epochNumber, previousBlockHeader, checkpoints, txs, l1ToL2Messages, attestations };
|
|
76
82
|
}
|
|
@@ -1,19 +1,25 @@
|
|
|
1
|
-
import { BatchedBlob, Blob } from '@aztec/blob-lib';
|
|
2
1
|
import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
|
|
3
2
|
import { asyncPool } from '@aztec/foundation/async-pool';
|
|
3
|
+
import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
|
|
4
4
|
import { padArrayEnd } from '@aztec/foundation/collection';
|
|
5
|
-
import { Fr } from '@aztec/foundation/
|
|
6
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
5
|
+
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
6
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
7
7
|
import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
8
8
|
import { Timer } from '@aztec/foundation/timer';
|
|
9
|
+
import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
|
|
10
|
+
import { protocolContractsHash } from '@aztec/protocol-contracts';
|
|
11
|
+
import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
|
|
9
12
|
import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
|
|
13
|
+
import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
|
|
10
14
|
import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
|
|
15
|
+
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
11
16
|
import {
|
|
12
17
|
type EpochProver,
|
|
13
18
|
type EpochProvingJobState,
|
|
14
19
|
EpochProvingJobTerminalState,
|
|
15
20
|
type ForkMerkleTreeOperations,
|
|
16
21
|
} from '@aztec/stdlib/interfaces/server';
|
|
22
|
+
import { CheckpointConstantData } from '@aztec/stdlib/rollup';
|
|
17
23
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
18
24
|
import type { ProcessedTx, Tx } from '@aztec/stdlib/tx';
|
|
19
25
|
import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
|
|
@@ -37,7 +43,7 @@ export type EpochProvingJobOptions = {
|
|
|
37
43
|
*/
|
|
38
44
|
export class EpochProvingJob implements Traceable {
|
|
39
45
|
private state: EpochProvingJobState = 'initialized';
|
|
40
|
-
private log
|
|
46
|
+
private log: Logger;
|
|
41
47
|
private uuid: string;
|
|
42
48
|
|
|
43
49
|
private runPromise: Promise<void> | undefined;
|
|
@@ -56,9 +62,14 @@ export class EpochProvingJob implements Traceable {
|
|
|
56
62
|
private metrics: ProverNodeJobMetrics,
|
|
57
63
|
private deadline: Date | undefined,
|
|
58
64
|
private config: EpochProvingJobOptions,
|
|
65
|
+
bindings?: LoggerBindings,
|
|
59
66
|
) {
|
|
60
67
|
validateEpochProvingJobData(data);
|
|
61
68
|
this.uuid = crypto.randomUUID();
|
|
69
|
+
this.log = createLogger('prover-node:epoch-proving-job', {
|
|
70
|
+
...bindings,
|
|
71
|
+
instanceId: `epoch-${data.epochNumber}`,
|
|
72
|
+
});
|
|
62
73
|
this.tracer = metrics.tracer;
|
|
63
74
|
}
|
|
64
75
|
|
|
@@ -70,7 +81,7 @@ export class EpochProvingJob implements Traceable {
|
|
|
70
81
|
return this.state;
|
|
71
82
|
}
|
|
72
83
|
|
|
73
|
-
public getEpochNumber():
|
|
84
|
+
public getEpochNumber(): EpochNumber {
|
|
74
85
|
return this.data.epochNumber;
|
|
75
86
|
}
|
|
76
87
|
|
|
@@ -86,8 +97,8 @@ export class EpochProvingJob implements Traceable {
|
|
|
86
97
|
return this.data.epochNumber;
|
|
87
98
|
}
|
|
88
99
|
|
|
89
|
-
private get
|
|
90
|
-
return this.data.
|
|
100
|
+
private get checkpoints() {
|
|
101
|
+
return this.data.checkpoints;
|
|
91
102
|
}
|
|
92
103
|
|
|
93
104
|
private get txs() {
|
|
@@ -102,7 +113,7 @@ export class EpochProvingJob implements Traceable {
|
|
|
102
113
|
* Proves the given epoch and submits the proof to L1.
|
|
103
114
|
*/
|
|
104
115
|
@trackSpan('EpochProvingJob.run', function () {
|
|
105
|
-
return { [Attributes.EPOCH_NUMBER]:
|
|
116
|
+
return { [Attributes.EPOCH_NUMBER]: this.data.epochNumber };
|
|
106
117
|
})
|
|
107
118
|
public async run() {
|
|
108
119
|
this.scheduleDeadlineStop();
|
|
@@ -111,14 +122,22 @@ export class EpochProvingJob implements Traceable {
|
|
|
111
122
|
}
|
|
112
123
|
|
|
113
124
|
const attestations = this.attestations.map(attestation => attestation.toViem());
|
|
114
|
-
const epochNumber =
|
|
115
|
-
const
|
|
116
|
-
const
|
|
117
|
-
const
|
|
118
|
-
|
|
125
|
+
const epochNumber = this.epochNumber;
|
|
126
|
+
const epochSizeCheckpoints = this.checkpoints.length;
|
|
127
|
+
const epochSizeBlocks = this.checkpoints.reduce((accum, checkpoint) => accum + checkpoint.blocks.length, 0);
|
|
128
|
+
const epochSizeTxs = this.checkpoints.reduce(
|
|
129
|
+
(accum, checkpoint) =>
|
|
130
|
+
accum + checkpoint.blocks.reduce((accumC, block) => accumC + block.body.txEffects.length, 0),
|
|
131
|
+
0,
|
|
132
|
+
);
|
|
133
|
+
const fromCheckpoint = this.checkpoints[0].number;
|
|
134
|
+
const toCheckpoint = this.checkpoints.at(-1)!.number;
|
|
135
|
+
const fromBlock = this.checkpoints[0].blocks[0].number;
|
|
136
|
+
const toBlock = this.checkpoints.at(-1)!.blocks.at(-1)!.number;
|
|
137
|
+
this.log.info(`Starting epoch ${epochNumber} proving job with checkpoints ${fromCheckpoint} to ${toCheckpoint}`, {
|
|
119
138
|
fromBlock,
|
|
120
139
|
toBlock,
|
|
121
|
-
|
|
140
|
+
epochSizeTxs,
|
|
122
141
|
epochNumber,
|
|
123
142
|
uuid: this.uuid,
|
|
124
143
|
});
|
|
@@ -129,51 +148,100 @@ export class EpochProvingJob implements Traceable {
|
|
|
129
148
|
this.runPromise = promise;
|
|
130
149
|
|
|
131
150
|
try {
|
|
132
|
-
const
|
|
133
|
-
|
|
134
|
-
|
|
151
|
+
const blobFieldsPerCheckpoint = this.checkpoints.map(checkpoint => checkpoint.toBlobFields());
|
|
152
|
+
this.log.info(`Blob fields per checkpoint: ${timer.ms()}ms`);
|
|
153
|
+
const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
|
|
154
|
+
this.log.info(`Final blob batching challeneger: ${timer.ms()}ms`);
|
|
155
|
+
|
|
156
|
+
this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
|
|
157
|
+
await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
|
|
158
|
+
|
|
159
|
+
// Everything in the epoch should have the same chainId and version.
|
|
160
|
+
const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
|
|
135
161
|
|
|
136
|
-
const
|
|
137
|
-
this.prover.startNewEpoch(epochNumber, fromBlock, epochSizeBlocks, finalBlobBatchingChallenges);
|
|
138
|
-
await this.prover.startTubeCircuits(Array.from(this.txs.values()));
|
|
162
|
+
const previousBlockHeaders = this.gatherPreviousBlockHeaders();
|
|
139
163
|
|
|
140
|
-
await asyncPool(this.config.parallelBlockLimit ?? 32, this.
|
|
164
|
+
await asyncPool(this.config.parallelBlockLimit ?? 32, this.checkpoints, async checkpoint => {
|
|
141
165
|
this.checkState();
|
|
142
166
|
|
|
143
|
-
const
|
|
144
|
-
const
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
|
|
155
|
-
previousHeader: previousHeader.hash(),
|
|
156
|
-
uuid: this.uuid,
|
|
157
|
-
...globalVariables,
|
|
167
|
+
const checkpointIndex = checkpoint.number - fromCheckpoint;
|
|
168
|
+
const checkpointConstants = CheckpointConstantData.from({
|
|
169
|
+
chainId,
|
|
170
|
+
version,
|
|
171
|
+
vkTreeRoot: getVKTreeRoot(),
|
|
172
|
+
protocolContractsHash: protocolContractsHash,
|
|
173
|
+
proverId: this.prover.getProverId().toField(),
|
|
174
|
+
slotNumber: checkpoint.header.slotNumber,
|
|
175
|
+
coinbase: checkpoint.header.coinbase,
|
|
176
|
+
feeRecipient: checkpoint.header.feeRecipient,
|
|
177
|
+
gasFees: checkpoint.header.gasFees,
|
|
158
178
|
});
|
|
179
|
+
const previousHeader = previousBlockHeaders[checkpointIndex];
|
|
180
|
+
const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
|
|
159
181
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, true);
|
|
166
|
-
const processed = await this.processTxs(publicProcessor, txs);
|
|
167
|
-
await this.prover.addTxs(processed);
|
|
168
|
-
await db.close();
|
|
169
|
-
this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
|
|
170
|
-
blockNumber: block.number,
|
|
171
|
-
blockHash: (await block.hash()).toString(),
|
|
182
|
+
this.log.verbose(`Starting processing checkpoint ${checkpoint.number}`, {
|
|
183
|
+
number: checkpoint.number,
|
|
184
|
+
checkpointHash: checkpoint.hash().toString(),
|
|
185
|
+
lastArchive: checkpoint.header.lastArchiveRoot,
|
|
186
|
+
previousHeader: previousHeader.hash(),
|
|
172
187
|
uuid: this.uuid,
|
|
173
188
|
});
|
|
174
189
|
|
|
175
|
-
|
|
176
|
-
|
|
190
|
+
await this.prover.startNewCheckpoint(
|
|
191
|
+
checkpointIndex,
|
|
192
|
+
checkpointConstants,
|
|
193
|
+
l1ToL2Messages,
|
|
194
|
+
checkpoint.blocks.length,
|
|
195
|
+
previousHeader,
|
|
196
|
+
);
|
|
197
|
+
|
|
198
|
+
for (let blockIndex = 0; blockIndex < checkpoint.blocks.length; blockIndex++) {
|
|
199
|
+
const block = checkpoint.blocks[blockIndex];
|
|
200
|
+
const globalVariables = block.header.globalVariables;
|
|
201
|
+
const txs = this.getTxs(block);
|
|
202
|
+
|
|
203
|
+
this.log.verbose(`Starting processing block ${block.number}`, {
|
|
204
|
+
number: block.number,
|
|
205
|
+
blockHash: (await block.hash()).toString(),
|
|
206
|
+
lastArchive: block.header.lastArchive.root,
|
|
207
|
+
noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
|
|
208
|
+
nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
|
|
209
|
+
publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
|
|
210
|
+
...globalVariables,
|
|
211
|
+
numTxs: txs.length,
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
// Start block proving
|
|
215
|
+
await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
|
|
216
|
+
|
|
217
|
+
// Process public fns. L1 to L2 messages are only inserted for the first block of a checkpoint,
|
|
218
|
+
// as the fork for subsequent blocks already includes them from the previous block's synced state.
|
|
219
|
+
const db = await this.createFork(
|
|
220
|
+
BlockNumber(block.number - 1),
|
|
221
|
+
blockIndex === 0 ? l1ToL2Messages : undefined,
|
|
222
|
+
);
|
|
223
|
+
const config = PublicSimulatorConfig.from({
|
|
224
|
+
proverId: this.prover.getProverId().toField(),
|
|
225
|
+
skipFeeEnforcement: false,
|
|
226
|
+
collectDebugLogs: false,
|
|
227
|
+
collectHints: true,
|
|
228
|
+
collectPublicInputs: true,
|
|
229
|
+
collectStatistics: false,
|
|
230
|
+
});
|
|
231
|
+
const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
|
|
232
|
+
const processed = await this.processTxs(publicProcessor, txs);
|
|
233
|
+
await this.prover.addTxs(processed);
|
|
234
|
+
await db.close();
|
|
235
|
+
this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
|
|
236
|
+
blockNumber: block.number,
|
|
237
|
+
blockHash: (await block.hash()).toString(),
|
|
238
|
+
uuid: this.uuid,
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
// Mark block as completed to pad it
|
|
242
|
+
const expectedBlockHeader = block.header;
|
|
243
|
+
await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
|
|
244
|
+
}
|
|
177
245
|
});
|
|
178
246
|
|
|
179
247
|
const executionTime = timer.ms();
|
|
@@ -186,16 +254,16 @@ export class EpochProvingJob implements Traceable {
|
|
|
186
254
|
|
|
187
255
|
if (this.config.skipSubmitProof) {
|
|
188
256
|
this.log.info(
|
|
189
|
-
`Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (
|
|
257
|
+
`Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`,
|
|
190
258
|
);
|
|
191
259
|
this.state = 'completed';
|
|
192
|
-
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
|
|
260
|
+
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
|
|
193
261
|
return;
|
|
194
262
|
}
|
|
195
263
|
|
|
196
264
|
const success = await this.publisher.submitEpochProof({
|
|
197
|
-
|
|
198
|
-
|
|
265
|
+
fromCheckpoint,
|
|
266
|
+
toCheckpoint,
|
|
199
267
|
epochNumber,
|
|
200
268
|
publicInputs,
|
|
201
269
|
proof,
|
|
@@ -206,12 +274,12 @@ export class EpochProvingJob implements Traceable {
|
|
|
206
274
|
throw new Error('Failed to submit epoch proof to L1');
|
|
207
275
|
}
|
|
208
276
|
|
|
209
|
-
this.log.info(`Submitted proof for epoch ${epochNumber} (
|
|
277
|
+
this.log.info(`Submitted proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`, {
|
|
210
278
|
epochNumber,
|
|
211
279
|
uuid: this.uuid,
|
|
212
280
|
});
|
|
213
281
|
this.state = 'completed';
|
|
214
|
-
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
|
|
282
|
+
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
|
|
215
283
|
} catch (err: any) {
|
|
216
284
|
if (err && err.name === 'HaltExecutionError') {
|
|
217
285
|
this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
|
|
@@ -234,22 +302,29 @@ export class EpochProvingJob implements Traceable {
|
|
|
234
302
|
}
|
|
235
303
|
|
|
236
304
|
/**
|
|
237
|
-
* Create a new db fork for tx processing, inserting
|
|
305
|
+
* Create a new db fork for tx processing, optionally inserting L1 to L2 messages.
|
|
306
|
+
* L1 to L2 messages should only be inserted for the first block in a checkpoint,
|
|
307
|
+
* as subsequent blocks' synced state already includes them.
|
|
238
308
|
* REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
|
|
239
309
|
*/
|
|
240
|
-
private async createFork(blockNumber:
|
|
310
|
+
private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[] | undefined) {
|
|
311
|
+
this.log.verbose(`Creating fork at ${blockNumber}`, { blockNumber });
|
|
241
312
|
const db = await this.dbProvider.fork(blockNumber);
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
313
|
+
|
|
314
|
+
if (l1ToL2Messages !== undefined) {
|
|
315
|
+
this.log.verbose(`Inserting ${l1ToL2Messages.length} L1 to L2 messages in fork`, {
|
|
316
|
+
blockNumber,
|
|
317
|
+
l1ToL2Messages: l1ToL2Messages.map(m => m.toString()),
|
|
318
|
+
});
|
|
319
|
+
const l1ToL2MessagesPadded = padArrayEnd<Fr, number>(
|
|
320
|
+
l1ToL2Messages,
|
|
321
|
+
Fr.ZERO,
|
|
322
|
+
NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
|
|
323
|
+
'Too many L1 to L2 messages',
|
|
324
|
+
);
|
|
325
|
+
await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
|
|
326
|
+
}
|
|
327
|
+
|
|
253
328
|
return db;
|
|
254
329
|
}
|
|
255
330
|
|
|
@@ -267,7 +342,6 @@ export class EpochProvingJob implements Traceable {
|
|
|
267
342
|
public async stop(state: EpochProvingJobTerminalState = 'stopped') {
|
|
268
343
|
this.state = state;
|
|
269
344
|
this.prover.cancel();
|
|
270
|
-
// TODO(palla/prover): Stop the publisher as well
|
|
271
345
|
if (this.runPromise) {
|
|
272
346
|
await this.runPromise;
|
|
273
347
|
}
|
|
@@ -307,11 +381,12 @@ export class EpochProvingJob implements Traceable {
|
|
|
307
381
|
const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
|
|
308
382
|
this.epochCheckPromise = new RunningPromise(
|
|
309
383
|
async () => {
|
|
310
|
-
const
|
|
311
|
-
const blockHashes = await Promise.all(
|
|
312
|
-
const
|
|
384
|
+
const blockHeaders = await l2BlockSource.getCheckpointedBlockHeadersForEpoch(this.epochNumber);
|
|
385
|
+
const blockHashes = await Promise.all(blockHeaders.map(header => header.hash()));
|
|
386
|
+
const thisBlocks = this.checkpoints.flatMap(checkpoint => checkpoint.blocks);
|
|
387
|
+
const thisBlockHashes = await Promise.all(thisBlocks.map(block => block.hash()));
|
|
313
388
|
if (
|
|
314
|
-
|
|
389
|
+
blockHeaders.length !== thisBlocks.length ||
|
|
315
390
|
!blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))
|
|
316
391
|
) {
|
|
317
392
|
this.log.warn('Epoch blocks changed underfoot', {
|
|
@@ -329,30 +404,18 @@ export class EpochProvingJob implements Traceable {
|
|
|
329
404
|
this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
|
|
330
405
|
}
|
|
331
406
|
|
|
332
|
-
/* Returns the header
|
|
333
|
-
private
|
|
334
|
-
const
|
|
335
|
-
|
|
336
|
-
return block.header;
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
if (blockNumber === Number(this.data.previousBlockHeader.getBlockNumber())) {
|
|
340
|
-
return this.data.previousBlockHeader;
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
throw new Error(
|
|
344
|
-
`Block header not found for block number ${blockNumber} (got ${this.blocks
|
|
345
|
-
.map(b => b.number)
|
|
346
|
-
.join(', ')} and previous header ${this.data.previousBlockHeader.getBlockNumber()})`,
|
|
347
|
-
);
|
|
407
|
+
/* Returns the last block header in the previous checkpoint for all checkpoints in the epoch */
|
|
408
|
+
private gatherPreviousBlockHeaders() {
|
|
409
|
+
const lastBlocks = this.checkpoints.map(checkpoint => checkpoint.blocks.at(-1)!);
|
|
410
|
+
return [this.data.previousBlockHeader, ...lastBlocks.map(block => block.header).slice(0, -1)];
|
|
348
411
|
}
|
|
349
412
|
|
|
350
413
|
private getTxs(block: L2Block): Tx[] {
|
|
351
414
|
return block.body.txEffects.map(txEffect => this.txs.get(txEffect.txHash.toString())!);
|
|
352
415
|
}
|
|
353
416
|
|
|
354
|
-
private getL1ToL2Messages(
|
|
355
|
-
return this.data.l1ToL2Messages[
|
|
417
|
+
private getL1ToL2Messages(checkpoint: Checkpoint) {
|
|
418
|
+
return this.data.l1ToL2Messages[checkpoint.number];
|
|
356
419
|
}
|
|
357
420
|
|
|
358
421
|
private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {
|
package/src/metrics.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { RollupContract } from '@aztec/ethereum';
|
|
1
|
+
import type { RollupContract } from '@aztec/ethereum/contracts';
|
|
2
2
|
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
3
3
|
import { createLogger } from '@aztec/foundation/log';
|
|
4
4
|
import type { L1PublishProofStats, L1PublishStats } from '@aztec/stdlib/stats';
|
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
type TelemetryClient,
|
|
14
14
|
type Tracer,
|
|
15
15
|
type UpDownCounter,
|
|
16
|
-
|
|
16
|
+
createUpDownCounterWithDefault,
|
|
17
17
|
} from '@aztec/telemetry-client';
|
|
18
18
|
|
|
19
19
|
import { formatEther, formatUnits } from 'viem';
|
|
@@ -21,6 +21,7 @@ import { formatEther, formatUnits } from 'viem';
|
|
|
21
21
|
export class ProverNodeJobMetrics {
|
|
22
22
|
proverEpochExecutionDuration: Histogram;
|
|
23
23
|
provingJobDuration: Histogram;
|
|
24
|
+
provingJobCheckpoints: Gauge;
|
|
24
25
|
provingJobBlocks: Gauge;
|
|
25
26
|
provingJobTransactions: Gauge;
|
|
26
27
|
|
|
@@ -29,29 +30,23 @@ export class ProverNodeJobMetrics {
|
|
|
29
30
|
public readonly tracer: Tracer,
|
|
30
31
|
private logger = createLogger('prover-node:publisher:metrics'),
|
|
31
32
|
) {
|
|
32
|
-
this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
|
|
38
|
-
description: 'Duration of proving job',
|
|
39
|
-
unit: 's',
|
|
40
|
-
valueType: ValueType.DOUBLE,
|
|
41
|
-
});
|
|
42
|
-
this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
|
|
43
|
-
description: 'Number of blocks in a proven epoch',
|
|
44
|
-
valueType: ValueType.INT,
|
|
45
|
-
});
|
|
46
|
-
this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
|
|
47
|
-
description: 'Number of transactions in a proven epoch',
|
|
48
|
-
valueType: ValueType.INT,
|
|
49
|
-
});
|
|
33
|
+
this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION);
|
|
34
|
+
this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION);
|
|
35
|
+
this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
|
|
36
|
+
this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
|
|
37
|
+
this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS);
|
|
50
38
|
}
|
|
51
39
|
|
|
52
|
-
public recordProvingJob(
|
|
40
|
+
public recordProvingJob(
|
|
41
|
+
executionTimeMs: number,
|
|
42
|
+
totalTimeMs: number,
|
|
43
|
+
numCheckpoints: number,
|
|
44
|
+
numBlocks: number,
|
|
45
|
+
numTxs: number,
|
|
46
|
+
) {
|
|
53
47
|
this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
|
|
54
48
|
this.provingJobDuration.record(totalTimeMs / 1000);
|
|
49
|
+
this.provingJobCheckpoints.record(Math.floor(numCheckpoints));
|
|
55
50
|
this.provingJobBlocks.record(Math.floor(numBlocks));
|
|
56
51
|
this.provingJobTransactions.record(Math.floor(numTxs));
|
|
57
52
|
}
|
|
@@ -69,15 +64,9 @@ export class ProverNodeRewardsMetrics {
|
|
|
69
64
|
private rollup: RollupContract,
|
|
70
65
|
private logger = createLogger('prover-node:publisher:metrics'),
|
|
71
66
|
) {
|
|
72
|
-
this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH
|
|
73
|
-
valueType: ValueType.DOUBLE,
|
|
74
|
-
description: 'The rewards earned',
|
|
75
|
-
});
|
|
67
|
+
this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH);
|
|
76
68
|
|
|
77
|
-
this.accumulatedRewards = this.meter
|
|
78
|
-
valueType: ValueType.DOUBLE,
|
|
79
|
-
description: 'The rewards earned (total)',
|
|
80
|
-
});
|
|
69
|
+
this.accumulatedRewards = createUpDownCounterWithDefault(this.meter, Metrics.PROVER_NODE_REWARDS_TOTAL);
|
|
81
70
|
}
|
|
82
71
|
|
|
83
72
|
public async start() {
|
|
@@ -97,7 +86,7 @@ export class ProverNodeRewardsMetrics {
|
|
|
97
86
|
// look at the prev epoch so that we get an accurate value, after proof submission window has closed
|
|
98
87
|
// For example, if proof submission window is 1 epoch, and we are in epoch 2, we should be looking at epoch 0.
|
|
99
88
|
// Similarly, if the proof submission window is 0, and we are in epoch 1, we should be looking at epoch 0.
|
|
100
|
-
const closedEpoch = epoch - BigInt(this.proofSubmissionEpochs) - 1n;
|
|
89
|
+
const closedEpoch = BigInt(epoch) - BigInt(this.proofSubmissionEpochs) - 1n;
|
|
101
90
|
const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase);
|
|
102
91
|
|
|
103
92
|
const fmt = parseFloat(formatUnits(rewards, 18));
|
|
@@ -138,68 +127,28 @@ export class ProverNodePublisherMetrics {
|
|
|
138
127
|
) {
|
|
139
128
|
this.meter = client.getMeter(name);
|
|
140
129
|
|
|
141
|
-
this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE
|
|
142
|
-
description: 'The gas price used for transactions',
|
|
143
|
-
unit: 'gwei',
|
|
144
|
-
valueType: ValueType.DOUBLE,
|
|
145
|
-
});
|
|
130
|
+
this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE);
|
|
146
131
|
|
|
147
|
-
this.txCount = this.meter
|
|
148
|
-
|
|
132
|
+
this.txCount = createUpDownCounterWithDefault(this.meter, Metrics.L1_PUBLISHER_TX_COUNT, {
|
|
133
|
+
[Attributes.L1_TX_TYPE]: ['submitProof'],
|
|
134
|
+
[Attributes.OK]: [true, false],
|
|
149
135
|
});
|
|
150
136
|
|
|
151
|
-
this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION
|
|
152
|
-
description: 'The duration of transaction processing',
|
|
153
|
-
unit: 'ms',
|
|
154
|
-
valueType: ValueType.INT,
|
|
155
|
-
});
|
|
137
|
+
this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION);
|
|
156
138
|
|
|
157
|
-
this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS
|
|
158
|
-
description: 'The gas consumed by transactions',
|
|
159
|
-
unit: 'gas',
|
|
160
|
-
valueType: ValueType.INT,
|
|
161
|
-
});
|
|
139
|
+
this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS);
|
|
162
140
|
|
|
163
|
-
this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE
|
|
164
|
-
description: 'The size of the calldata in transactions',
|
|
165
|
-
unit: 'By',
|
|
166
|
-
valueType: ValueType.INT,
|
|
167
|
-
});
|
|
141
|
+
this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE);
|
|
168
142
|
|
|
169
|
-
this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS
|
|
170
|
-
description: 'The gas consumed by the calldata in transactions',
|
|
171
|
-
unit: 'gas',
|
|
172
|
-
valueType: ValueType.INT,
|
|
173
|
-
});
|
|
143
|
+
this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS);
|
|
174
144
|
|
|
175
|
-
this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED
|
|
176
|
-
description: 'The amount of blob gas used in transactions',
|
|
177
|
-
unit: 'gas',
|
|
178
|
-
valueType: ValueType.INT,
|
|
179
|
-
});
|
|
145
|
+
this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED);
|
|
180
146
|
|
|
181
|
-
this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST
|
|
182
|
-
description: 'The gas cost of blobs in transactions',
|
|
183
|
-
unit: 'gwei',
|
|
184
|
-
valueType: ValueType.INT,
|
|
185
|
-
});
|
|
147
|
+
this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST);
|
|
186
148
|
|
|
187
|
-
this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE
|
|
188
|
-
description: 'How much L1 tx costs',
|
|
189
|
-
unit: 'gwei',
|
|
190
|
-
valueType: ValueType.DOUBLE,
|
|
191
|
-
advice: {
|
|
192
|
-
explicitBucketBoundaries: [
|
|
193
|
-
0.001, 0.002, 0.004, 0.008, 0.01, 0.02, 0.04, 0.08, 0.1, 0.2, 0.4, 0.8, 1, 1.2, 1.4, 1.8, 2,
|
|
194
|
-
],
|
|
195
|
-
},
|
|
196
|
-
});
|
|
149
|
+
this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE);
|
|
197
150
|
|
|
198
|
-
this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE
|
|
199
|
-
unit: 'eth',
|
|
200
|
-
description: 'The balance of the sender address',
|
|
201
|
-
valueType: ValueType.DOUBLE,
|
|
202
|
-
});
|
|
151
|
+
this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE);
|
|
203
152
|
}
|
|
204
153
|
|
|
205
154
|
recordFailedTx() {
|