@aztec/prover-node 0.0.1-commit.001888fc → 0.0.1-commit.017a351
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/actions/download-epoch-proving-job.js +1 -1
- package/dest/actions/rerun-epoch-proving-job.d.ts +3 -2
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +105 -25
- package/dest/bin/run-failed-epoch.js +1 -3
- package/dest/factory.d.ts +1 -1
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +21 -2
- package/dest/job/epoch-proving-job.d.ts +6 -2
- package/dest/job/epoch-proving-job.d.ts.map +1 -1
- package/dest/job/epoch-proving-job.js +186 -36
- package/dest/metrics.d.ts +11 -1
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +22 -0
- package/dest/monitors/epoch-monitor.js +6 -2
- package/dest/prover-node-publisher.d.ts +20 -1
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +200 -5
- package/dest/prover-node.d.ts +6 -6
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +77 -32
- package/dest/prover-publisher-factory.d.ts +2 -2
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +3 -3
- package/package.json +23 -23
- package/src/actions/download-epoch-proving-job.ts +1 -1
- package/src/actions/rerun-epoch-proving-job.ts +16 -5
- package/src/bin/run-failed-epoch.ts +1 -2
- package/src/factory.ts +18 -2
- package/src/job/epoch-proving-job.ts +102 -36
- package/src/metrics.ts +37 -0
- package/src/monitors/epoch-monitor.ts +2 -2
- package/src/prover-node-publisher.ts +232 -9
- package/src/prover-node.ts +81 -44
- package/src/prover-publisher-factory.ts +3 -3
package/src/metrics.ts
CHANGED
|
@@ -140,6 +140,13 @@ export class ProverNodeRewardsMetrics {
|
|
|
140
140
|
};
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
+
export type EstimatedSubmitProofStats = {
|
|
144
|
+
gasLimit: bigint;
|
|
145
|
+
baseFeePerGas: bigint;
|
|
146
|
+
maxPriorityFeePerGas: bigint;
|
|
147
|
+
estimatedTotalFee: bigint;
|
|
148
|
+
};
|
|
149
|
+
|
|
143
150
|
export class ProverNodePublisherMetrics {
|
|
144
151
|
gasPrice: Histogram;
|
|
145
152
|
txCount: UpDownCounter;
|
|
@@ -151,6 +158,10 @@ export class ProverNodePublisherMetrics {
|
|
|
151
158
|
txBlobDataGasCost: Histogram;
|
|
152
159
|
txTotalFee: Histogram;
|
|
153
160
|
|
|
161
|
+
private txGasEstimated: Histogram;
|
|
162
|
+
private gasPriceEstimated: Histogram;
|
|
163
|
+
private txTotalFeeEstimated: Histogram;
|
|
164
|
+
|
|
154
165
|
private senderBalance: Gauge;
|
|
155
166
|
private meter: Meter;
|
|
156
167
|
|
|
@@ -182,6 +193,12 @@ export class ProverNodePublisherMetrics {
|
|
|
182
193
|
|
|
183
194
|
this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE);
|
|
184
195
|
|
|
196
|
+
this.txGasEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS);
|
|
197
|
+
|
|
198
|
+
this.gasPriceEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE);
|
|
199
|
+
|
|
200
|
+
this.txTotalFeeEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE);
|
|
201
|
+
|
|
185
202
|
this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE);
|
|
186
203
|
}
|
|
187
204
|
|
|
@@ -196,6 +213,26 @@ export class ProverNodePublisherMetrics {
|
|
|
196
213
|
this.recordTx(durationMs, stats);
|
|
197
214
|
}
|
|
198
215
|
|
|
216
|
+
public recordEstimatedSubmitProof(stats: EstimatedSubmitProofStats) {
|
|
217
|
+
const attributes = { [Attributes.L1_TX_TYPE]: 'submitProof' } as const;
|
|
218
|
+
|
|
219
|
+
this.txGasEstimated.record(Number(stats.gasLimit), attributes);
|
|
220
|
+
|
|
221
|
+
try {
|
|
222
|
+
this.gasPriceEstimated.record(
|
|
223
|
+
parseInt(formatEther(stats.baseFeePerGas + stats.maxPriorityFeePerGas, 'gwei'), 10),
|
|
224
|
+
);
|
|
225
|
+
} catch {
|
|
226
|
+
// ignore
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
try {
|
|
230
|
+
this.txTotalFeeEstimated.record(parseFloat(formatEther(stats.estimatedTotalFee)));
|
|
231
|
+
} catch {
|
|
232
|
+
// ignore
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
199
236
|
public recordSenderBalance(wei: bigint, senderAddress: string) {
|
|
200
237
|
const eth = parseFloat(formatEther(wei, 'wei'));
|
|
201
238
|
this.senderBalance.record(eth, {
|
|
@@ -96,9 +96,9 @@ export class EpochMonitor implements Traceable {
|
|
|
96
96
|
}
|
|
97
97
|
|
|
98
98
|
private async getEpochNumberToProve() {
|
|
99
|
-
const lastBlockProven = await this.l2BlockSource.
|
|
99
|
+
const lastBlockProven = (await this.l2BlockSource.getBlockNumber({ tag: 'proven' })) ?? BlockNumber.ZERO;
|
|
100
100
|
const firstBlockToProve = BlockNumber(lastBlockProven + 1);
|
|
101
|
-
const firstBlockHeaderToProve = await this.l2BlockSource.
|
|
101
|
+
const firstBlockHeaderToProve = (await this.l2BlockSource.getBlockData({ number: firstBlockToProve }))?.header;
|
|
102
102
|
if (!firstBlockHeaderToProve) {
|
|
103
103
|
return { epochToProve: undefined, blockNumber: firstBlockToProve };
|
|
104
104
|
}
|
|
@@ -8,20 +8,22 @@ import { areArraysEqual } from '@aztec/foundation/collection';
|
|
|
8
8
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
9
9
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
10
10
|
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
11
|
+
import { retryUntil } from '@aztec/foundation/retry';
|
|
11
12
|
import type { Tuple } from '@aztec/foundation/serialize';
|
|
12
13
|
import { Timer } from '@aztec/foundation/timer';
|
|
13
14
|
import { RollupAbi } from '@aztec/l1-artifacts';
|
|
14
15
|
import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
|
|
15
16
|
import { CommitteeAttestation, CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
|
|
17
|
+
import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
16
18
|
import type { Proof } from '@aztec/stdlib/proofs';
|
|
17
19
|
import type { FeeRecipient, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
|
|
18
20
|
import type { L1PublishProofStats } from '@aztec/stdlib/stats';
|
|
19
21
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
20
22
|
|
|
21
23
|
import { inspect } from 'util';
|
|
22
|
-
import { type Hex, type TransactionReceipt, encodeFunctionData } from 'viem';
|
|
24
|
+
import { type Hex, type TransactionReceipt, encodeFunctionData, formatEther, formatGwei } from 'viem';
|
|
23
25
|
|
|
24
|
-
import { ProverNodePublisherMetrics } from './metrics.js';
|
|
26
|
+
import { type EstimatedSubmitProofStats, ProverNodePublisherMetrics } from './metrics.js';
|
|
25
27
|
|
|
26
28
|
/** Arguments to the submitEpochProof method of the rollup contract */
|
|
27
29
|
export type L1SubmitEpochProofArgs = {
|
|
@@ -101,6 +103,11 @@ export class ProverNodePublisher {
|
|
|
101
103
|
const ctx = { epochNumber, fromCheckpoint, toCheckpoint };
|
|
102
104
|
|
|
103
105
|
if (!this.interrupted) {
|
|
106
|
+
if (!(await this.waitUntilStartBuildsOnProven(args))) {
|
|
107
|
+
this.log.verbose('Checkpoint data syncing interrupted', ctx);
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
|
|
104
111
|
const timer = new Timer();
|
|
105
112
|
// Validate epoch proof range and hashes are correct before submitting
|
|
106
113
|
await this.validateEpochProofSubmission(args);
|
|
@@ -147,6 +154,53 @@ export class ProverNodePublisher {
|
|
|
147
154
|
return false;
|
|
148
155
|
}
|
|
149
156
|
|
|
157
|
+
private async waitUntilStartBuildsOnProven(args: { epochNumber: EpochNumber; fromCheckpoint: CheckpointNumber }) {
|
|
158
|
+
const { epochNumber, fromCheckpoint } = args;
|
|
159
|
+
const provenCheckpoint = await this.getProvenCheckpoint();
|
|
160
|
+
if (this.isStartBuildingOnProven(fromCheckpoint, provenCheckpoint)) {
|
|
161
|
+
return true;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const timeout = await this.getSecondsUntilProofSubmissionWindowEnd(epochNumber);
|
|
165
|
+
this.log.info(`Waiting for proven checkpoint to reach proof start`, {
|
|
166
|
+
epochNumber,
|
|
167
|
+
fromCheckpoint,
|
|
168
|
+
provenCheckpoint,
|
|
169
|
+
timeout,
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
await retryUntil(
|
|
173
|
+
async () => {
|
|
174
|
+
if (this.interrupted) {
|
|
175
|
+
return true;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const proven = await this.getProvenCheckpoint();
|
|
179
|
+
this.log.verbose(`Proven checkpoint is at ${proven} (waiting for ${fromCheckpoint - 1})`, { epochNumber });
|
|
180
|
+
return this.isStartBuildingOnProven(fromCheckpoint, proven) ? true : undefined;
|
|
181
|
+
},
|
|
182
|
+
`proven checkpoint to reach ${fromCheckpoint - 1}`,
|
|
183
|
+
timeout,
|
|
184
|
+
4,
|
|
185
|
+
);
|
|
186
|
+
|
|
187
|
+
return !this.interrupted;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private async getProvenCheckpoint() {
|
|
191
|
+
return (await this.rollupContract.getTips()).proven;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
private isStartBuildingOnProven(fromCheckpoint: CheckpointNumber, provenCheckpoint: CheckpointNumber) {
|
|
195
|
+
return fromCheckpoint - 1 <= provenCheckpoint;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
private async getSecondsUntilProofSubmissionWindowEnd(epochNumber: EpochNumber) {
|
|
199
|
+
const deadline = getProofSubmissionDeadlineTimestamp(epochNumber, await this.rollupContract.getRollupConstants());
|
|
200
|
+
const now = BigInt(Math.floor(Date.now() / 1000));
|
|
201
|
+
return Math.max(Number(deadline - now), 0.001);
|
|
202
|
+
}
|
|
203
|
+
|
|
150
204
|
private async validateEpochProofSubmission(args: {
|
|
151
205
|
fromCheckpoint: CheckpointNumber;
|
|
152
206
|
toCheckpoint: CheckpointNumber;
|
|
@@ -168,7 +222,7 @@ export class ProverNodePublisher {
|
|
|
168
222
|
// toCheckpoint can't be greater than pending
|
|
169
223
|
if (toCheckpoint > pending) {
|
|
170
224
|
throw new Error(
|
|
171
|
-
`Cannot submit epoch proof for ${fromCheckpoint}-${toCheckpoint} as
|
|
225
|
+
`Cannot submit epoch proof for ${fromCheckpoint}-${toCheckpoint} as proposed checkpoint is ${pending}`,
|
|
172
226
|
);
|
|
173
227
|
}
|
|
174
228
|
|
|
@@ -203,13 +257,85 @@ export class ProverNodePublisher {
|
|
|
203
257
|
const argsPublicInputs = [...publicInputs.toFields()];
|
|
204
258
|
|
|
205
259
|
if (!areArraysEqual(rollupPublicInputs, argsPublicInputs, (a, b) => a.equals(b))) {
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
260
|
+
throw await reportPublicInputsMismatch({
|
|
261
|
+
rollupPublicInputs,
|
|
262
|
+
argsPublicInputs,
|
|
263
|
+
fromCheckpoint,
|
|
264
|
+
toCheckpoint,
|
|
265
|
+
rollupContract: this.rollupContract,
|
|
266
|
+
log: this.log,
|
|
267
|
+
});
|
|
210
268
|
}
|
|
211
269
|
}
|
|
212
270
|
|
|
271
|
+
/**
|
|
272
|
+
* Estimates what submitting the epoch proof would have cost on L1 without actually sending it.
|
|
273
|
+
* Runs the same validation as `submitEpochProof`, encodes the calldata, estimates gas, and records metrics.
|
|
274
|
+
* Used when proof publishing is disabled (e.g. PROVER_NODE_DISABLE_PROOF_PUBLISH=true on mainnet).
|
|
275
|
+
*/
|
|
276
|
+
public async analyzeEpochProofSubmission(args: {
|
|
277
|
+
epochNumber: EpochNumber;
|
|
278
|
+
fromCheckpoint: CheckpointNumber;
|
|
279
|
+
toCheckpoint: CheckpointNumber;
|
|
280
|
+
publicInputs: RootRollupPublicInputs;
|
|
281
|
+
proof: Proof;
|
|
282
|
+
batchedBlobInputs: BatchedBlob;
|
|
283
|
+
attestations: ViemCommitteeAttestation[];
|
|
284
|
+
}): Promise<void> {
|
|
285
|
+
const { epochNumber, fromCheckpoint, toCheckpoint } = args;
|
|
286
|
+
|
|
287
|
+
await this.validateEpochProofSubmission(args);
|
|
288
|
+
|
|
289
|
+
const data = this.encodeSubmitEpochProofCalldata(args);
|
|
290
|
+
const senderAddress = this.l1TxUtils.getSenderAddress();
|
|
291
|
+
|
|
292
|
+
const [gasLimit, gasPrice, latestBlock] = await Promise.all([
|
|
293
|
+
this.l1TxUtils.estimateGas(senderAddress.toString() as `0x${string}`, { to: this.rollupContract.address, data }),
|
|
294
|
+
this.l1TxUtils.getGasPrice(),
|
|
295
|
+
this.l1TxUtils.client.getBlock({ blockTag: 'latest' }),
|
|
296
|
+
]);
|
|
297
|
+
|
|
298
|
+
const baseFeePerGas = latestBlock.baseFeePerGas ?? 0n;
|
|
299
|
+
const { maxPriorityFeePerGas } = gasPrice;
|
|
300
|
+
|
|
301
|
+
const effectiveFeePerGas = baseFeePerGas + maxPriorityFeePerGas;
|
|
302
|
+
const estimatedTotalFee = gasLimit * effectiveFeePerGas;
|
|
303
|
+
|
|
304
|
+
const stats: EstimatedSubmitProofStats = {
|
|
305
|
+
gasLimit,
|
|
306
|
+
baseFeePerGas,
|
|
307
|
+
maxPriorityFeePerGas,
|
|
308
|
+
estimatedTotalFee,
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
this.log.info(`Estimated epoch proof submission cost (not submitted)`, {
|
|
312
|
+
epochNumber,
|
|
313
|
+
fromCheckpoint,
|
|
314
|
+
toCheckpoint,
|
|
315
|
+
gasLimit: gasLimit.toString(),
|
|
316
|
+
baseFeePerGas: formatGwei(baseFeePerGas),
|
|
317
|
+
maxPriorityFeePerGas: formatGwei(maxPriorityFeePerGas),
|
|
318
|
+
estimatedTotalFeeEth: formatEther(estimatedTotalFee),
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
this.metrics.recordEstimatedSubmitProof(stats);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
private encodeSubmitEpochProofCalldata(args: {
|
|
325
|
+
fromCheckpoint: CheckpointNumber;
|
|
326
|
+
toCheckpoint: CheckpointNumber;
|
|
327
|
+
publicInputs: RootRollupPublicInputs;
|
|
328
|
+
proof: Proof;
|
|
329
|
+
batchedBlobInputs: BatchedBlob;
|
|
330
|
+
attestations: ViemCommitteeAttestation[];
|
|
331
|
+
}): Hex {
|
|
332
|
+
return encodeFunctionData({
|
|
333
|
+
abi: RollupAbi,
|
|
334
|
+
functionName: 'submitEpochRootProof',
|
|
335
|
+
args: [this.getSubmitEpochProofArgs(args)],
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
|
|
213
339
|
private async sendSubmitEpochProofTx(args: {
|
|
214
340
|
fromCheckpoint: CheckpointNumber;
|
|
215
341
|
toCheckpoint: CheckpointNumber;
|
|
@@ -296,11 +422,108 @@ export class ProverNodePublisher {
|
|
|
296
422
|
end: argsArray[1],
|
|
297
423
|
args: argsArray[2],
|
|
298
424
|
fees: argsArray[3],
|
|
299
|
-
attestations:
|
|
425
|
+
attestations: CommitteeAttestationsAndSigners.packAttestations(
|
|
300
426
|
args.attestations.map(a => CommitteeAttestation.fromViem(a)),
|
|
301
|
-
)
|
|
427
|
+
),
|
|
302
428
|
blobInputs: argsArray[4],
|
|
303
429
|
proof: proofHex,
|
|
304
430
|
};
|
|
305
431
|
}
|
|
306
432
|
}
|
|
433
|
+
|
|
434
|
+
/**
|
|
435
|
+
* Decodes a `Root rollup public inputs mismatch`, fetches the on-chain CheckpointLog for any
|
|
436
|
+
* mismatching `checkpointHeaderHashes[i]`, emits a structured error log, and returns a thrown-ready
|
|
437
|
+
* Error with a human-readable summary.
|
|
438
|
+
*
|
|
439
|
+
* Layout of `RootRollupPublicInputs.toFields()`:
|
|
440
|
+
* [0] previousArchiveRoot
|
|
441
|
+
* [1] endArchiveRoot
|
|
442
|
+
* [2] outHash
|
|
443
|
+
* [3 .. 3+N-1] checkpointHeaderHashes[i] for i in 0..N-1 (N = MAX_CHECKPOINTS_PER_EPOCH)
|
|
444
|
+
* [3+N .. 3+3N-1] fees[i] = (recipient, value) for i in 0..N-1
|
|
445
|
+
* [3+3N .. 3+3N+4] EpochConstantData (chainId, version, vkTreeRoot, protocolContractsHash, proverId)
|
|
446
|
+
* [3+3N+5 ..] blobPublicInputs (FinalBlobAccumulator)
|
|
447
|
+
*/
|
|
448
|
+
async function reportPublicInputsMismatch(input: {
|
|
449
|
+
rollupPublicInputs: readonly Fr[];
|
|
450
|
+
argsPublicInputs: readonly Fr[];
|
|
451
|
+
fromCheckpoint: CheckpointNumber;
|
|
452
|
+
toCheckpoint: CheckpointNumber;
|
|
453
|
+
rollupContract: RollupContract;
|
|
454
|
+
log: Logger;
|
|
455
|
+
}): Promise<Error> {
|
|
456
|
+
const { rollupPublicInputs, argsPublicInputs, fromCheckpoint, toCheckpoint, rollupContract, log } = input;
|
|
457
|
+
const N = MAX_CHECKPOINTS_PER_EPOCH;
|
|
458
|
+
const constantsStart = 3 + 3 * N;
|
|
459
|
+
const blobStart = constantsStart + 5;
|
|
460
|
+
const constantLabels = ['chainId', 'version', 'vkTreeRoot', 'protocolContractsHash', 'proverId'];
|
|
461
|
+
|
|
462
|
+
const diffs: { index: number; label: string; rollup: Fr; computed: Fr; checkpointIndex?: number }[] = [];
|
|
463
|
+
const len = Math.max(rollupPublicInputs.length, argsPublicInputs.length);
|
|
464
|
+
for (let i = 0; i < len; i++) {
|
|
465
|
+
const a = rollupPublicInputs[i] ?? Fr.ZERO;
|
|
466
|
+
const b = argsPublicInputs[i] ?? Fr.ZERO;
|
|
467
|
+
if (a.equals(b)) {
|
|
468
|
+
continue;
|
|
469
|
+
}
|
|
470
|
+
let label: string;
|
|
471
|
+
let checkpointIndex: number | undefined;
|
|
472
|
+
if (i === 0) {
|
|
473
|
+
label = 'previousArchiveRoot';
|
|
474
|
+
} else if (i === 1) {
|
|
475
|
+
label = 'endArchiveRoot';
|
|
476
|
+
} else if (i === 2) {
|
|
477
|
+
label = 'outHash';
|
|
478
|
+
} else if (i < 3 + N) {
|
|
479
|
+
checkpointIndex = i - 3;
|
|
480
|
+
label = `checkpointHeaderHashes[${checkpointIndex}]`;
|
|
481
|
+
} else if (i < 3 + 3 * N) {
|
|
482
|
+
const feePairIndex = i - (3 + N);
|
|
483
|
+
const feeIndex = Math.floor(feePairIndex / 2);
|
|
484
|
+
const sub = feePairIndex % 2 === 0 ? 'recipient' : 'value';
|
|
485
|
+
label = `fees[${feeIndex}].${sub}`;
|
|
486
|
+
} else if (i < blobStart) {
|
|
487
|
+
label = `constants.${constantLabels[i - constantsStart]}`;
|
|
488
|
+
} else {
|
|
489
|
+
label = `blobPublicInputs[${i - blobStart}]`;
|
|
490
|
+
}
|
|
491
|
+
diffs.push({ index: i, label, rollup: a, computed: b, checkpointIndex });
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
// For each mismatching checkpointHeaderHash, fetch the L1 CheckpointLog so the operator can
|
|
495
|
+
// see what was published on-chain alongside the prover's recomputed hash.
|
|
496
|
+
const onChainCheckpoints = await Promise.all(
|
|
497
|
+
diffs
|
|
498
|
+
.filter(d => d.checkpointIndex !== undefined)
|
|
499
|
+
.map(async d => {
|
|
500
|
+
const checkpointNumber = CheckpointNumber(fromCheckpoint + d.checkpointIndex!);
|
|
501
|
+
try {
|
|
502
|
+
const cp = await rollupContract.getCheckpoint(checkpointNumber);
|
|
503
|
+
return { checkpointIndex: d.checkpointIndex!, checkpointNumber, headerHash: cp.headerHash.toString() };
|
|
504
|
+
} catch (err) {
|
|
505
|
+
return { checkpointIndex: d.checkpointIndex!, checkpointNumber, error: (err as Error).message };
|
|
506
|
+
}
|
|
507
|
+
}),
|
|
508
|
+
);
|
|
509
|
+
|
|
510
|
+
log.error(`Root rollup public inputs mismatch`, undefined, {
|
|
511
|
+
fromCheckpoint,
|
|
512
|
+
toCheckpoint,
|
|
513
|
+
numDiffs: diffs.length,
|
|
514
|
+
diffs: diffs.map(d => ({
|
|
515
|
+
index: d.index,
|
|
516
|
+
label: d.label,
|
|
517
|
+
rollup: d.rollup.toString(),
|
|
518
|
+
computed: d.computed.toString(),
|
|
519
|
+
})),
|
|
520
|
+
onChainCheckpoints,
|
|
521
|
+
});
|
|
522
|
+
|
|
523
|
+
const fmt = (inputs: readonly Fr[]) => inputs.map(x => x.toString()).join(', ');
|
|
524
|
+
const summary = diffs.map(d => `[${d.index} ${d.label}] L1=${d.rollup} prover=${d.computed}`).join('\n');
|
|
525
|
+
return new Error(
|
|
526
|
+
`Root rollup public inputs mismatch (${diffs.length} fields differ):\n${summary}\n` +
|
|
527
|
+
`Rollup: ${fmt(rollupPublicInputs)}\nComputed:${fmt(argsPublicInputs)}`,
|
|
528
|
+
);
|
|
529
|
+
}
|
package/src/prover-node.ts
CHANGED
|
@@ -15,7 +15,6 @@ import type { ContractDataSource } from '@aztec/stdlib/contract';
|
|
|
15
15
|
import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
16
16
|
import {
|
|
17
17
|
type EpochProverManager,
|
|
18
|
-
EpochProvingJobTerminalState,
|
|
19
18
|
type ITxProvider,
|
|
20
19
|
type ProverNodeApi,
|
|
21
20
|
type Service,
|
|
@@ -39,7 +38,7 @@ import {
|
|
|
39
38
|
import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
|
|
40
39
|
import type { SpecificProverNodeConfig } from './config.js';
|
|
41
40
|
import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
|
|
42
|
-
import { EpochProvingJob, type EpochProvingJobState } from './job/epoch-proving-job.js';
|
|
41
|
+
import { EpochProvingJob, type EpochProvingJobOptions, type EpochProvingJobState } from './job/epoch-proving-job.js';
|
|
43
42
|
import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
|
|
44
43
|
import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js';
|
|
45
44
|
import type { ProverNodePublisher } from './prover-node-publisher.js';
|
|
@@ -60,6 +59,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
60
59
|
private config: ProverNodeOptions;
|
|
61
60
|
private jobMetrics: ProverNodeJobMetrics;
|
|
62
61
|
private rewardsMetrics: ProverNodeRewardsMetrics;
|
|
62
|
+
private startingProofEpochs: Set<EpochNumber> = new Set();
|
|
63
63
|
|
|
64
64
|
public readonly tracer: Tracer;
|
|
65
65
|
|
|
@@ -127,15 +127,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
127
127
|
this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
|
|
128
128
|
jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
|
|
129
129
|
});
|
|
130
|
-
|
|
131
|
-
if (activeJobs.length > 0) {
|
|
132
|
-
this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
|
|
133
|
-
activeJobs: activeJobs.map(job => job.uuid),
|
|
134
|
-
});
|
|
135
|
-
return true;
|
|
136
|
-
}
|
|
137
|
-
await this.startProof(epochNumber);
|
|
138
|
-
return true;
|
|
130
|
+
return await this.startProofIfNeeded(epochNumber);
|
|
139
131
|
} catch (err) {
|
|
140
132
|
if (err instanceof EmptyEpochError) {
|
|
141
133
|
this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
|
|
@@ -166,10 +158,10 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
166
158
|
async stop() {
|
|
167
159
|
this.log.info('Stopping ProverNode');
|
|
168
160
|
await this.epochsMonitor.stop();
|
|
169
|
-
await this.prover.stop();
|
|
170
|
-
await tryStop(this.publisherFactory);
|
|
171
161
|
this.publisher?.interrupt();
|
|
172
162
|
await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
|
|
163
|
+
await this.prover.stop();
|
|
164
|
+
await tryStop(this.publisherFactory);
|
|
173
165
|
this.rewardsMetrics.stop();
|
|
174
166
|
this.l1Metrics.stop();
|
|
175
167
|
await this.telemetryClient.stop();
|
|
@@ -191,7 +183,38 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
191
183
|
* Starts a proving process and returns immediately.
|
|
192
184
|
*/
|
|
193
185
|
public async startProof(epochNumber: EpochNumber) {
|
|
194
|
-
|
|
186
|
+
await this.startProofIfNeeded(epochNumber, { skipEpochCheck: true });
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
private async startProofIfNeeded(epochNumber: EpochNumber, opts: EpochProvingJobOptions = {}): Promise<boolean> {
|
|
190
|
+
if (this.startingProofEpochs.has(epochNumber)) {
|
|
191
|
+
this.log.warn(`Not starting proof for ${epochNumber} since a proof is already being started for the epoch`, {
|
|
192
|
+
epochNumber,
|
|
193
|
+
});
|
|
194
|
+
return false;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
this.startingProofEpochs.add(epochNumber);
|
|
198
|
+
|
|
199
|
+
try {
|
|
200
|
+
const activeJobs = await this.activeJobsCoverEpoch(epochNumber);
|
|
201
|
+
if (activeJobs.length > 0) {
|
|
202
|
+
this.log.warn(`Not starting proof for ${epochNumber} since an active job already covers the epoch`, {
|
|
203
|
+
epochNumber,
|
|
204
|
+
activeJobs,
|
|
205
|
+
});
|
|
206
|
+
return true;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
await this.startProofInternal(epochNumber, opts);
|
|
210
|
+
return true;
|
|
211
|
+
} finally {
|
|
212
|
+
this.startingProofEpochs.delete(epochNumber);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
private async startProofInternal(epochNumber: EpochNumber, opts: EpochProvingJobOptions = {}) {
|
|
217
|
+
const job = await this.createProvingJob(epochNumber, opts);
|
|
195
218
|
void this.runJob(job);
|
|
196
219
|
}
|
|
197
220
|
|
|
@@ -245,20 +268,42 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
245
268
|
* Returns an array of jobs being processed.
|
|
246
269
|
*/
|
|
247
270
|
public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
|
|
248
|
-
return Promise.resolve(
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
271
|
+
return Promise.resolve(this.getJobsInternal());
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
private getJobsInternal(): { uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[] {
|
|
275
|
+
return Array.from(this.jobs.entries()).map(([uuid, job]) => ({
|
|
276
|
+
uuid,
|
|
277
|
+
status: job.getState(),
|
|
278
|
+
epochNumber: job.getEpochNumber(),
|
|
279
|
+
}));
|
|
255
280
|
}
|
|
256
281
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
282
|
+
private async activeJobsCoverEpoch(epochNumber: EpochNumber): Promise<string[]> {
|
|
283
|
+
const checkpoints = await this.l2BlockSource.getCheckpointsData({ epoch: epochNumber });
|
|
284
|
+
if (checkpoints.length === 0) {
|
|
285
|
+
return [];
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
const firstCheckpoint = checkpoints.at(0)!.checkpointNumber;
|
|
289
|
+
const latestCheckpoint = checkpoints.at(-1)!.checkpointNumber;
|
|
290
|
+
|
|
291
|
+
const jobs: string[] = [];
|
|
292
|
+
for (const job of this.jobs.values()) {
|
|
293
|
+
if (job.getEpochNumber() !== epochNumber) {
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const jobCheckpoints = job.getProvingData().checkpoints;
|
|
298
|
+
const checkpointOverlap =
|
|
299
|
+
jobCheckpoints.at(0)!.number <= firstCheckpoint && jobCheckpoints.at(-1)!.number >= latestCheckpoint;
|
|
300
|
+
|
|
301
|
+
if (checkpointOverlap && !['failed', 'stopped', 'timed-out'].includes(job.getState())) {
|
|
302
|
+
jobs.push(job.getId());
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
return jobs;
|
|
262
307
|
}
|
|
263
308
|
|
|
264
309
|
private checkMaximumPendingJobs() {
|
|
@@ -312,26 +357,21 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
312
357
|
|
|
313
358
|
@trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
|
|
314
359
|
private async gatherEpochData(epochNumber: EpochNumber): Promise<EpochProvingJobData> {
|
|
315
|
-
const
|
|
360
|
+
const publishedCheckpoints = await this.l2BlockSource.getCheckpoints({ epoch: epochNumber });
|
|
361
|
+
if (publishedCheckpoints.length === 0) {
|
|
362
|
+
throw new EmptyEpochError(epochNumber);
|
|
363
|
+
}
|
|
364
|
+
const checkpoints = publishedCheckpoints.map(p => p.checkpoint);
|
|
365
|
+
const attestations = publishedCheckpoints.at(-1)?.attestations ?? [];
|
|
316
366
|
const txArray = await this.gatherTxs(epochNumber, checkpoints);
|
|
317
367
|
const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
|
|
318
368
|
const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
|
|
319
369
|
const [firstBlock] = checkpoints[0].blocks;
|
|
320
370
|
const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
|
|
321
|
-
const [lastPublishedCheckpoint] = await this.l2BlockSource.getCheckpoints(checkpoints.at(-1)!.number, 1);
|
|
322
|
-
const attestations = lastPublishedCheckpoint?.attestations ?? [];
|
|
323
371
|
|
|
324
372
|
return { checkpoints, txs, l1ToL2Messages, epochNumber, previousBlockHeader, attestations };
|
|
325
373
|
}
|
|
326
374
|
|
|
327
|
-
private async gatherCheckpoints(epochNumber: EpochNumber) {
|
|
328
|
-
const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
|
|
329
|
-
if (checkpoints.length === 0) {
|
|
330
|
-
throw new EmptyEpochError(epochNumber);
|
|
331
|
-
}
|
|
332
|
-
return checkpoints;
|
|
333
|
-
}
|
|
334
|
-
|
|
335
375
|
private async gatherTxs(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
|
|
336
376
|
const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
|
|
337
377
|
const txProvider = this.p2pClient.getTxProvider();
|
|
@@ -360,16 +400,13 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
360
400
|
}
|
|
361
401
|
|
|
362
402
|
private async gatherPreviousBlockHeader(epochNumber: EpochNumber, previousBlockNumber: number) {
|
|
363
|
-
const
|
|
364
|
-
|
|
365
|
-
: this.l2BlockSource.getBlockHeader(BlockNumber(previousBlockNumber)));
|
|
366
|
-
|
|
367
|
-
if (!header) {
|
|
403
|
+
const data = await this.l2BlockSource.getBlockData({ number: BlockNumber(previousBlockNumber) });
|
|
404
|
+
if (!data?.header) {
|
|
368
405
|
throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
|
|
369
406
|
}
|
|
370
407
|
|
|
371
|
-
this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
|
|
372
|
-
return header;
|
|
408
|
+
this.log.verbose(`Gathered previous block header ${data.header.getBlockNumber()} for epoch ${epochNumber}`);
|
|
409
|
+
return data.header;
|
|
373
410
|
}
|
|
374
411
|
|
|
375
412
|
/** Extracted for testing purposes. */
|
|
@@ -19,11 +19,11 @@ export class ProverPublisherFactory {
|
|
|
19
19
|
) {}
|
|
20
20
|
|
|
21
21
|
public async start() {
|
|
22
|
-
await this.deps.publisherManager.
|
|
22
|
+
await this.deps.publisherManager.start();
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
public stop() {
|
|
26
|
-
this.deps.publisherManager.
|
|
25
|
+
public async stop() {
|
|
26
|
+
await this.deps.publisherManager.stop();
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
/**
|