@aztec-labs/prover-node 6.0.0-rc.3 → 6.0.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/actions/upload-epoch-proof-failure.d.ts +1 -1
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
- package/dest/actions/upload-epoch-proof-failure.js +1 -0
- package/dest/config.d.ts +2 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +5 -0
- package/dest/factory.d.ts +1 -1
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +9 -15
- package/dest/gas_constants.d.ts +14 -0
- package/dest/gas_constants.d.ts.map +1 -0
- package/dest/gas_constants.js +14 -0
- package/dest/job/checkpoint-prover.d.ts +1 -1
- package/dest/job/checkpoint-prover.d.ts.map +1 -1
- package/dest/job/checkpoint-prover.js +6 -0
- package/dest/metrics.d.ts +4 -3
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +5 -8
- package/dest/proof-publishing-service.d.ts +12 -2
- package/dest/proof-publishing-service.d.ts.map +1 -1
- package/dest/proof-publishing-service.js +40 -2
- package/dest/prover-node-publisher.d.ts +5 -2
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +81 -16
- package/dest/prover-node.d.ts +1 -1
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +8 -1
- package/dest/prover-publisher-factory.d.ts +6 -2
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +7 -2
- package/dest/session-manager.d.ts +6 -1
- package/dest/session-manager.d.ts.map +1 -1
- package/dest/session-manager.js +5 -3
- package/package.json +22 -23
- package/src/actions/upload-epoch-proof-failure.ts +1 -0
- package/src/config.ts +8 -0
- package/src/factory.ts +13 -22
- package/src/gas_constants.ts +19 -0
- package/src/job/checkpoint-prover.ts +8 -0
- package/src/metrics.ts +10 -10
- package/src/proof-publishing-service.ts +55 -3
- package/src/prover-node-publisher.ts +96 -21
- package/src/prover-node.ts +7 -0
- package/src/prover-publisher-factory.ts +7 -2
- package/src/session-manager.ts +10 -3
package/src/metrics.ts
CHANGED
|
@@ -32,6 +32,7 @@ export class ProverNodeJobMetrics {
|
|
|
32
32
|
private checkpointProvingDuration: Histogram;
|
|
33
33
|
private checkpointBlocks: Histogram;
|
|
34
34
|
private checkpointTransactions: Histogram;
|
|
35
|
+
private proofSubmissionUnaffordable: UpDownCounter;
|
|
35
36
|
|
|
36
37
|
/** Observable gauges for live state. Registered via `observeState(...)` once the
|
|
37
38
|
* CheckpointStore and SessionManager are available. */
|
|
@@ -56,6 +57,10 @@ export class ProverNodeJobMetrics {
|
|
|
56
57
|
this.checkpointProvingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROVING_DURATION);
|
|
57
58
|
this.checkpointBlocks = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_BLOCKS);
|
|
58
59
|
this.checkpointTransactions = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_TRANSACTIONS);
|
|
60
|
+
this.proofSubmissionUnaffordable = createUpDownCounterWithDefault(
|
|
61
|
+
this.meter,
|
|
62
|
+
Metrics.PROVER_NODE_PROOF_SUBMISSION_UNAFFORDABLE_COUNT,
|
|
63
|
+
);
|
|
59
64
|
}
|
|
60
65
|
|
|
61
66
|
public recordProvingJob(totalTimeMs: number, numCheckpoints: number, numBlocks: number, numTxs: number) {
|
|
@@ -83,6 +88,11 @@ export class ProverNodeJobMetrics {
|
|
|
83
88
|
this.checkpointProvingDuration.record(Math.ceil(durationMs));
|
|
84
89
|
}
|
|
85
90
|
|
|
91
|
+
/** Records a finished epoch proof held back from submission because no publisher could afford sending it. */
|
|
92
|
+
public recordProofSubmissionUnaffordable() {
|
|
93
|
+
this.proofSubmissionUnaffordable.add(1);
|
|
94
|
+
}
|
|
95
|
+
|
|
86
96
|
/**
|
|
87
97
|
* Registers observable gauges for the prover-node's live state: how many canonical
|
|
88
98
|
* checkpoint provers are in the store, and how many epoch sessions are live (broken
|
|
@@ -203,7 +213,6 @@ export class ProverNodePublisherMetrics {
|
|
|
203
213
|
private gasPriceEstimated: Histogram;
|
|
204
214
|
private txTotalFeeEstimated: Histogram;
|
|
205
215
|
|
|
206
|
-
private senderBalance: Gauge;
|
|
207
216
|
private meter: Meter;
|
|
208
217
|
|
|
209
218
|
constructor(
|
|
@@ -239,8 +248,6 @@ export class ProverNodePublisherMetrics {
|
|
|
239
248
|
this.gasPriceEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE);
|
|
240
249
|
|
|
241
250
|
this.txTotalFeeEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE);
|
|
242
|
-
|
|
243
|
-
this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE);
|
|
244
251
|
}
|
|
245
252
|
|
|
246
253
|
recordFailedTx() {
|
|
@@ -274,13 +281,6 @@ export class ProverNodePublisherMetrics {
|
|
|
274
281
|
}
|
|
275
282
|
}
|
|
276
283
|
|
|
277
|
-
public recordSenderBalance(wei: bigint, senderAddress: string) {
|
|
278
|
-
const eth = parseFloat(formatEther(wei, 'wei'));
|
|
279
|
-
this.senderBalance.record(eth, {
|
|
280
|
-
[Attributes.SENDER_ADDRESS]: senderAddress,
|
|
281
|
-
});
|
|
282
|
-
}
|
|
283
|
-
|
|
284
284
|
private recordTx(durationMs: number, stats: L1PublishStats) {
|
|
285
285
|
const attributes = {
|
|
286
286
|
[Attributes.L1_TX_TYPE]: 'submitProof',
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { BatchedBlob } from '@aztec-labs/blob-lib';
|
|
2
2
|
import type { ViemCommitteeAttestations } from '@aztec-labs/ethereum/contracts';
|
|
3
|
+
import { NoAffordablePublisherError } from '@aztec-labs/ethereum/publisher-manager';
|
|
3
4
|
import { BlockNumber, type CheckpointNumber, type EpochNumber } from '@aztec-labs/foundation/branded-types';
|
|
4
5
|
import { type Logger, type LoggerBindings, createLogger } from '@aztec-labs/foundation/log';
|
|
5
6
|
import { promiseWithResolvers } from '@aztec-labs/foundation/promise';
|
|
@@ -9,6 +10,7 @@ import type { L2BlockSource } from '@aztec-labs/stdlib/block';
|
|
|
9
10
|
import type { Proof } from '@aztec-labs/stdlib/proofs';
|
|
10
11
|
import type { CheckpointHeader, RootRollupPublicInputs } from '@aztec-labs/stdlib/rollup';
|
|
11
12
|
|
|
13
|
+
import type { ProverNodeJobMetrics } from './metrics.js';
|
|
12
14
|
import type { ProverNodePublisher } from './prover-node-publisher.js';
|
|
13
15
|
import type { ProverPublisherFactory } from './prover-publisher-factory.js';
|
|
14
16
|
|
|
@@ -51,7 +53,9 @@ export type PublishCandidate = {
|
|
|
51
53
|
|
|
52
54
|
/**
|
|
53
55
|
* Terminal outcome for a candidate. The promise from `submit()` resolves with one of these. `'already-submitted'`
|
|
54
|
-
* means L1 already holds a proof of this length from this prover for the epoch, so nothing was sent.
|
|
56
|
+
* means L1 already holds a proof of this length from this prover for the epoch, so nothing was sent. For a full
|
|
57
|
+
* candidate, only a full-epoch registration counts: a proof registered while the epoch was still open is non-full,
|
|
58
|
+
* so a full candidate is resent to replace it.
|
|
55
59
|
*/
|
|
56
60
|
export type PublishOutcome = 'published' | 'already-submitted' | 'superseded' | 'failed' | 'withdrawn' | 'expired';
|
|
57
61
|
|
|
@@ -72,6 +76,7 @@ export type ProofPublishingServiceDeps = {
|
|
|
72
76
|
l2BlockSource: Pick<L2BlockSource, 'getBlockNumber'>;
|
|
73
77
|
dateProvider: DateProvider;
|
|
74
78
|
config: ProofPublishingServiceConfig;
|
|
79
|
+
metrics?: Pick<ProverNodeJobMetrics, 'recordProofSubmissionUnaffordable'>;
|
|
75
80
|
bindings?: LoggerBindings;
|
|
76
81
|
};
|
|
77
82
|
|
|
@@ -89,6 +94,9 @@ type EpochBucket = {
|
|
|
89
94
|
*/
|
|
90
95
|
const PUBLISHER_ACQUIRE_RETRY_DELAY_MS = 1_000;
|
|
91
96
|
|
|
97
|
+
/** Backoff when no publisher can afford the submission: a balance only changes with a new L1 block or a top-up. */
|
|
98
|
+
const PUBLISHER_UNAFFORDABLE_RETRY_DELAY_MS = 12_000;
|
|
99
|
+
|
|
92
100
|
/**
|
|
93
101
|
* Central owner of L1 proof submission. Sessions offer their proofs here as
|
|
94
102
|
* `PublishCandidate`s; the service serialises one publish at a time, picks the
|
|
@@ -125,6 +133,10 @@ export class ProofPublishingService {
|
|
|
125
133
|
/** Tracks the candidate currently being published. Set while drain is awaiting the L1 publish. */
|
|
126
134
|
private inFlight: { id: string } | undefined;
|
|
127
135
|
private stopped = false;
|
|
136
|
+
/** Candidates for which the lack of an affordable publisher has already been logged. */
|
|
137
|
+
private readonly unaffordableCandidates = new Set<string>();
|
|
138
|
+
/** Pending drain retry after a failed publisher acquisition, so repeated failures never stack up retry timers. */
|
|
139
|
+
private retryTimer: NodeJS.Timeout | undefined;
|
|
128
140
|
|
|
129
141
|
constructor(private readonly deps: ProofPublishingServiceDeps) {
|
|
130
142
|
this.log = createLogger('prover-node:proof-publishing-service', deps.bindings);
|
|
@@ -203,6 +215,8 @@ export class ProofPublishingService {
|
|
|
203
215
|
*/
|
|
204
216
|
public async stop(): Promise<void> {
|
|
205
217
|
this.stopped = true;
|
|
218
|
+
clearTimeout(this.retryTimer);
|
|
219
|
+
this.retryTimer = undefined;
|
|
206
220
|
await this.drainQueue.end();
|
|
207
221
|
// Anything still parked in a bucket never ran through drain — resolve it as withdrawn so
|
|
208
222
|
// callers awaiting `submit()` aren't left hanging.
|
|
@@ -227,6 +241,17 @@ export class ProofPublishingService {
|
|
|
227
241
|
});
|
|
228
242
|
}
|
|
229
243
|
|
|
244
|
+
/** Schedules a drain after the delay, unless a retry is already pending. */
|
|
245
|
+
private scheduleRetry(delayMs: number): void {
|
|
246
|
+
if (this.stopped || this.retryTimer) {
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
this.retryTimer = setTimeout(() => {
|
|
250
|
+
this.retryTimer = undefined;
|
|
251
|
+
this.scheduleDrain();
|
|
252
|
+
}, delayMs);
|
|
253
|
+
}
|
|
254
|
+
|
|
230
255
|
private async drain(): Promise<void> {
|
|
231
256
|
if (this.stopped) {
|
|
232
257
|
return;
|
|
@@ -295,8 +320,34 @@ export class ProofPublishingService {
|
|
|
295
320
|
private async publishWinner(epoch: EpochNumber, winner: PublishCandidate, bucket: EpochBucket): Promise<void> {
|
|
296
321
|
let publisher: PublisherLike;
|
|
297
322
|
try {
|
|
298
|
-
publisher = await this.deps.publisherFactory.create(
|
|
323
|
+
publisher = await this.deps.publisherFactory.create({
|
|
324
|
+
requireAffordableSubmission: !this.deps.config.skipSubmitProof,
|
|
325
|
+
});
|
|
299
326
|
} catch (err) {
|
|
327
|
+
if (err instanceof NoAffordablePublisherError) {
|
|
328
|
+
// The candidate may have expired or been withdrawn while the publisher was being acquired.
|
|
329
|
+
if (!bucket.candidates.has(winner.id)) {
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
// Logged once per candidate: retrying every few seconds until a top-up or the deadline would spam errors.
|
|
333
|
+
if (!this.unaffordableCandidates.has(winner.id)) {
|
|
334
|
+
this.unaffordableCandidates.add(winner.id);
|
|
335
|
+
this.deps.metrics?.recordProofSubmissionUnaffordable();
|
|
336
|
+
this.log.error(
|
|
337
|
+
`Holding back the proof for candidate ${winner.id}: no publisher can afford submitting it, ` +
|
|
338
|
+
`${err.describeFunding()}; retrying`,
|
|
339
|
+
err,
|
|
340
|
+
{
|
|
341
|
+
candidateId: winner.id,
|
|
342
|
+
epoch: winner.epoch,
|
|
343
|
+
retryDelayMs: PUBLISHER_UNAFFORDABLE_RETRY_DELAY_MS,
|
|
344
|
+
...err.toLogContext(),
|
|
345
|
+
},
|
|
346
|
+
);
|
|
347
|
+
}
|
|
348
|
+
this.scheduleRetry(PUBLISHER_UNAFFORDABLE_RETRY_DELAY_MS);
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
300
351
|
// Treat this as transient: the publisher pool may be temporarily exhausted
|
|
301
352
|
// (every signer busy, funding tx in flight, etc.). Leave the candidate queued and
|
|
302
353
|
// schedule another drain after a short backoff. If the failure persists past the
|
|
@@ -307,7 +358,7 @@ export class ProofPublishingService {
|
|
|
307
358
|
retryDelayMs: PUBLISHER_ACQUIRE_RETRY_DELAY_MS,
|
|
308
359
|
err,
|
|
309
360
|
});
|
|
310
|
-
|
|
361
|
+
this.scheduleRetry(PUBLISHER_ACQUIRE_RETRY_DELAY_MS);
|
|
311
362
|
return;
|
|
312
363
|
}
|
|
313
364
|
|
|
@@ -381,6 +432,7 @@ export class ProofPublishingService {
|
|
|
381
432
|
}
|
|
382
433
|
bucket.candidates.delete(id);
|
|
383
434
|
bucket.resolvers.delete(id);
|
|
435
|
+
this.unaffordableCandidates.delete(id);
|
|
384
436
|
if (resolve) {
|
|
385
437
|
this.log.info(`Candidate ${id} resolved as ${outcome}`, { candidateId: id, outcome });
|
|
386
438
|
resolve(outcome);
|
|
@@ -12,6 +12,7 @@ import { CheckpointNumber, EpochNumber } from '@aztec-labs/foundation/branded-ty
|
|
|
12
12
|
import { areArraysEqual } from '@aztec-labs/foundation/collection';
|
|
13
13
|
import { Fr } from '@aztec-labs/foundation/curves/bn254';
|
|
14
14
|
import { EthAddress } from '@aztec-labs/foundation/eth-address';
|
|
15
|
+
import { inspect } from '@aztec-labs/foundation/inspect';
|
|
15
16
|
import { type Logger, type LoggerBindings, createLogger } from '@aztec-labs/foundation/log';
|
|
16
17
|
import { Timer } from '@aztec-labs/foundation/timer';
|
|
17
18
|
import type { PublisherConfig, TxSenderConfig } from '@aztec-labs/sequencer-client';
|
|
@@ -19,7 +20,6 @@ import type { Proof } from '@aztec-labs/stdlib/proofs';
|
|
|
19
20
|
import type { CheckpointHeader, RootRollupPublicInputs } from '@aztec-labs/stdlib/rollup';
|
|
20
21
|
import type { L1PublishProofStats } from '@aztec-labs/stdlib/stats';
|
|
21
22
|
import { type TelemetryClient, getTelemetryClient } from '@aztec-labs/telemetry-client';
|
|
22
|
-
import { inspect } from 'util';
|
|
23
23
|
import { type Hex, type TransactionReceipt, encodeFunctionData, formatEther, formatGwei } from 'viem';
|
|
24
24
|
|
|
25
25
|
import { type EstimatedSubmitProofStats, ProverNodePublisherMetrics } from './metrics.js';
|
|
@@ -38,10 +38,15 @@ export type L1SubmitEpochProofArgs = {
|
|
|
38
38
|
|
|
39
39
|
/**
|
|
40
40
|
* Result of a proof submission attempt. `'already-submitted'` means this prover had already registered a proof of
|
|
41
|
-
* the same length for the epoch on L1, so nothing was sent; it is not a failure.
|
|
41
|
+
* the same length for the epoch on L1, so nothing was sent; it is not a failure. For a full proof, only a full-epoch
|
|
42
|
+
* registration counts: a proof registered while the epoch was still open is non-full, so a full proof is resent to
|
|
43
|
+
* replace it.
|
|
42
44
|
*/
|
|
43
45
|
export type SubmitEpochProofResult = 'published' | 'already-submitted' | 'failed';
|
|
44
46
|
|
|
47
|
+
/** Maximum number of times one epoch proof is sent to L1, counting resends after it ran out of gas. */
|
|
48
|
+
const MAX_SUBMIT_EPOCH_PROOF_ATTEMPTS = 3;
|
|
49
|
+
|
|
45
50
|
export class ProverNodePublisher {
|
|
46
51
|
private metrics: ProverNodePublisherMetrics;
|
|
47
52
|
|
|
@@ -102,10 +107,15 @@ export class ProverNodePublisher {
|
|
|
102
107
|
|
|
103
108
|
// The rollup reverts on a second submission from the same prover for the same epoch and length, so don't
|
|
104
109
|
// spend gas on one. Reachable when re-running an epoch we have already submitted a proof for, which is
|
|
105
|
-
// not a failure: our reward shares for it are already registered.
|
|
110
|
+
// not a failure: our reward shares for it are already registered. The one exception is a full-epoch proof
|
|
111
|
+
// replacing a non-full registration, which is the only way to get the activity-score increase for the epoch.
|
|
106
112
|
const proverId = EthAddress.fromField(publicInputs.constants.proverId);
|
|
107
113
|
const length = toCheckpoint - fromCheckpoint + 1;
|
|
108
|
-
|
|
114
|
+
const [registered, registeredFullEpoch] = await Promise.all([
|
|
115
|
+
this.rollupContract.getHasSubmittedProof(epochNumber, length, proverId),
|
|
116
|
+
this.rollupContract.getHasSubmittedFullEpochProof(epochNumber, length, proverId),
|
|
117
|
+
]);
|
|
118
|
+
if (registeredFullEpoch || (registered && args.kind === 'partial')) {
|
|
109
119
|
this.log.warn(`Skipping epoch proof submission as prover already submitted a proof for this epoch`, {
|
|
110
120
|
...ctx,
|
|
111
121
|
proverId,
|
|
@@ -113,24 +123,46 @@ export class ProverNodePublisher {
|
|
|
113
123
|
});
|
|
114
124
|
return 'already-submitted';
|
|
115
125
|
}
|
|
126
|
+
if (registered) {
|
|
127
|
+
// L1 did not count the earlier proof as covering the whole epoch when it was submitted, e.g. it was sent
|
|
128
|
+
// while the epoch was still open: our own partial proof, or a third party proving under our prover id.
|
|
129
|
+
// If L1 does not count this proof as full either, the rollup rejects it and this returns 'failed' rather than
|
|
130
|
+
// 'already-submitted'. That is unlikely: a full session only starts once the epoch is complete, and proving a
|
|
131
|
+
// whole epoch takes long enough that the proof lands after the epoch has closed on L1. The remaining case is
|
|
132
|
+
// an under-attested checkpoint of the epoch still pending after our range when we resend. Gas estimation
|
|
133
|
+
// catches the revert before anything is sent.
|
|
134
|
+
this.log.warn(`Resubmitting full epoch proof to replace a non-full registration of the same length`, {
|
|
135
|
+
...ctx,
|
|
136
|
+
proverId,
|
|
137
|
+
length,
|
|
138
|
+
});
|
|
139
|
+
}
|
|
116
140
|
|
|
117
141
|
// Validate epoch proof range and hashes are correct before submitting
|
|
118
142
|
const provenPrefixLength = await this.validateEpochProofSubmission(args);
|
|
119
143
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
144
|
+
let sent = await this.sendSubmitEpochProofTx(args, provenPrefixLength);
|
|
145
|
+
for (let attempt = 1; sent?.receipt.status === 'reverted' && attempt < MAX_SUBMIT_EPOCH_PROOF_ATTEMPTS; attempt++) {
|
|
146
|
+
const gasLimit = await this.getOutOfGasRetryGasLimit(args, provenPrefixLength, sent.gasLimit);
|
|
147
|
+
if (gasLimit === undefined) {
|
|
148
|
+
break;
|
|
149
|
+
}
|
|
150
|
+
this.metrics.recordFailedTx();
|
|
151
|
+
this.log.warn(`Resending epoch proof with a fresh gas limit after it ran out of gas`, {
|
|
152
|
+
...ctx,
|
|
153
|
+
revertedTxHash: sent.receipt.transactionHash,
|
|
154
|
+
revertedGasLimit: sent.gasLimit,
|
|
155
|
+
gasLimit,
|
|
156
|
+
attempt: attempt + 1,
|
|
157
|
+
});
|
|
158
|
+
sent = await this.sendSubmitEpochProofTx(args, provenPrefixLength, gasLimit);
|
|
124
159
|
}
|
|
125
160
|
|
|
126
|
-
|
|
127
|
-
this.
|
|
128
|
-
|
|
129
|
-
this.l1TxUtils.getSenderAddress().toString(),
|
|
130
|
-
);
|
|
131
|
-
} catch (err) {
|
|
132
|
-
this.log.warn(`Failed to record the ETH balance of the prover node: ${err}`);
|
|
161
|
+
if (!sent) {
|
|
162
|
+
this.log.error(`Failed to mine submitEpochProof tx`, undefined, ctx);
|
|
163
|
+
return 'failed';
|
|
133
164
|
}
|
|
165
|
+
const txReceipt = sent.receipt;
|
|
134
166
|
|
|
135
167
|
// Tx was mined successfully
|
|
136
168
|
if (txReceipt.status === 'success') {
|
|
@@ -331,13 +363,15 @@ export class ProverNodePublisher {
|
|
|
331
363
|
headers: CheckpointHeader[];
|
|
332
364
|
},
|
|
333
365
|
provenPrefixLength: number,
|
|
334
|
-
|
|
366
|
+
gasLimit?: bigint,
|
|
367
|
+
): Promise<{ receipt: TransactionReceipt; gasLimit: bigint } | undefined> {
|
|
335
368
|
const txArgs = [this.getSubmitEpochProofArgs(args, provenPrefixLength)] as const;
|
|
336
369
|
|
|
337
370
|
this.log.info(`Submitting epoch proof to L1 rollup contract`, {
|
|
338
371
|
proofSize: args.proof.withoutPublicInputs().length,
|
|
339
372
|
fromCheckpoint: args.fromCheckpoint,
|
|
340
373
|
toCheckpoint: args.toCheckpoint,
|
|
374
|
+
gasLimit,
|
|
341
375
|
});
|
|
342
376
|
const data = encodeFunctionData({
|
|
343
377
|
abi: RollupAbi,
|
|
@@ -345,9 +379,9 @@ export class ProverNodePublisher {
|
|
|
345
379
|
args: txArgs,
|
|
346
380
|
});
|
|
347
381
|
try {
|
|
348
|
-
const { receipt } = await this.l1TxUtils.sendAndMonitorTransaction(
|
|
382
|
+
const { receipt, state } = await this.l1TxUtils.sendAndMonitorTransaction(
|
|
349
383
|
{ to: this.proofSubmissionTarget, data },
|
|
350
|
-
{ txTimeoutAt: args.deadline },
|
|
384
|
+
{ ...(gasLimit !== undefined && { gasLimit }), txTimeoutAt: args.deadline },
|
|
351
385
|
);
|
|
352
386
|
if (receipt.status !== 'success') {
|
|
353
387
|
const errorMsg = await this.l1TxUtils.tryGetErrorFromRevertedTx(
|
|
@@ -361,16 +395,57 @@ export class ProverNodePublisher {
|
|
|
361
395
|
/*blobInputs*/ undefined,
|
|
362
396
|
/*stateOverride*/ [],
|
|
363
397
|
);
|
|
364
|
-
this.log.error(`Rollup submit epoch proof tx reverted with ${errorMsg ?? 'unknown error'}
|
|
365
|
-
|
|
398
|
+
this.log.error(`Rollup submit epoch proof tx reverted with ${errorMsg ?? 'unknown error'}`, undefined, {
|
|
399
|
+
txHash: receipt.transactionHash,
|
|
400
|
+
gasUsed: receipt.gasUsed,
|
|
401
|
+
gasLimit: state.gasLimit,
|
|
402
|
+
});
|
|
366
403
|
}
|
|
367
|
-
return receipt;
|
|
404
|
+
return { receipt, gasLimit: state.gasLimit };
|
|
368
405
|
} catch (err) {
|
|
369
406
|
this.log.error(`Rollup submit epoch proof failed`, err);
|
|
370
407
|
return undefined;
|
|
371
408
|
}
|
|
372
409
|
}
|
|
373
410
|
|
|
411
|
+
/**
|
|
412
|
+
* Returns the gas limit to resend a reverted proof tx with, or undefined if it should not be resent.
|
|
413
|
+
*
|
|
414
|
+
* L1 state changes between our estimate and inclusion can raise the proof's gas cost without invalidating it. A
|
|
415
|
+
* receipt does not say why a tx reverted, so a proof is resent only if a fresh estimate at the latest state succeeds
|
|
416
|
+
* and asks for more gas than the reverted tx had: a revert the rollup would repeat fails the estimate, and a revert
|
|
417
|
+
* that was not out of gas gets no larger limit.
|
|
418
|
+
*/
|
|
419
|
+
private async getOutOfGasRetryGasLimit(
|
|
420
|
+
args: {
|
|
421
|
+
fromCheckpoint: CheckpointNumber;
|
|
422
|
+
toCheckpoint: CheckpointNumber;
|
|
423
|
+
publicInputs: RootRollupPublicInputs;
|
|
424
|
+
proof: Proof;
|
|
425
|
+
batchedBlobInputs: BatchedBlob;
|
|
426
|
+
attestations: ViemCommitteeAttestations;
|
|
427
|
+
headers: CheckpointHeader[];
|
|
428
|
+
},
|
|
429
|
+
provenPrefixLength: number,
|
|
430
|
+
revertedGasLimit: bigint,
|
|
431
|
+
): Promise<bigint | undefined> {
|
|
432
|
+
const data = this.encodeSubmitEpochProofCalldata(args, provenPrefixLength);
|
|
433
|
+
try {
|
|
434
|
+
const gasLimit = await this.l1TxUtils.estimateGas(this.l1TxUtils.getSenderAddress().toString(), {
|
|
435
|
+
to: this.proofSubmissionTarget,
|
|
436
|
+
data,
|
|
437
|
+
});
|
|
438
|
+
return gasLimit > revertedGasLimit ? gasLimit : undefined;
|
|
439
|
+
} catch (err) {
|
|
440
|
+
this.log.warn(`Not resending reverted epoch proof: gas estimation fails at the latest state`, {
|
|
441
|
+
fromCheckpoint: args.fromCheckpoint,
|
|
442
|
+
toCheckpoint: args.toCheckpoint,
|
|
443
|
+
err,
|
|
444
|
+
});
|
|
445
|
+
return undefined;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
374
449
|
private getEpochProofPublicInputsArgs(args: {
|
|
375
450
|
fromCheckpoint: CheckpointNumber;
|
|
376
451
|
toCheckpoint: CheckpointNumber;
|
package/src/prover-node.ts
CHANGED
|
@@ -529,9 +529,15 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
529
529
|
l2BlockSource: this.l2BlockSource,
|
|
530
530
|
dateProvider: this.dateProvider,
|
|
531
531
|
config: { skipSubmitProof: !!this.config.proverNodeDisableProofPublish },
|
|
532
|
+
metrics: this.jobMetrics,
|
|
532
533
|
bindings: this.log.getBindings(),
|
|
533
534
|
});
|
|
534
535
|
this.sessionManager = this.createSessionManager(this.publishingService);
|
|
536
|
+
if (this.config.proverNodeDisableFullEpochProving) {
|
|
537
|
+
this.log.info('Full epoch proving disabled: proving only the epochs requested through startProof', {
|
|
538
|
+
proverNodeDisableFullEpochProving: true,
|
|
539
|
+
});
|
|
540
|
+
}
|
|
535
541
|
// SessionManager owns its own periodic tick; start it here so it begins picking up
|
|
536
542
|
// epochs that become complete by time (no fresh checkpoint event) and advances once
|
|
537
543
|
// the previous epoch is proven on L1.
|
|
@@ -611,6 +617,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
611
617
|
maxPendingJobs: this.config.proverNodeMaxPendingJobs,
|
|
612
618
|
tickIntervalMs: this.config.proverNodePollingIntervalMs,
|
|
613
619
|
finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs,
|
|
620
|
+
disableFullEpochProving: !!this.config.proverNodeDisableFullEpochProving,
|
|
614
621
|
},
|
|
615
622
|
onSessionFailed: async session => {
|
|
616
623
|
await this.tryUploadEpochFailure(session.getId(), session.getCheckpoints());
|
|
@@ -6,6 +6,7 @@ import type { LoggerBindings } from '@aztec-labs/foundation/log';
|
|
|
6
6
|
import type { ProverPublisherConfig, ProverTxSenderConfig } from '@aztec-labs/sequencer-client';
|
|
7
7
|
import type { TelemetryClient } from '@aztec-labs/telemetry-client';
|
|
8
8
|
|
|
9
|
+
import { SUBMIT_EPOCH_PROOF_REQUIREMENT } from './gas_constants.js';
|
|
9
10
|
import { ProverNodePublisher } from './prover-node-publisher.js';
|
|
10
11
|
|
|
11
12
|
export class ProverPublisherFactory {
|
|
@@ -30,10 +31,14 @@ export class ProverPublisherFactory {
|
|
|
30
31
|
|
|
31
32
|
/**
|
|
32
33
|
* Creates a new Prover Publisher instance.
|
|
34
|
+
* @param opts.requireAffordableSubmission - Only select a publisher that can afford submitting an epoch proof, and
|
|
35
|
+
* throw NoAffordablePublisherError if none can.
|
|
33
36
|
* @returns A new ProverNodePublisher instance.
|
|
34
37
|
*/
|
|
35
|
-
public async create(): Promise<ProverNodePublisher> {
|
|
36
|
-
const l1Publisher = await this.deps.publisherManager.getAvailablePublisher(
|
|
38
|
+
public async create(opts?: { requireAffordableSubmission?: boolean }): Promise<ProverNodePublisher> {
|
|
39
|
+
const l1Publisher = await this.deps.publisherManager.getAvailablePublisher(undefined, {
|
|
40
|
+
requirement: opts?.requireAffordableSubmission ? SUBMIT_EPOCH_PROOF_REQUIREMENT : undefined,
|
|
41
|
+
});
|
|
37
42
|
return new ProverNodePublisher(
|
|
38
43
|
this.config,
|
|
39
44
|
{
|
package/src/session-manager.ts
CHANGED
|
@@ -45,6 +45,11 @@ export type SessionManagerConfig = {
|
|
|
45
45
|
tickIntervalMs: number;
|
|
46
46
|
/** Forwarded to every session: delay before top-tree proving, letting late reorgs settle. */
|
|
47
47
|
finalizationDelayMs: number | undefined;
|
|
48
|
+
/**
|
|
49
|
+
* Never open full sessions on our own; prove only the epochs requested through `startProof`. For a prover
|
|
50
|
+
* that serves paid partial-epoch proofs without competing for full epochs.
|
|
51
|
+
*/
|
|
52
|
+
disableFullEpochProving?: boolean;
|
|
48
53
|
};
|
|
49
54
|
|
|
50
55
|
export type SessionManagerDeps = {
|
|
@@ -235,9 +240,11 @@ export class SessionManager {
|
|
|
235
240
|
|
|
236
241
|
this.recreateInvalidSessions();
|
|
237
242
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
243
|
+
if (!this.deps.config.disableFullEpochProving) {
|
|
244
|
+
const implicatedEpochs = await this.epochsForTrigger(trigger);
|
|
245
|
+
for (const epoch of implicatedEpochs) {
|
|
246
|
+
await this.openFullSessionIfReady(epoch);
|
|
247
|
+
}
|
|
241
248
|
}
|
|
242
249
|
|
|
243
250
|
if (trigger.kind === 'start-proof') {
|