@aztec/prover-node 0.0.1-commit.3100065 → 0.0.1-commit.321f6a9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -34
- package/dest/actions/rerun-epoch-proving-job.d.ts +11 -2
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +195 -55
- package/dest/checkpoint-store.d.ts +9 -2
- package/dest/checkpoint-store.d.ts.map +1 -1
- package/dest/checkpoint-store.js +9 -0
- package/dest/config.d.ts +3 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +7 -0
- package/dest/factory.d.ts +4 -1
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +2 -1
- package/dest/job/checkpoint-prover.d.ts +34 -4
- package/dest/job/checkpoint-prover.d.ts.map +1 -1
- package/dest/job/checkpoint-prover.js +41 -8
- package/dest/job/epoch-session.d.ts +17 -3
- package/dest/job/epoch-session.d.ts.map +1 -1
- package/dest/job/epoch-session.js +28 -4
- package/dest/job/top-tree-job.d.ts +2 -2
- package/dest/job/top-tree-job.d.ts.map +1 -1
- package/dest/job/top-tree-job.js +4 -4
- package/dest/prover-node-publisher.d.ts +4 -1
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +5 -3
- package/dest/prover-node.d.ts +37 -8
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +82 -19
- package/dest/prover-publisher-factory.d.ts +3 -1
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +1 -0
- package/dest/session-manager.d.ts +16 -16
- package/dest/session-manager.d.ts.map +1 -1
- package/dest/session-manager.js +67 -63
- package/package.json +24 -23
- package/src/actions/rerun-epoch-proving-job.ts +139 -66
- package/src/checkpoint-store.ts +20 -2
- package/src/config.ts +10 -0
- package/src/factory.ts +5 -0
- package/src/job/checkpoint-prover.ts +61 -7
- package/src/job/epoch-session.ts +30 -4
- package/src/job/top-tree-job.ts +4 -4
- package/src/prover-node-publisher.ts +7 -3
- package/src/prover-node.ts +97 -20
- package/src/prover-publisher-factory.ts +3 -0
- package/src/session-manager.ts +73 -67
|
@@ -38,9 +38,25 @@ export type CheckpointProverDeps = {
|
|
|
38
38
|
txGatheringTimeoutMs: number;
|
|
39
39
|
/** Public processor deadline. */
|
|
40
40
|
deadline: Date | undefined;
|
|
41
|
+
/**
|
|
42
|
+
* Fired once when the prover's block proofs reject for a genuine (non-cancel) reason — a sub-tree
|
|
43
|
+
* fault or a prune-induced fork fault. Useful for performing post mortem on failures.
|
|
44
|
+
*/
|
|
45
|
+
onFailed?: (prover: CheckpointProver) => void;
|
|
46
|
+
/**
|
|
47
|
+
* Test-only hook: if set, invoked at the start of checkpoint execution instead of proving. Lets e2e
|
|
48
|
+
* tests force a sub-tree failure (it should throw) to exercise the checkpoint failure/upload path.
|
|
49
|
+
*/
|
|
50
|
+
checkpointProveOverride?: () => Promise<never>;
|
|
41
51
|
log: Logger;
|
|
42
52
|
};
|
|
43
53
|
|
|
54
|
+
/** Test-only hooks the store injects into every `CheckpointProver` it constructs. */
|
|
55
|
+
export type CheckpointProverTestHooks = {
|
|
56
|
+
/** If set, invoked at the start of checkpoint execution instead of proving; should throw to fail. */
|
|
57
|
+
checkpointProveOverride?: () => Promise<never>;
|
|
58
|
+
};
|
|
59
|
+
|
|
44
60
|
/** Inputs that fully describe a checkpoint at register time. */
|
|
45
61
|
export type CheckpointProverArgs = {
|
|
46
62
|
checkpoint: Checkpoint;
|
|
@@ -89,9 +105,18 @@ export class CheckpointProver {
|
|
|
89
105
|
/** Resolved by the sub-tree on success, rejected on cancel/failure. */
|
|
90
106
|
private readonly blockProofs: PromiseWithResolvers<SubTreeResult['blockProofOutputs']> = promiseWithResolvers();
|
|
91
107
|
|
|
108
|
+
// Three independent lifecycle facts — deliberately not collapsed into one status enum, because several
|
|
109
|
+
// combinations are legal and relied on: a prover can be `completed` and then `cancelled` (routine
|
|
110
|
+
// teardown of an already-proven checkpoint), or `completed` and then `failed` (block proving was
|
|
111
|
+
// enqueued, but the sub-tree subsequently faulted). Only `failed` + `cancelled` is excluded — a cancel
|
|
112
|
+
// is not a failure (enforced in `failBlockProofs`).
|
|
113
|
+
/** Block-level proving was fully *enqueued* (a progress marker; the sub-tree may still be proving). */
|
|
114
|
+
private completed = false;
|
|
115
|
+
/** Block proofs rejected for a genuine (non-cancel) reason — a sub-tree or prune-induced fork fault. */
|
|
116
|
+
private failed = false;
|
|
117
|
+
/** The prover was torn down (prune / reap / shutdown). */
|
|
92
118
|
private cancelled = false;
|
|
93
119
|
private subTree?: CheckpointSubTreeOrchestrator;
|
|
94
|
-
private completed = false;
|
|
95
120
|
private readonly abortController = new AbortController();
|
|
96
121
|
|
|
97
122
|
/** Tracks the eager gather+execute task so `cancel()` and `whenDone()` can await its unwind. */
|
|
@@ -139,9 +164,14 @@ export class CheckpointProver {
|
|
|
139
164
|
return this.cancelled;
|
|
140
165
|
}
|
|
141
166
|
|
|
142
|
-
/**
|
|
143
|
-
|
|
144
|
-
|
|
167
|
+
/**
|
|
168
|
+
* True once this prover's block proofs have rejected for a genuine (non-cancel) reason — a sub-tree
|
|
169
|
+
* proving fault or a prune-induced world-state fork fault. A failed prover cannot produce its block
|
|
170
|
+
* proofs, so the reconciler must not build (or rebuild) an EpochSession over it; it is cleared only by
|
|
171
|
+
* a prune/re-add replacing it with a fresh prover, or by expiry reaping it.
|
|
172
|
+
*/
|
|
173
|
+
public isFailed(): boolean {
|
|
174
|
+
return this.failed;
|
|
145
175
|
}
|
|
146
176
|
|
|
147
177
|
/** AbortSignal that fires on cancel — for callers that want to wire their own tasks. */
|
|
@@ -179,10 +209,29 @@ export class CheckpointProver {
|
|
|
179
209
|
this.deps.log.error(`Error in CheckpointProver ${this.id}`, err, {
|
|
180
210
|
checkpointNumber: this.checkpoint.number,
|
|
181
211
|
});
|
|
182
|
-
this.
|
|
212
|
+
this.failBlockProofs(err instanceof Error ? err : new Error(String(err)));
|
|
183
213
|
}
|
|
184
214
|
}
|
|
185
215
|
|
|
216
|
+
/**
|
|
217
|
+
* Rejects the block-proof promise and, unless this is a cancellation, records the prover as failed so
|
|
218
|
+
* the reconciler won't build an EpochSession over it. First rejection wins, so a later duplicate reject
|
|
219
|
+
* (e.g. the executeCheckpoint `finally`) is a harmless no-op.
|
|
220
|
+
*/
|
|
221
|
+
private failBlockProofs(err: Error): void {
|
|
222
|
+
if (!this.cancelled && !this.failed) {
|
|
223
|
+
this.failed = true;
|
|
224
|
+
// Notify the owner so it can upload a post-mortem for this checkpoint. Fire-and-forget: the
|
|
225
|
+
// callback must not block the prover's teardown, and a throw in it must not mask the rejection.
|
|
226
|
+
try {
|
|
227
|
+
this.deps.onFailed?.(this);
|
|
228
|
+
} catch (err) {
|
|
229
|
+
this.deps.log.error(`Error in CheckpointProver onFailed callback for ${this.id}`, err);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
this.blockProofs.reject(err);
|
|
233
|
+
}
|
|
234
|
+
|
|
186
235
|
private async gatherTxs(): Promise<Map<string, Tx>> {
|
|
187
236
|
const deadline = new Date(this.deps.dateProvider.now() + this.deps.txGatheringTimeoutMs);
|
|
188
237
|
const txsByBlock = await Promise.all(
|
|
@@ -205,6 +254,11 @@ export class CheckpointProver {
|
|
|
205
254
|
let subTreeStarted = false;
|
|
206
255
|
|
|
207
256
|
try {
|
|
257
|
+
// Test hook: force a sub-tree failure to exercise the checkpoint failure/upload path.
|
|
258
|
+
if (this.deps.checkpointProveOverride) {
|
|
259
|
+
await this.deps.checkpointProveOverride();
|
|
260
|
+
}
|
|
261
|
+
|
|
208
262
|
for (const [hash, tx] of txs) {
|
|
209
263
|
this.txs.set(hash, tx);
|
|
210
264
|
}
|
|
@@ -248,7 +302,7 @@ export class CheckpointProver {
|
|
|
248
302
|
this.deps.metrics.recordCheckpointProving(checkpointTimer.ms());
|
|
249
303
|
this.blockProofs.resolve(result.blockProofOutputs);
|
|
250
304
|
},
|
|
251
|
-
err => this.
|
|
305
|
+
err => this.failBlockProofs(err instanceof Error ? err : new Error(String(err))),
|
|
252
306
|
);
|
|
253
307
|
if (signal.aborted) {
|
|
254
308
|
return;
|
|
@@ -328,7 +382,7 @@ export class CheckpointProver {
|
|
|
328
382
|
if (subTreeStarted) {
|
|
329
383
|
await this.teardownSubTree();
|
|
330
384
|
}
|
|
331
|
-
this.
|
|
385
|
+
this.failBlockProofs(new Error(`Checkpoint ${this.id} did not complete block processing`));
|
|
332
386
|
}
|
|
333
387
|
}
|
|
334
388
|
}
|
package/src/job/epoch-session.ts
CHANGED
|
@@ -95,7 +95,11 @@ export type EpochSessionDeps = {
|
|
|
95
95
|
* initialized → awaiting-checkpoints → awaiting-root → publishing-proof → completed
|
|
96
96
|
*
|
|
97
97
|
* Terminal states map the publishing outcome: `published` → `completed`, `superseded` →
|
|
98
|
-
* `superseded`, `
|
|
98
|
+
* `superseded`, `expired` → `timed-out`, `withdrawn` → `cancelled`. A fault ends the attempt in one
|
|
99
|
+
* of two terminal states depending on its cause: `stopped` if a checkpoint prover under it failed
|
|
100
|
+
* (possibly a prune — the reconciler will rebuild over a fresh prover on re-add), or `failed` if the
|
|
101
|
+
* session's own top-tree/submit work failed while every prover was healthy (a genuine, non-prune
|
|
102
|
+
* failure the reconciler retains and uploads — see `hasFailed()`).
|
|
99
103
|
* Additionally, the session-level deadline fires `cancel('deadline')` and transitions
|
|
100
104
|
* to `timed-out` for the pre-submit window (top-tree proving) — the publishing service
|
|
101
105
|
* handles the post-submit window via the candidate's `deadline`.
|
|
@@ -187,6 +191,17 @@ export class EpochSession implements Traceable {
|
|
|
187
191
|
return EpochProvingJobTerminalState.includes(this.state);
|
|
188
192
|
}
|
|
189
193
|
|
|
194
|
+
/**
|
|
195
|
+
* True if the session ended in its own genuine failure — top-tree proving or L1 submission failed
|
|
196
|
+
* while every checkpoint prover succeeded. Because healthy provers rule out a prune-induced fault,
|
|
197
|
+
* this is a race-free "the epoch could not be proven" signal: the reconciler retains such a (full)
|
|
198
|
+
* session rather than re-proving it, and uploads a post-mortem. A `stopped` session (a checkpoint
|
|
199
|
+
* prover failed under it) is NOT a session failure in this sense.
|
|
200
|
+
*/
|
|
201
|
+
public hasFailed(): boolean {
|
|
202
|
+
return this.state === 'failed';
|
|
203
|
+
}
|
|
204
|
+
|
|
190
205
|
/** First block this session proves. */
|
|
191
206
|
public getStartBlockNumber(): BlockNumber {
|
|
192
207
|
return BlockNumber(this.checkpoints[0].checkpoint.blocks[0].number);
|
|
@@ -213,8 +228,17 @@ export class EpochSession implements Traceable {
|
|
|
213
228
|
uuid: this.uuid,
|
|
214
229
|
...this.spec,
|
|
215
230
|
});
|
|
231
|
+
// Distinguish the two ways an attempt can fault:
|
|
232
|
+
// - a checkpoint prover in the set has failed OR was cancelled (a sub-tree fault, a prune-induced
|
|
233
|
+
// fork fault, or a control-plane cancel that reached this catch before the reconcile marked the
|
|
234
|
+
// session 'cancelled'): end in the non-declaring terminal 'stopped'. This is not the session's own
|
|
235
|
+
// failure and may be a prune, so the reconciler does not upload it; a re-add installs a fresh prover.
|
|
236
|
+
// - no prover failed or was cancelled, yet top-tree proving or L1 submission failed: this is the
|
|
237
|
+
// session's own, genuine failure — and, because every prover is healthy and un-cancelled, it is
|
|
238
|
+
// definitively NOT a prune. End in terminal 'failed' so the reconciler retains it (no pointless
|
|
239
|
+
// re-prove) and uploads a race-free post-mortem.
|
|
216
240
|
if (!this.isTerminal()) {
|
|
217
|
-
this.state = 'failed';
|
|
241
|
+
this.state = this.checkpoints.some(c => c.isFailed() || c.isCancelled()) ? 'stopped' : 'failed';
|
|
218
242
|
}
|
|
219
243
|
} finally {
|
|
220
244
|
clearTimeout(this.deadlineTimeoutHandler);
|
|
@@ -228,7 +252,7 @@ export class EpochSession implements Traceable {
|
|
|
228
252
|
* Cancels the session. Idempotent. Withdraws any submitted candidate from the
|
|
229
253
|
* publishing service so the in-flight publisher (if any) is interrupted.
|
|
230
254
|
*/
|
|
231
|
-
public async cancel(reason = 'cancelled'): Promise<void> {
|
|
255
|
+
public async cancel(reason = 'cancelled', { abortJobs = true }: { abortJobs?: boolean } = {}): Promise<void> {
|
|
232
256
|
if (this.isTerminal()) {
|
|
233
257
|
return;
|
|
234
258
|
}
|
|
@@ -247,7 +271,9 @@ export class EpochSession implements Traceable {
|
|
|
247
271
|
if (this.topTreeJob && !this.topTreeJob.isCancelled()) {
|
|
248
272
|
const job = this.topTreeJob;
|
|
249
273
|
this.topTreeJob = undefined;
|
|
250
|
-
|
|
274
|
+
// On a clean shutdown we leave the in-flight broker jobs alone so a restart can reuse them;
|
|
275
|
+
// other cancellations (reorg, supersede, deadline) abort them since their inputs are stale.
|
|
276
|
+
job.cancel(abortJobs);
|
|
251
277
|
this.pendingTopTreeCleanups.push(job);
|
|
252
278
|
}
|
|
253
279
|
await this.teardownTopTreeIfNeeded();
|
package/src/job/top-tree-job.ts
CHANGED
|
@@ -128,7 +128,7 @@ export class TopTreeJob {
|
|
|
128
128
|
* via `whenDone()` — the parent collects the cancelled job and awaits all
|
|
129
129
|
* pending top-tree teardowns at the end of the epoch.
|
|
130
130
|
*/
|
|
131
|
-
public cancel(): void {
|
|
131
|
+
public cancel(abortJobs = true): void {
|
|
132
132
|
if (this.cancelled) {
|
|
133
133
|
return;
|
|
134
134
|
}
|
|
@@ -146,7 +146,7 @@ export class TopTreeJob {
|
|
|
146
146
|
// Fire and forget: parent awaits the cancel-driven teardown via whenDone(); the
|
|
147
147
|
// chained .catch swallows rejections so the unawaited promise doesn't surface
|
|
148
148
|
// as an unhandled rejection.
|
|
149
|
-
this.cancelPromise = this.runCancel().catch(() => {});
|
|
149
|
+
this.cancelPromise = this.runCancel(abortJobs).catch(() => {});
|
|
150
150
|
}
|
|
151
151
|
|
|
152
152
|
/** Resolves once the cancel-driven teardown of the underlying orchestrator has unwound. */
|
|
@@ -156,9 +156,9 @@ export class TopTreeJob {
|
|
|
156
156
|
}
|
|
157
157
|
}
|
|
158
158
|
|
|
159
|
-
private async runCancel(): Promise<void> {
|
|
159
|
+
private async runCancel(abortJobs: boolean): Promise<void> {
|
|
160
160
|
try {
|
|
161
|
-
this.topTree.cancel({ abortJobs
|
|
161
|
+
this.topTree.cancel({ abortJobs });
|
|
162
162
|
} catch (err) {
|
|
163
163
|
this.deps.log.error('Error cancelling top tree', err);
|
|
164
164
|
}
|
|
@@ -40,6 +40,8 @@ export class ProverNodePublisher {
|
|
|
40
40
|
|
|
41
41
|
protected rollupContract: RollupContract;
|
|
42
42
|
|
|
43
|
+
protected proofSubmissionTarget: Hex;
|
|
44
|
+
|
|
43
45
|
public readonly l1TxUtils: L1TxUtils;
|
|
44
46
|
|
|
45
47
|
constructor(
|
|
@@ -47,6 +49,7 @@ export class ProverNodePublisher {
|
|
|
47
49
|
deps: {
|
|
48
50
|
rollupContract: RollupContract;
|
|
49
51
|
l1TxUtils: L1TxUtils;
|
|
52
|
+
proofSubmissionTarget?: EthAddress;
|
|
50
53
|
telemetry?: TelemetryClient;
|
|
51
54
|
},
|
|
52
55
|
bindings?: LoggerBindings,
|
|
@@ -57,6 +60,7 @@ export class ProverNodePublisher {
|
|
|
57
60
|
this.log = createLogger('prover-node:l1-tx-publisher', bindings);
|
|
58
61
|
|
|
59
62
|
this.rollupContract = deps.rollupContract;
|
|
63
|
+
this.proofSubmissionTarget = deps.proofSubmissionTarget?.toString() ?? deps.rollupContract.address;
|
|
60
64
|
this.l1TxUtils = deps.l1TxUtils;
|
|
61
65
|
}
|
|
62
66
|
|
|
@@ -217,7 +221,7 @@ export class ProverNodePublisher {
|
|
|
217
221
|
const senderAddress = this.l1TxUtils.getSenderAddress();
|
|
218
222
|
|
|
219
223
|
const [gasLimit, gasPrice, latestBlock] = await Promise.all([
|
|
220
|
-
this.l1TxUtils.estimateGas(senderAddress.toString() as `0x${string}`, { to: this.
|
|
224
|
+
this.l1TxUtils.estimateGas(senderAddress.toString() as `0x${string}`, { to: this.proofSubmissionTarget, data }),
|
|
221
225
|
this.l1TxUtils.getGasPrice(),
|
|
222
226
|
this.l1TxUtils.client.getBlock({ blockTag: 'latest' }),
|
|
223
227
|
]);
|
|
@@ -288,7 +292,7 @@ export class ProverNodePublisher {
|
|
|
288
292
|
});
|
|
289
293
|
try {
|
|
290
294
|
const { receipt } = await this.l1TxUtils.sendAndMonitorTransaction(
|
|
291
|
-
{ to: this.
|
|
295
|
+
{ to: this.proofSubmissionTarget, data },
|
|
292
296
|
{ txTimeoutAt: args.deadline },
|
|
293
297
|
);
|
|
294
298
|
if (receipt.status !== 'success') {
|
|
@@ -298,7 +302,7 @@ export class ProverNodePublisher {
|
|
|
298
302
|
args: [...txArgs],
|
|
299
303
|
functionName: 'submitEpochRootProof',
|
|
300
304
|
abi: RollupAbi,
|
|
301
|
-
address: this.
|
|
305
|
+
address: this.proofSubmissionTarget,
|
|
302
306
|
},
|
|
303
307
|
/*blobInputs*/ undefined,
|
|
304
308
|
/*stateOverride*/ [],
|
package/src/prover-node.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
|
|
|
10
10
|
import type { EpochProverFactory } from '@aztec/prover-client';
|
|
11
11
|
import { getLastSiblingPath } from '@aztec/prover-client/helpers';
|
|
12
12
|
import { ChonkCache } from '@aztec/prover-client/orchestrator';
|
|
13
|
-
import { PublicProcessorFactory } from '@aztec/simulator/server';
|
|
13
|
+
import { type AvmSimulator, PublicProcessorFactory } from '@aztec/simulator/server';
|
|
14
14
|
import {
|
|
15
15
|
EventDrivenL2BlockStream,
|
|
16
16
|
type L2BlockId,
|
|
@@ -47,7 +47,8 @@ import {
|
|
|
47
47
|
import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
|
|
48
48
|
import { CheckpointStore, type RegisterCheckpointData } from './checkpoint-store.js';
|
|
49
49
|
import type { SpecificProverNodeConfig } from './config.js';
|
|
50
|
-
import type {
|
|
50
|
+
import type { CheckpointProver, CheckpointProverTestHooks } from './job/checkpoint-prover.js';
|
|
51
|
+
import type { EpochSessionHooks } from './job/epoch-session.js';
|
|
51
52
|
import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
|
|
52
53
|
import { ProofPublishingService } from './proof-publishing-service.js';
|
|
53
54
|
import type { ProverPublisherFactory } from './prover-publisher-factory.js';
|
|
@@ -121,6 +122,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
121
122
|
protected readonly p2pClient: { getTxProvider(): ITxProvider } & Partial<Service>,
|
|
122
123
|
protected readonly rollupContract: RollupContract,
|
|
123
124
|
protected readonly l1Metrics: L1Metrics,
|
|
125
|
+
private readonly avmSimulator: AvmSimulator,
|
|
124
126
|
config: Partial<ProverNodeOptions> = {},
|
|
125
127
|
protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
|
|
126
128
|
private delayer?: Delayer,
|
|
@@ -157,6 +159,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
157
159
|
chonkCache: this.chonkCache,
|
|
158
160
|
publicProcessorFactory: new PublicProcessorFactory(
|
|
159
161
|
this.contractDataSource,
|
|
162
|
+
this.avmSimulator,
|
|
160
163
|
this.dateProvider,
|
|
161
164
|
this.telemetryClient,
|
|
162
165
|
this.log.getBindings(),
|
|
@@ -168,6 +171,9 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
168
171
|
metrics: this.jobMetrics,
|
|
169
172
|
txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
|
|
170
173
|
deadline: undefined,
|
|
174
|
+
// A checkpoint prover that fails (a sub-tree fault or a prune-induced fork fault) uploads a
|
|
175
|
+
// post-mortem for its own checkpoint, independently of any session. Fire-and-forget.
|
|
176
|
+
onFailed: prover => void this.tryUploadCheckpointFailure(prover),
|
|
171
177
|
},
|
|
172
178
|
this.log.getBindings(),
|
|
173
179
|
);
|
|
@@ -239,11 +245,9 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
239
245
|
break;
|
|
240
246
|
}
|
|
241
247
|
}
|
|
242
|
-
//
|
|
243
|
-
|
|
244
|
-
//
|
|
245
|
-
// failure above propagates to the L2BlockStream (which logs and stops this poll pass) and
|
|
246
|
-
// skips this update, so the event is re-emitted on the next poll rather than skipped (A-1041).
|
|
248
|
+
// Advance the local tips store only after the proving-side handling (registration / prune) has
|
|
249
|
+
// succeeded. Any failure above propagates to the L2BlockStream (which logs and stops this poll
|
|
250
|
+
// pass) and skips this update, so the event is re-emitted on the next poll rather than skipped
|
|
247
251
|
await this.tipsStore.handleBlockStreamEvent(event);
|
|
248
252
|
}
|
|
249
253
|
|
|
@@ -458,8 +462,8 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
458
462
|
}
|
|
459
463
|
|
|
460
464
|
/**
|
|
461
|
-
* Releases chonk-cache entries for every block in the supplied epoch (best-effort) and
|
|
462
|
-
*
|
|
465
|
+
* Releases chonk-cache entries for every block in the supplied epoch (best-effort) and reaps every
|
|
466
|
+
* CheckpointProver in the store whose epoch is at or below it.
|
|
463
467
|
*/
|
|
464
468
|
private async expireEpoch(epoch: EpochNumber): Promise<void> {
|
|
465
469
|
try {
|
|
@@ -515,8 +519,9 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
515
519
|
});
|
|
516
520
|
this.blockStream.start();
|
|
517
521
|
|
|
518
|
-
//
|
|
519
|
-
//
|
|
522
|
+
// The periodic ticker is the sole driver of the expiry sweep: it fires every poll interval whether
|
|
523
|
+
// or not block-stream events arrive, and RunningPromise never overlaps its own runs, so the sweep's
|
|
524
|
+
// `lastExpiredEpoch` high-water mark advances — and each epoch's post-mortem uploads — exactly once.
|
|
520
525
|
this.expiryTicker = new RunningPromise(
|
|
521
526
|
() => this.checkEpochExpiry(),
|
|
522
527
|
this.log,
|
|
@@ -559,9 +564,10 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
559
564
|
}
|
|
560
565
|
|
|
561
566
|
/**
|
|
562
|
-
* Constructs the session manager. Extracted so subclasses (test harness) can swap
|
|
563
|
-
*
|
|
564
|
-
*
|
|
567
|
+
* Constructs the session manager. Extracted so subclasses (test harness) can swap the
|
|
568
|
+
* implementation. Wired to upload a post-mortem when a full session ends in its own genuine failure
|
|
569
|
+
* (`EpochSession.hasFailed()` — top-tree/submit failed with every prover healthy, so definitively not
|
|
570
|
+
* a prune). A `stopped` session (a prover under it failed) is not uploaded; it recovers on re-add.
|
|
565
571
|
*/
|
|
566
572
|
protected createSessionManager(publishingService: ProofPublishingService): SessionManager {
|
|
567
573
|
return new SessionManager({
|
|
@@ -578,7 +584,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
578
584
|
finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs,
|
|
579
585
|
},
|
|
580
586
|
onSessionFailed: async session => {
|
|
581
|
-
await this.
|
|
587
|
+
await this.tryUploadEpochFailure(session.getId(), session.getCheckpoints());
|
|
582
588
|
},
|
|
583
589
|
bindings: this.log.getBindings(),
|
|
584
590
|
});
|
|
@@ -596,15 +602,32 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
596
602
|
this.sessionManager.setSessionHooks(hooks);
|
|
597
603
|
}
|
|
598
604
|
|
|
599
|
-
/**
|
|
600
|
-
|
|
601
|
-
|
|
605
|
+
/**
|
|
606
|
+
* Installs checkpoint-prover test hooks (e.g. forcing a sub-tree failure) applied to every
|
|
607
|
+
* CheckpointProver constructed after this call. For the e2e harness only.
|
|
608
|
+
*/
|
|
609
|
+
public setCheckpointHooks(hooks: CheckpointProverTestHooks): void {
|
|
610
|
+
this.checkpointStore.setTestHooks(hooks);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/**
|
|
614
|
+
* Uploads a post-mortem snapshot for an epoch whose full session failed to prove, built from that
|
|
615
|
+
* session's checkpoint provers. Fired from the session manager's `onSessionFailed` callback (a
|
|
616
|
+
* genuine, race-free failure). Exposed as a method so tests can spy on it. No-ops if no failed-epoch
|
|
617
|
+
* store is configured or the checkpoint set is empty.
|
|
618
|
+
*/
|
|
619
|
+
public async tryUploadEpochFailure(
|
|
620
|
+
id: string,
|
|
621
|
+
checkpoints: readonly CheckpointProver[],
|
|
622
|
+
): Promise<string | undefined> {
|
|
623
|
+
if (!this.config.proverNodeFailedEpochStore || checkpoints.length === 0) {
|
|
602
624
|
return undefined;
|
|
603
625
|
}
|
|
604
|
-
const data = SessionManager.
|
|
626
|
+
const data = SessionManager.buildProvingData(checkpoints);
|
|
605
627
|
return await uploadEpochProofFailure(
|
|
606
628
|
this.config.proverNodeFailedEpochStore,
|
|
607
|
-
session.
|
|
629
|
+
// The session's own id; `uploadEpochProofFailure` already prefixes the path with the epoch number.
|
|
630
|
+
id,
|
|
608
631
|
data,
|
|
609
632
|
this.l2BlockSource as Archiver,
|
|
610
633
|
this.worldState,
|
|
@@ -613,8 +636,62 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
613
636
|
);
|
|
614
637
|
}
|
|
615
638
|
|
|
639
|
+
/**
|
|
640
|
+
* Uploads a post-mortem for a single failed checkpoint prover, built from just that checkpoint's
|
|
641
|
+
* proving data. Fired (fire-and-forget) from the store's `onFailed` callback for any non-cancel
|
|
642
|
+
* block-proof failure — a genuine sub-tree fault or a prune-induced fork fault alike. No-ops if no
|
|
643
|
+
* failed-epoch store is configured, or if the checkpoint is no longer canonical (a prune left nothing
|
|
644
|
+
* to diagnose). Swallows its own errors so a fire-and-forget caller can't leak.
|
|
645
|
+
*/
|
|
646
|
+
public async tryUploadCheckpointFailure(prover: CheckpointProver): Promise<string | undefined> {
|
|
647
|
+
if (!this.config.proverNodeFailedEpochStore) {
|
|
648
|
+
return undefined;
|
|
649
|
+
}
|
|
650
|
+
try {
|
|
651
|
+
// A prune-induced fork fault and a genuine sub-tree failure are indistinguishable at the moment the
|
|
652
|
+
// prover rejects (no control-plane cancel has landed yet). But the archiver is the authoritative
|
|
653
|
+
// committed chain: if this checkpoint was pruned out, its last block is no longer canonical there.
|
|
654
|
+
// Only upload for a checkpoint that still exists on-chain — a prune leaves nothing to diagnose, and
|
|
655
|
+
// the snapshot (full world-state + archiver) is expensive to produce and store.
|
|
656
|
+
if (!(await this.isCheckpointCanonical(prover.checkpoint))) {
|
|
657
|
+
this.log.debug(`Skipping checkpoint-failure upload for ${prover.id}: no longer canonical (pruned)`, {
|
|
658
|
+
checkpointNumber: prover.checkpoint.number,
|
|
659
|
+
});
|
|
660
|
+
return undefined;
|
|
661
|
+
}
|
|
662
|
+
const data = SessionManager.buildProvingData([prover]);
|
|
663
|
+
return await uploadEpochProofFailure(
|
|
664
|
+
this.config.proverNodeFailedEpochStore,
|
|
665
|
+
// The prover's content-addressed id; the epoch number is already in the upload path.
|
|
666
|
+
prover.id,
|
|
667
|
+
data,
|
|
668
|
+
this.l2BlockSource as Archiver,
|
|
669
|
+
this.worldState,
|
|
670
|
+
assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
|
|
671
|
+
this.log,
|
|
672
|
+
);
|
|
673
|
+
} catch (err) {
|
|
674
|
+
this.log.error(`Error uploading checkpoint failure for ${prover.id}`, err);
|
|
675
|
+
return undefined;
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
616
679
|
// ---------------- helpers ----------------
|
|
617
680
|
|
|
681
|
+
/**
|
|
682
|
+
* True if the checkpoint still exists on the canonical chain: the archiver holds a block at its last
|
|
683
|
+
* block's height whose archive root matches. A prune (fork fault) leaves the block missing or replaced,
|
|
684
|
+
* so this returns false. Protected for direct unit-test access.
|
|
685
|
+
*/
|
|
686
|
+
protected async isCheckpointCanonical(checkpoint: Checkpoint): Promise<boolean> {
|
|
687
|
+
const lastBlock = checkpoint.blocks.at(-1);
|
|
688
|
+
if (!lastBlock) {
|
|
689
|
+
return false;
|
|
690
|
+
}
|
|
691
|
+
const onChain = await this.l2BlockSource.getBlock({ number: lastBlock.number });
|
|
692
|
+
return !!onChain && onChain.archive.root.equals(checkpoint.archive.root);
|
|
693
|
+
}
|
|
694
|
+
|
|
618
695
|
@memoize
|
|
619
696
|
private getL1Constants(): Promise<L1RollupConstants> {
|
|
620
697
|
return this.l2BlockSource.getL1Constants();
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { RollupContract } from '@aztec/ethereum/contracts';
|
|
2
2
|
import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
|
|
3
3
|
import type { PublisherManager } from '@aztec/ethereum/publisher-manager';
|
|
4
|
+
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
4
5
|
import type { LoggerBindings } from '@aztec/foundation/log';
|
|
5
6
|
import type { ProverPublisherConfig, ProverTxSenderConfig } from '@aztec/sequencer-client';
|
|
6
7
|
import type { TelemetryClient } from '@aztec/telemetry-client';
|
|
@@ -13,6 +14,7 @@ export class ProverPublisherFactory {
|
|
|
13
14
|
private deps: {
|
|
14
15
|
rollupContract: RollupContract;
|
|
15
16
|
publisherManager: PublisherManager<L1TxUtils>;
|
|
17
|
+
proofSubmissionTarget?: EthAddress;
|
|
16
18
|
telemetry?: TelemetryClient;
|
|
17
19
|
},
|
|
18
20
|
private bindings?: LoggerBindings,
|
|
@@ -37,6 +39,7 @@ export class ProverPublisherFactory {
|
|
|
37
39
|
{
|
|
38
40
|
rollupContract: this.deps.rollupContract,
|
|
39
41
|
l1TxUtils: l1Publisher,
|
|
42
|
+
proofSubmissionTarget: this.deps.proofSubmissionTarget,
|
|
40
43
|
telemetry: this.deps.telemetry,
|
|
41
44
|
},
|
|
42
45
|
this.bindings,
|