@aztec/prover-node 5.0.0-rc.1 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -68
- package/dest/actions/rerun-epoch-proving-job.d.ts +2 -2
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/checkpoint-store.d.ts +42 -37
- package/dest/checkpoint-store.d.ts.map +1 -1
- package/dest/checkpoint-store.js +65 -77
- package/dest/job/checkpoint-prover.d.ts +6 -16
- package/dest/job/checkpoint-prover.d.ts.map +1 -1
- package/dest/job/checkpoint-prover.js +15 -35
- package/dest/job/epoch-session.d.ts +2 -2
- package/dest/job/epoch-session.d.ts.map +1 -1
- package/dest/job/epoch-session.js +20 -9
- package/dest/metrics.d.ts +7 -4
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +14 -6
- package/dest/proof-publishing-service.d.ts +4 -2
- package/dest/proof-publishing-service.d.ts.map +1 -1
- package/dest/proof-publishing-service.js +1 -0
- package/dest/prover-node-publisher.d.ts +5 -5
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +2 -3
- package/dest/prover-node.d.ts +25 -15
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +152 -51
- package/dest/session-manager.d.ts +2 -2
- package/dest/session-manager.d.ts.map +1 -1
- package/dest/session-manager.js +38 -8
- package/package.json +23 -23
- package/src/checkpoint-store.ts +66 -85
- package/src/job/checkpoint-prover.ts +17 -40
- package/src/job/epoch-session.ts +21 -9
- package/src/metrics.ts +15 -12
- package/src/proof-publishing-service.ts +4 -1
- package/src/prover-node-publisher.ts +11 -10
- package/src/prover-node.ts +170 -60
- package/src/session-manager.ts +38 -7
package/src/checkpoint-store.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import type { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
1
|
+
import type { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
3
|
-
import { RunningPromise } from '@aztec/foundation/promise';
|
|
4
3
|
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
5
4
|
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
6
5
|
import { type L1RollupConstants, getEpochAtSlot, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
|
|
@@ -19,54 +18,64 @@ export type CheckpointProverFactory = (args: CheckpointProverArgs, deps: Checkpo
|
|
|
19
18
|
*
|
|
20
19
|
* The store survives every epoch / session boundary. A prover lives from its first
|
|
21
20
|
* `addOrUpdate` call until either:
|
|
22
|
-
* -
|
|
21
|
+
* - its checkpoint is pruned by an L1 reorg (`cancelAndRemoveAboveBlock`), or
|
|
23
22
|
* - its epoch's proof-submission window has closed (`reapExpired`), so the proof could no
|
|
24
23
|
* longer be accepted on L1 even if produced.
|
|
25
24
|
*
|
|
26
|
-
* A
|
|
27
|
-
*
|
|
28
|
-
*
|
|
25
|
+
* A prover's sub-tree work forks world-state per block and does not survive a prune of a base
|
|
26
|
+
* block, so there is nothing to preserve across a reorg: a pruned prover is cancelled and
|
|
27
|
+
* dropped, and a re-add (even of identical content) constructs a fresh prover.
|
|
29
28
|
*/
|
|
30
29
|
export class CheckpointStore {
|
|
31
30
|
private readonly provers = new Map<string, CheckpointProver>();
|
|
32
|
-
|
|
31
|
+
/**
|
|
32
|
+
* Teardowns of provers already removed from `provers` (by prune or reap), awaited on `stop()`.
|
|
33
|
+
* Keyed by a monotonic id rather than the prover's content id: a prune-then-re-add can leave two
|
|
34
|
+
* teardowns for the same content id in flight at once, which a content-id key would clobber.
|
|
35
|
+
*/
|
|
36
|
+
private readonly pendingTeardowns = new Map<number, Promise<void>>();
|
|
37
|
+
private nextTeardownId = 0;
|
|
33
38
|
private readonly log: Logger;
|
|
34
39
|
|
|
35
40
|
constructor(
|
|
36
|
-
private readonly l2BlockSource: Pick<L2BlockSource, '
|
|
41
|
+
private readonly l2BlockSource: Pick<L2BlockSource, 'getL1Constants'>,
|
|
37
42
|
private readonly proverDeps: Omit<CheckpointProverDeps, 'log'>,
|
|
38
|
-
private readonly options: { slotWatcherPollIntervalMs: number },
|
|
39
43
|
bindings?: LoggerBindings,
|
|
40
44
|
private readonly proverFactoryFn: CheckpointProverFactory = (args, deps) => new CheckpointProver(args, deps),
|
|
41
45
|
) {
|
|
42
46
|
this.log = createLogger('prover-node:checkpoint-store', bindings);
|
|
43
|
-
this.slotWatcher = new RunningPromise(
|
|
44
|
-
() => this.reapPrunedPastSlot(),
|
|
45
|
-
this.log,
|
|
46
|
-
this.options.slotWatcherPollIntervalMs,
|
|
47
|
-
);
|
|
48
47
|
}
|
|
49
48
|
|
|
50
49
|
public start(): Promise<void> {
|
|
51
|
-
this.slotWatcher.start();
|
|
52
50
|
return Promise.resolve();
|
|
53
51
|
}
|
|
54
52
|
|
|
55
53
|
public async stop(): Promise<void> {
|
|
56
|
-
await
|
|
57
|
-
//
|
|
54
|
+
// Cancel every live prover, then await both their teardown and any still in flight for provers
|
|
55
|
+
// already removed by a prune or reap.
|
|
58
56
|
const provers = Array.from(this.provers.values());
|
|
59
57
|
this.provers.clear();
|
|
60
58
|
for (const prover of provers) {
|
|
61
59
|
prover.cancel();
|
|
62
60
|
}
|
|
63
|
-
await Promise.allSettled(provers.map(p => p.whenDone()));
|
|
61
|
+
await Promise.allSettled([...provers.map(p => p.whenDone()), ...this.pendingTeardowns.values()]);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Tracks the teardown of a prover just removed from the store so `stop()` can await it. The entry
|
|
66
|
+
* removes itself once teardown settles, so the map stays bounded by the number in flight.
|
|
67
|
+
*/
|
|
68
|
+
private trackTeardown(prover: CheckpointProver): void {
|
|
69
|
+
const id = this.nextTeardownId++;
|
|
70
|
+
const done = prover.whenDone();
|
|
71
|
+
this.pendingTeardowns.set(id, done);
|
|
72
|
+
void done.finally(() => this.pendingTeardowns.delete(id));
|
|
64
73
|
}
|
|
65
74
|
|
|
66
75
|
/**
|
|
67
76
|
* Registers a checkpoint with the store. If a prover already exists for the
|
|
68
|
-
* `(number, slot, archive root)` content key
|
|
69
|
-
* otherwise a new prover is constructed.
|
|
77
|
+
* `(number, slot, archive root)` content key it is reused (an at-least-once re-registration of
|
|
78
|
+
* still-canonical content); otherwise a new prover is constructed.
|
|
70
79
|
*/
|
|
71
80
|
public async addOrUpdate(checkpoint: Checkpoint, data: RegisterCheckpointData): Promise<CheckpointProver> {
|
|
72
81
|
const l1Constants = await this.l2BlockSource.getL1Constants();
|
|
@@ -75,18 +84,18 @@ export class CheckpointStore {
|
|
|
75
84
|
|
|
76
85
|
const existing = this.provers.get(id);
|
|
77
86
|
if (existing) {
|
|
78
|
-
existing.markCanonical();
|
|
79
87
|
return existing;
|
|
80
88
|
}
|
|
81
89
|
|
|
82
|
-
// At most one canonical checkpoint per slot. A different
|
|
83
|
-
//
|
|
84
|
-
//
|
|
90
|
+
// At most one canonical checkpoint per slot. A different checkpoint at the same slot means the
|
|
91
|
+
// caller forgot to prune the old chain before adding the replacement — surface it rather than
|
|
92
|
+
// silently creating a parallel canonical chain. A pruned checkpoint has already been removed,
|
|
93
|
+
// so every prover still in the store is canonical.
|
|
85
94
|
for (const prover of this.provers.values()) {
|
|
86
|
-
if (prover.slotNumber === checkpoint.header.slotNumber
|
|
95
|
+
if (prover.slotNumber === checkpoint.header.slotNumber) {
|
|
87
96
|
throw new Error(
|
|
88
97
|
`Cannot add checkpoint ${checkpoint.number} (archive ${checkpoint.archive.root}) at slot ${checkpoint.header.slotNumber}: ` +
|
|
89
|
-
`a different
|
|
98
|
+
`a different checkpoint already occupies this slot. Prune it first.`,
|
|
90
99
|
);
|
|
91
100
|
}
|
|
92
101
|
}
|
|
@@ -97,15 +106,25 @@ export class CheckpointStore {
|
|
|
97
106
|
}
|
|
98
107
|
|
|
99
108
|
/**
|
|
100
|
-
*
|
|
101
|
-
* `
|
|
102
|
-
*
|
|
109
|
+
* Cancels and removes every prover that holds a block above the prune target. A checkpoint is orphaned by a prune to
|
|
110
|
+
* block `targetBlockNumber` iff its last block sits above the target — including a checkpoint whose range straddles
|
|
111
|
+
* the target (partially orphaned), which block-range marking catches without boundary ambiguity. Keying off the
|
|
112
|
+
* surviving block number (rather than a checkpoint number) is correct even when the source has already
|
|
113
|
+
* re-checkpointed past the divergence: the prune event reports the highest surviving block, which by construction
|
|
114
|
+
* survives on the source, whereas the source's current checkpointed tip can sit above the prune target.
|
|
115
|
+
*
|
|
116
|
+
* The prover's in-flight sub-tree work forks world-state per block and faults once its base block is pruned, so it
|
|
117
|
+
* cannot be reused; it is cancelled (aborting the fork reads) and dropped. A subsequent re-add constructs a fresh
|
|
118
|
+
* prover. Returns the removed provers.
|
|
103
119
|
*/
|
|
104
|
-
public
|
|
120
|
+
public cancelAndRemoveAboveBlock(targetBlockNumber: BlockNumber): CheckpointProver[] {
|
|
105
121
|
const affected: CheckpointProver[] = [];
|
|
106
|
-
for (const prover of this.provers.
|
|
107
|
-
|
|
108
|
-
|
|
122
|
+
for (const [id, prover] of Array.from(this.provers.entries())) {
|
|
123
|
+
const lastBlockNumber = prover.checkpoint.blocks.at(-1)!.number;
|
|
124
|
+
if (lastBlockNumber > targetBlockNumber) {
|
|
125
|
+
prover.cancel();
|
|
126
|
+
this.trackTeardown(prover);
|
|
127
|
+
this.provers.delete(id);
|
|
109
128
|
affected.push(prover);
|
|
110
129
|
}
|
|
111
130
|
}
|
|
@@ -113,20 +132,17 @@ export class CheckpointStore {
|
|
|
113
132
|
}
|
|
114
133
|
|
|
115
134
|
/**
|
|
116
|
-
* Drops
|
|
117
|
-
*
|
|
118
|
-
*
|
|
135
|
+
* Drops provers whose epoch is at or below the supplied expired epoch. Once an epoch's
|
|
136
|
+
* proof-submission window has closed, its proof can no longer be accepted on L1, so the
|
|
137
|
+
* prover is no longer needed.
|
|
119
138
|
*/
|
|
120
139
|
public reapExpired(expiredEpoch: EpochNumber): void {
|
|
121
140
|
const reaped: { id: string; checkpointNumber: CheckpointNumber; epochNumber: EpochNumber }[] = [];
|
|
122
141
|
for (const [id, prover] of Array.from(this.provers.entries())) {
|
|
123
|
-
if (prover.isPruned()) {
|
|
124
|
-
continue;
|
|
125
|
-
}
|
|
126
142
|
if (prover.epochNumber <= expiredEpoch) {
|
|
127
143
|
reaped.push({ id, checkpointNumber: prover.checkpoint.number, epochNumber: prover.epochNumber });
|
|
128
144
|
prover.cancel({ routine: true });
|
|
129
|
-
|
|
145
|
+
this.trackTeardown(prover);
|
|
130
146
|
this.provers.delete(id);
|
|
131
147
|
}
|
|
132
148
|
}
|
|
@@ -149,63 +165,28 @@ export class CheckpointStore {
|
|
|
149
165
|
return this.provers.get(CheckpointProver.idFor(checkpoint));
|
|
150
166
|
}
|
|
151
167
|
|
|
152
|
-
/** Every prover currently in the store
|
|
168
|
+
/** Every prover currently in the store, in insertion order. */
|
|
153
169
|
public listAll(): CheckpointProver[] {
|
|
154
170
|
return Array.from(this.provers.values());
|
|
155
171
|
}
|
|
156
172
|
|
|
157
|
-
/**
|
|
158
|
-
public
|
|
159
|
-
return Array.from(this.provers.values())
|
|
160
|
-
.filter(p => !p.isPruned())
|
|
161
|
-
.sort((a, b) => a.checkpoint.number - b.checkpoint.number);
|
|
173
|
+
/** Provers in the store, sorted by checkpoint number. */
|
|
174
|
+
public list(): CheckpointProver[] {
|
|
175
|
+
return Array.from(this.provers.values()).sort((a, b) => a.checkpoint.number - b.checkpoint.number);
|
|
162
176
|
}
|
|
163
177
|
|
|
164
178
|
/**
|
|
165
|
-
*
|
|
166
|
-
* checkpoint number.
|
|
179
|
+
* Provers whose slot is in the supplied epoch's slot range, sorted by checkpoint number.
|
|
167
180
|
*/
|
|
168
|
-
public async
|
|
181
|
+
public async listForEpoch(epoch: EpochNumber): Promise<CheckpointProver[]> {
|
|
169
182
|
const l1Constants = await this.l2BlockSource.getL1Constants();
|
|
170
183
|
const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, l1Constants);
|
|
171
|
-
return this.
|
|
184
|
+
return this.listInSlotRange(fromSlot, toSlot);
|
|
172
185
|
}
|
|
173
186
|
|
|
174
|
-
/**
|
|
175
|
-
public
|
|
176
|
-
return this.
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
/**
|
|
180
|
-
* SlotWatcher tick: reap pruned provers whose slot has passed the chain's synced
|
|
181
|
-
* slot. Once the chain has moved past, no re-add can revive the prover and its
|
|
182
|
-
* content key is unique enough that an actual re-add would create a new entry.
|
|
183
|
-
*
|
|
184
|
-
* Protected so unit tests can drive a single tick without spinning up the
|
|
185
|
-
* `RunningPromise` and waiting on its interval.
|
|
186
|
-
*/
|
|
187
|
-
protected async reapPrunedPastSlot(): Promise<void> {
|
|
188
|
-
let syncedSlot: SlotNumber | undefined;
|
|
189
|
-
try {
|
|
190
|
-
syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
191
|
-
} catch (err) {
|
|
192
|
-
this.log.debug(`SlotWatcher could not read synced slot`, { error: `${err}` });
|
|
193
|
-
return;
|
|
194
|
-
}
|
|
195
|
-
if (syncedSlot === undefined) {
|
|
196
|
-
return;
|
|
197
|
-
}
|
|
198
|
-
for (const [id, prover] of Array.from(this.provers.entries())) {
|
|
199
|
-
if (prover.isPruned() && prover.slotNumber < syncedSlot) {
|
|
200
|
-
this.log.info(`Reaping pruned CheckpointProver ${id}: slot ${prover.slotNumber} < synced ${syncedSlot}`, {
|
|
201
|
-
checkpointNumber: prover.checkpoint.number,
|
|
202
|
-
slotNumber: prover.slotNumber,
|
|
203
|
-
});
|
|
204
|
-
prover.cancel();
|
|
205
|
-
void prover.whenDone();
|
|
206
|
-
this.provers.delete(id);
|
|
207
|
-
}
|
|
208
|
-
}
|
|
187
|
+
/** Provers whose slot falls within `[fromSlot, toSlot]`, sorted by checkpoint number. */
|
|
188
|
+
public listInSlotRange(fromSlot: SlotNumber, toSlot: SlotNumber): CheckpointProver[] {
|
|
189
|
+
return this.list().filter(p => p.slotNumber >= fromSlot && p.slotNumber <= toSlot);
|
|
209
190
|
}
|
|
210
191
|
}
|
|
211
192
|
|
|
@@ -59,16 +59,16 @@ export type CheckpointProverArgs = {
|
|
|
59
59
|
* The store creates a CheckpointProver once per content-key. Keying on the checkpoint's
|
|
60
60
|
* own archive root (its post-state) means two checkpoints are "the same" iff they
|
|
61
61
|
* produce the same archive — so a reorg branch, or a replacement built on the same
|
|
62
|
-
* predecessor but with different content, keys to a distinct prover
|
|
63
|
-
* re-add keys to the same one and reuses its in-flight sub-tree work.
|
|
62
|
+
* predecessor but with different content, keys to a distinct prover.
|
|
64
63
|
*
|
|
65
64
|
* The prover eagerly starts its own tx gather and sub-tree work in the constructor, so
|
|
66
65
|
* callers only need to call `whenBlockProofsReady()` to obtain the resulting block-rollup
|
|
67
66
|
* proofs.
|
|
68
67
|
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
* a
|
|
68
|
+
* A CheckpointProver does not survive a prune: its sub-tree work forks world-state per
|
|
69
|
+
* block, and an L1 prune of a base block faults those reads. The store therefore cancels and
|
|
70
|
+
* discards a prover when its checkpoint is pruned, and a re-add (even of identical content)
|
|
71
|
+
* constructs a fresh prover.
|
|
72
72
|
*
|
|
73
73
|
* `cancel()` is idempotent. It aborts the gather + sub-tree, rejects the block-proof
|
|
74
74
|
* promise, and exposes a `whenDone()` that resolves once teardown has unwound.
|
|
@@ -92,8 +92,6 @@ export class CheckpointProver {
|
|
|
92
92
|
private cancelled = false;
|
|
93
93
|
private subTree?: CheckpointSubTreeOrchestrator;
|
|
94
94
|
private completed = false;
|
|
95
|
-
/** Pruned in the canonical chain but not yet reaped — sub-tree continues running. */
|
|
96
|
-
private pruned = false;
|
|
97
95
|
private readonly abortController = new AbortController();
|
|
98
96
|
|
|
99
97
|
/** Tracks the eager gather+execute task so `cancel()` and `whenDone()` can await its unwind. */
|
|
@@ -146,37 +144,6 @@ export class CheckpointProver {
|
|
|
146
144
|
return this.completed;
|
|
147
145
|
}
|
|
148
146
|
|
|
149
|
-
public isPruned(): boolean {
|
|
150
|
-
return this.pruned;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
/**
|
|
154
|
-
* Mark this prover as no longer present in the canonical chain. Sub-tree proving keeps
|
|
155
|
-
* running so the work survives if the checkpoint is re-added. Idempotent.
|
|
156
|
-
*/
|
|
157
|
-
public markPruned(): void {
|
|
158
|
-
if (this.pruned) {
|
|
159
|
-
return;
|
|
160
|
-
}
|
|
161
|
-
this.pruned = true;
|
|
162
|
-
this.deps.log.info(`Marking CheckpointProver ${this.id} as pruned`, {
|
|
163
|
-
checkpointNumber: this.checkpoint.number,
|
|
164
|
-
slotNumber: this.slotNumber,
|
|
165
|
-
});
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
/** Mark this prover as part of the canonical chain again after a re-add. Idempotent. */
|
|
169
|
-
public markCanonical(): void {
|
|
170
|
-
if (!this.pruned) {
|
|
171
|
-
return;
|
|
172
|
-
}
|
|
173
|
-
this.pruned = false;
|
|
174
|
-
this.deps.log.info(`Marking CheckpointProver ${this.id} as canonical`, {
|
|
175
|
-
checkpointNumber: this.checkpoint.number,
|
|
176
|
-
slotNumber: this.slotNumber,
|
|
177
|
-
});
|
|
178
|
-
}
|
|
179
|
-
|
|
180
147
|
/** AbortSignal that fires on cancel — for callers that want to wire their own tasks. */
|
|
181
148
|
public getAbortSignal(): AbortSignal {
|
|
182
149
|
return this.abortController.signal;
|
|
@@ -277,6 +244,8 @@ export class CheckpointProver {
|
|
|
277
244
|
checkpointNumber: this.checkpoint.number,
|
|
278
245
|
blockProofCount: result.blockProofOutputs.length,
|
|
279
246
|
});
|
|
247
|
+
// Spans processing + proving (from executeCheckpoint start, after tx gathering) to proofs ready.
|
|
248
|
+
this.deps.metrics.recordCheckpointProving(checkpointTimer.ms());
|
|
280
249
|
this.blockProofs.resolve(result.blockProofOutputs);
|
|
281
250
|
},
|
|
282
251
|
err => this.blockProofs.reject(err),
|
|
@@ -344,7 +313,8 @@ export class CheckpointProver {
|
|
|
344
313
|
}
|
|
345
314
|
|
|
346
315
|
this.completed = true;
|
|
347
|
-
this.
|
|
316
|
+
const numTxs = this.checkpoint.blocks.reduce((acc, block) => acc + block.body.txEffects.length, 0);
|
|
317
|
+
this.deps.metrics.recordCheckpointProcessing(checkpointTimer.ms(), this.checkpoint.blocks.length, numTxs);
|
|
348
318
|
this.deps.log.info(
|
|
349
319
|
`Finished enqueueing block-level proving for checkpoint ${this.checkpoint.number} in ${checkpointTimer.ms()}ms`,
|
|
350
320
|
{
|
|
@@ -429,7 +399,14 @@ export class CheckpointProver {
|
|
|
429
399
|
}
|
|
430
400
|
|
|
431
401
|
private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {
|
|
432
|
-
|
|
402
|
+
// Pass the abort signal so a prune-driven cancel stops the current block's public execution
|
|
403
|
+
// immediately, rather than running it to completion before the next `signal.aborted` check.
|
|
404
|
+
// On abort `process` returns a partial result, the length check below throws, and
|
|
405
|
+
// `gatherAndExecute` swallows it via its `cancelled` guard.
|
|
406
|
+
const [processedTxs, failedTxs] = await publicProcessor.process(txs, {
|
|
407
|
+
deadline: this.deps.deadline,
|
|
408
|
+
signal: this.abortController.signal,
|
|
409
|
+
});
|
|
433
410
|
|
|
434
411
|
if (failedTxs.length) {
|
|
435
412
|
const failedTxHashes = await Promise.all(failedTxs.map(({ tx }) => tx.getTxHash()));
|
package/src/job/epoch-session.ts
CHANGED
|
@@ -92,7 +92,7 @@ export type EpochSessionDeps = {
|
|
|
92
92
|
*
|
|
93
93
|
* Lifecycle (happy path):
|
|
94
94
|
*
|
|
95
|
-
* initialized → awaiting-checkpoints → completed
|
|
95
|
+
* initialized → awaiting-checkpoints → awaiting-root → publishing-proof → completed
|
|
96
96
|
*
|
|
97
97
|
* Terminal states map the publishing outcome: `published` → `completed`, `superseded` →
|
|
98
98
|
* `superseded`, `failed` → `failed`, `expired` → `timed-out`, `withdrawn` → `cancelled`.
|
|
@@ -320,6 +320,12 @@ export class EpochSession implements Traceable {
|
|
|
320
320
|
0,
|
|
321
321
|
);
|
|
322
322
|
|
|
323
|
+
// Reflect the publish phase. Guard against a terminal state set concurrently by cancel() — the
|
|
324
|
+
// post-submit isTerminal() check below relies on cancel still winning.
|
|
325
|
+
if (!this.isTerminal()) {
|
|
326
|
+
this.state = 'publishing-proof';
|
|
327
|
+
}
|
|
328
|
+
|
|
323
329
|
const outcome = await this.deps.publishingService.submit({
|
|
324
330
|
id: this.uuid,
|
|
325
331
|
epoch: this.spec.epochNumber,
|
|
@@ -333,6 +339,7 @@ export class EpochSession implements Traceable {
|
|
|
333
339
|
proof: proof.proof,
|
|
334
340
|
batchedBlobInputs: proof.batchedBlobInputs,
|
|
335
341
|
attestations,
|
|
342
|
+
headers: this.checkpoints.map(c => c.checkpoint.header),
|
|
336
343
|
});
|
|
337
344
|
|
|
338
345
|
if (this.isTerminal()) {
|
|
@@ -347,7 +354,7 @@ export class EpochSession implements Traceable {
|
|
|
347
354
|
{ uuid: this.uuid, ...this.spec },
|
|
348
355
|
);
|
|
349
356
|
this.state = 'completed';
|
|
350
|
-
this.deps.metrics.recordProvingJob(timer.ms(),
|
|
357
|
+
this.deps.metrics.recordProvingJob(timer.ms(), checkpointCount, epochSizeBlocks, epochSizeTxs);
|
|
351
358
|
return;
|
|
352
359
|
case 'superseded':
|
|
353
360
|
this.log.info(`EpochSession ${this.uuid} superseded by a longer candidate`, {
|
|
@@ -410,15 +417,20 @@ export class EpochSession implements Traceable {
|
|
|
410
417
|
}
|
|
411
418
|
}
|
|
412
419
|
|
|
413
|
-
private toTopTreeHooks(): TopTreeJobHooks
|
|
420
|
+
private toTopTreeHooks(): TopTreeJobHooks {
|
|
414
421
|
const hooks = this.deps.hooks;
|
|
415
|
-
if (!hooks?.beforeTopTreeProve && !hooks?.afterTopTreeProve && !hooks?.topTreeProveOverride) {
|
|
416
|
-
return undefined;
|
|
417
|
-
}
|
|
418
422
|
return {
|
|
419
|
-
beforeProve
|
|
420
|
-
|
|
421
|
-
|
|
423
|
+
// `beforeProve` fires once the sub-tree (checkpoint block) proofs are ready and the root prove is
|
|
424
|
+
// about to start — the boundary between `awaiting-checkpoints` and proving the top tree. Don't
|
|
425
|
+
// clobber a terminal state set concurrently by cancel().
|
|
426
|
+
beforeProve: async () => {
|
|
427
|
+
if (!this.isTerminal()) {
|
|
428
|
+
this.state = 'awaiting-root';
|
|
429
|
+
}
|
|
430
|
+
await hooks?.beforeTopTreeProve?.();
|
|
431
|
+
},
|
|
432
|
+
afterProve: hooks?.afterTopTreeProve,
|
|
433
|
+
proveOverride: hooks?.topTreeProveOverride,
|
|
422
434
|
};
|
|
423
435
|
}
|
|
424
436
|
}
|
package/src/metrics.ts
CHANGED
|
@@ -22,7 +22,6 @@ import type { CheckpointStore } from './checkpoint-store.js';
|
|
|
22
22
|
import type { SessionManager } from './session-manager.js';
|
|
23
23
|
|
|
24
24
|
export class ProverNodeJobMetrics {
|
|
25
|
-
proverEpochExecutionDuration: Histogram;
|
|
26
25
|
provingJobDuration: Histogram;
|
|
27
26
|
provingJobCheckpoints: Gauge;
|
|
28
27
|
provingJobBlocks: Gauge;
|
|
@@ -31,6 +30,9 @@ export class ProverNodeJobMetrics {
|
|
|
31
30
|
private blobProcessingDuration: Gauge;
|
|
32
31
|
private blockProcessingDuration: Histogram;
|
|
33
32
|
private checkpointProcessingDuration: Histogram;
|
|
33
|
+
private checkpointProvingDuration: Histogram;
|
|
34
|
+
private checkpointBlocks: Histogram;
|
|
35
|
+
private checkpointTransactions: Histogram;
|
|
34
36
|
|
|
35
37
|
/** Observable gauges for live state. Registered via `observeState(...)` once the
|
|
36
38
|
* CheckpointStore and SessionManager are available. */
|
|
@@ -44,7 +46,6 @@ export class ProverNodeJobMetrics {
|
|
|
44
46
|
public readonly tracer: Tracer,
|
|
45
47
|
private logger = createLogger('prover-node:publisher:metrics'),
|
|
46
48
|
) {
|
|
47
|
-
this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION);
|
|
48
49
|
this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION);
|
|
49
50
|
this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
|
|
50
51
|
this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
|
|
@@ -53,16 +54,12 @@ export class ProverNodeJobMetrics {
|
|
|
53
54
|
this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
|
|
54
55
|
this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
|
|
55
56
|
this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
|
|
57
|
+
this.checkpointProvingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROVING_DURATION);
|
|
58
|
+
this.checkpointBlocks = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_BLOCKS);
|
|
59
|
+
this.checkpointTransactions = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_TRANSACTIONS);
|
|
56
60
|
}
|
|
57
61
|
|
|
58
|
-
public recordProvingJob(
|
|
59
|
-
executionTimeMs: number,
|
|
60
|
-
totalTimeMs: number,
|
|
61
|
-
numCheckpoints: number,
|
|
62
|
-
numBlocks: number,
|
|
63
|
-
numTxs: number,
|
|
64
|
-
) {
|
|
65
|
-
this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
|
|
62
|
+
public recordProvingJob(totalTimeMs: number, numCheckpoints: number, numBlocks: number, numTxs: number) {
|
|
66
63
|
this.provingJobDuration.record(totalTimeMs / 1000);
|
|
67
64
|
this.provingJobCheckpoints.record(Math.floor(numCheckpoints));
|
|
68
65
|
this.provingJobBlocks.record(Math.floor(numBlocks));
|
|
@@ -77,8 +74,14 @@ export class ProverNodeJobMetrics {
|
|
|
77
74
|
this.blockProcessingDuration.record(Math.ceil(durationMs));
|
|
78
75
|
}
|
|
79
76
|
|
|
80
|
-
public recordCheckpointProcessing(durationMs: number) {
|
|
77
|
+
public recordCheckpointProcessing(durationMs: number, numBlocks: number, numTxs: number) {
|
|
81
78
|
this.checkpointProcessingDuration.record(Math.ceil(durationMs));
|
|
79
|
+
this.checkpointBlocks.record(Math.floor(numBlocks));
|
|
80
|
+
this.checkpointTransactions.record(Math.floor(numTxs));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
public recordCheckpointProving(durationMs: number) {
|
|
84
|
+
this.checkpointProvingDuration.record(Math.ceil(durationMs));
|
|
82
85
|
}
|
|
83
86
|
|
|
84
87
|
/**
|
|
@@ -93,7 +96,7 @@ export class ProverNodeJobMetrics {
|
|
|
93
96
|
this.activeCheckpoints = this.meter.createObservableGauge(Metrics.PROVER_NODE_ACTIVE_CHECKPOINTS);
|
|
94
97
|
this.activeEpochSessions = this.meter.createObservableGauge(Metrics.PROVER_NODE_ACTIVE_EPOCH_SESSIONS);
|
|
95
98
|
this.stateObserver = (observer: BatchObservableResult) => {
|
|
96
|
-
observer.observe(this.activeCheckpoints!, checkpointStore.
|
|
99
|
+
observer.observe(this.activeCheckpoints!, checkpointStore.listAll().length);
|
|
97
100
|
let full = 0;
|
|
98
101
|
let partial = 0;
|
|
99
102
|
for (const session of sessionManager.allSessions()) {
|
|
@@ -7,7 +7,7 @@ import { SerialQueue } from '@aztec/foundation/queue';
|
|
|
7
7
|
import type { DateProvider } from '@aztec/foundation/timer';
|
|
8
8
|
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
9
9
|
import type { Proof } from '@aztec/stdlib/proofs';
|
|
10
|
-
import type { RootRollupPublicInputs } from '@aztec/stdlib/rollup';
|
|
10
|
+
import type { CheckpointHeader, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
|
|
11
11
|
|
|
12
12
|
import type { ProverNodePublisher } from './prover-node-publisher.js';
|
|
13
13
|
import type { ProverPublisherFactory } from './prover-publisher-factory.js';
|
|
@@ -45,6 +45,8 @@ export type PublishCandidate = {
|
|
|
45
45
|
proof: Proof;
|
|
46
46
|
batchedBlobInputs: BatchedBlob;
|
|
47
47
|
attestations: ViemCommitteeAttestation[];
|
|
48
|
+
/** Committee-attested checkpoint headers for the range, supplying the L1-verified fee recipient/value. */
|
|
49
|
+
headers: CheckpointHeader[];
|
|
48
50
|
};
|
|
49
51
|
|
|
50
52
|
/** Terminal outcome for a candidate. The promise from `submit()` resolves with one of these. */
|
|
@@ -333,6 +335,7 @@ export class ProofPublishingService {
|
|
|
333
335
|
proof: candidate.proof,
|
|
334
336
|
batchedBlobInputs: candidate.batchedBlobInputs,
|
|
335
337
|
attestations: candidate.attestations,
|
|
338
|
+
headers: candidate.headers,
|
|
336
339
|
// Stop the L1 tx retrying past the candidate's submission-window deadline.
|
|
337
340
|
deadline: candidate.deadline,
|
|
338
341
|
};
|
|
@@ -2,19 +2,17 @@ import { BatchedBlob, getEthBlobEvaluationInputs } from '@aztec/blob-lib';
|
|
|
2
2
|
import { MAX_CHECKPOINTS_PER_EPOCH } from '@aztec/constants';
|
|
3
3
|
import type { RollupContract, ViemCommitteeAttestation } from '@aztec/ethereum/contracts';
|
|
4
4
|
import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
|
|
5
|
-
import { makeTuple } from '@aztec/foundation/array';
|
|
6
5
|
import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
|
|
7
6
|
import { areArraysEqual } from '@aztec/foundation/collection';
|
|
8
7
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
9
8
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
10
9
|
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
11
|
-
import type { Tuple } from '@aztec/foundation/serialize';
|
|
12
10
|
import { Timer } from '@aztec/foundation/timer';
|
|
13
11
|
import { RollupAbi } from '@aztec/l1-artifacts';
|
|
14
12
|
import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
|
|
15
13
|
import { CommitteeAttestation, CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
|
|
16
14
|
import type { Proof } from '@aztec/stdlib/proofs';
|
|
17
|
-
import type {
|
|
15
|
+
import type { CheckpointHeader, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
|
|
18
16
|
import type { L1PublishProofStats } from '@aztec/stdlib/stats';
|
|
19
17
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
20
18
|
|
|
@@ -31,7 +29,7 @@ export type L1SubmitEpochProofArgs = {
|
|
|
31
29
|
endTimestamp: Fr;
|
|
32
30
|
outHash: Fr;
|
|
33
31
|
proverId: Fr;
|
|
34
|
-
|
|
32
|
+
headers: CheckpointHeader[];
|
|
35
33
|
proof: Proof;
|
|
36
34
|
};
|
|
37
35
|
|
|
@@ -78,6 +76,7 @@ export class ProverNodePublisher {
|
|
|
78
76
|
proof: Proof;
|
|
79
77
|
batchedBlobInputs: BatchedBlob;
|
|
80
78
|
attestations: ViemCommitteeAttestation[];
|
|
79
|
+
headers: CheckpointHeader[];
|
|
81
80
|
/** Wall-clock deadline (proof-submission window end) past which the L1 tx should stop retrying. */
|
|
82
81
|
deadline?: Date;
|
|
83
82
|
}): Promise<boolean> {
|
|
@@ -134,6 +133,7 @@ export class ProverNodePublisher {
|
|
|
134
133
|
proof: Proof;
|
|
135
134
|
batchedBlobInputs: BatchedBlob;
|
|
136
135
|
attestations: ViemCommitteeAttestation[];
|
|
136
|
+
headers: CheckpointHeader[];
|
|
137
137
|
}) {
|
|
138
138
|
const { fromCheckpoint, toCheckpoint, publicInputs, batchedBlobInputs } = args;
|
|
139
139
|
|
|
@@ -207,6 +207,7 @@ export class ProverNodePublisher {
|
|
|
207
207
|
proof: Proof;
|
|
208
208
|
batchedBlobInputs: BatchedBlob;
|
|
209
209
|
attestations: ViemCommitteeAttestation[];
|
|
210
|
+
headers: CheckpointHeader[];
|
|
210
211
|
}): Promise<void> {
|
|
211
212
|
const { epochNumber, fromCheckpoint, toCheckpoint } = args;
|
|
212
213
|
|
|
@@ -254,6 +255,7 @@ export class ProverNodePublisher {
|
|
|
254
255
|
proof: Proof;
|
|
255
256
|
batchedBlobInputs: BatchedBlob;
|
|
256
257
|
attestations: ViemCommitteeAttestation[];
|
|
258
|
+
headers: CheckpointHeader[];
|
|
257
259
|
}): Hex {
|
|
258
260
|
return encodeFunctionData({
|
|
259
261
|
abi: RollupAbi,
|
|
@@ -270,6 +272,7 @@ export class ProverNodePublisher {
|
|
|
270
272
|
proof: Proof;
|
|
271
273
|
batchedBlobInputs: BatchedBlob;
|
|
272
274
|
attestations: ViemCommitteeAttestation[];
|
|
275
|
+
headers: CheckpointHeader[];
|
|
273
276
|
}): Promise<TransactionReceipt | undefined> {
|
|
274
277
|
const txArgs = [this.getSubmitEpochProofArgs(args)] as const;
|
|
275
278
|
|
|
@@ -316,6 +319,7 @@ export class ProverNodePublisher {
|
|
|
316
319
|
publicInputs: RootRollupPublicInputs;
|
|
317
320
|
batchedBlobInputs: BatchedBlob;
|
|
318
321
|
attestations: ViemCommitteeAttestation[];
|
|
322
|
+
headers: CheckpointHeader[];
|
|
319
323
|
}) {
|
|
320
324
|
// Returns arguments for EpochProofLib.sol -> getEpochProofPublicInputs()
|
|
321
325
|
return [
|
|
@@ -327,11 +331,7 @@ export class ProverNodePublisher {
|
|
|
327
331
|
outHash: args.publicInputs.outHash.toString(),
|
|
328
332
|
proverId: EthAddress.fromField(args.publicInputs.constants.proverId).toString(),
|
|
329
333
|
} /*_args*/,
|
|
330
|
-
|
|
331
|
-
i % 2 === 0
|
|
332
|
-
? args.publicInputs.fees[i / 2].recipient.toField().toString()
|
|
333
|
-
: args.publicInputs.fees[(i - 1) / 2].value.toString(),
|
|
334
|
-
) /*_fees*/,
|
|
334
|
+
args.headers.map(header => header.toViem()) /*_headers*/,
|
|
335
335
|
getEthBlobEvaluationInputs(args.batchedBlobInputs) /*_blobPublicInputs*/,
|
|
336
336
|
] as const;
|
|
337
337
|
}
|
|
@@ -343,6 +343,7 @@ export class ProverNodePublisher {
|
|
|
343
343
|
proof: Proof;
|
|
344
344
|
batchedBlobInputs: BatchedBlob;
|
|
345
345
|
attestations: ViemCommitteeAttestation[];
|
|
346
|
+
headers: CheckpointHeader[];
|
|
346
347
|
}) {
|
|
347
348
|
// Returns arguments for EpochProofLib.sol -> submitEpochRootProof()
|
|
348
349
|
const proofHex: Hex = `0x${args.proof.withoutPublicInputs().toString('hex')}`;
|
|
@@ -351,7 +352,7 @@ export class ProverNodePublisher {
|
|
|
351
352
|
start: argsArray[0],
|
|
352
353
|
end: argsArray[1],
|
|
353
354
|
args: argsArray[2],
|
|
354
|
-
|
|
355
|
+
headers: argsArray[3],
|
|
355
356
|
attestations: CommitteeAttestationsAndSigners.packAttestations(
|
|
356
357
|
args.attestations.map(a => CommitteeAttestation.fromViem(a)),
|
|
357
358
|
),
|