@aztec/prover-node 0.0.1-commit.993d240 → 0.0.1-commit.9a89641
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +572 -0
- package/dest/actions/rerun-epoch-proving-job.d.ts +12 -3
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +240 -20
- package/dest/checkpoint-store.d.ts +95 -0
- package/dest/checkpoint-store.d.ts.map +1 -0
- package/dest/checkpoint-store.js +178 -0
- package/dest/config.d.ts +3 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +8 -1
- package/dest/factory.d.ts +4 -1
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +2 -6
- package/dest/index.d.ts +2 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/job/checkpoint-prover.d.ts +165 -0
- package/dest/job/checkpoint-prover.d.ts.map +1 -0
- package/dest/job/checkpoint-prover.js +405 -0
- package/dest/job/epoch-session.d.ts +160 -0
- package/dest/job/epoch-session.d.ts.map +1 -0
- package/dest/job/epoch-session.js +744 -0
- package/dest/job/top-tree-job.d.ts +82 -0
- package/dest/job/top-tree-job.d.ts.map +1 -0
- package/dest/job/top-tree-job.js +152 -0
- package/dest/metrics.d.ts +25 -8
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +64 -14
- package/dest/proof-publishing-service.d.ts +161 -0
- package/dest/proof-publishing-service.d.ts.map +1 -0
- package/dest/proof-publishing-service.js +335 -0
- package/dest/prover-node-publisher.d.ts +10 -19
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +43 -99
- package/dest/prover-node.d.ts +133 -65
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +531 -212
- package/dest/prover-publisher-factory.d.ts +3 -1
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +1 -0
- package/dest/session-manager.d.ts +158 -0
- package/dest/session-manager.d.ts.map +1 -0
- package/dest/session-manager.js +492 -0
- package/dest/test/index.d.ts +7 -6
- package/dest/test/index.d.ts.map +1 -1
- package/package.json +24 -23
- package/src/actions/rerun-epoch-proving-job.ts +177 -29
- package/src/checkpoint-store.ts +212 -0
- package/src/config.ts +12 -1
- package/src/factory.ts +5 -8
- package/src/index.ts +1 -0
- package/src/job/checkpoint-prover.ts +538 -0
- package/src/job/epoch-session.ts +462 -0
- package/src/job/top-tree-job.ts +227 -0
- package/src/metrics.ts +65 -23
- package/src/proof-publishing-service.ts +427 -0
- package/src/prover-node-publisher.ts +58 -123
- package/src/prover-node.ts +618 -233
- package/src/prover-publisher-factory.ts +3 -0
- package/src/session-manager.ts +592 -0
- package/src/test/index.ts +6 -6
- package/dest/job/epoch-proving-job.d.ts +0 -67
- package/dest/job/epoch-proving-job.d.ts.map +0 -1
- package/dest/job/epoch-proving-job.js +0 -912
- package/src/job/epoch-proving-job.ts +0 -531
|
@@ -0,0 +1,492 @@
|
|
|
1
|
+
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
2
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
3
|
+
import { SerialQueue } from '@aztec/foundation/queue';
|
|
4
|
+
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
5
|
+
import { getEpochAtSlot, getProofSubmissionDeadlineTimestamp, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
|
|
6
|
+
import { CheckpointProver } from './job/checkpoint-prover.js';
|
|
7
|
+
import { EpochSession, specKey } from './job/epoch-session.js';
|
|
8
|
+
/**
|
|
9
|
+
* Owns the lifecycle of every `EpochSession`. Each L2BlockStream event and periodic tick
|
|
10
|
+
* arrives via a dedicated entry point (`onCheckpointAdded`, `onPrune`, `onTick`, etc.) which
|
|
11
|
+
* schedules a `reconcile(trigger)` on a serial queue. Reconcile walks both session
|
|
12
|
+
* maps, cancels any session whose canonical content has shifted, re-creates it with
|
|
13
|
+
* the same spec but new content, and opens fresh full sessions for any epoch implicated
|
|
14
|
+
* by the trigger.
|
|
15
|
+
*/ export class SessionManager {
|
|
16
|
+
deps;
|
|
17
|
+
log;
|
|
18
|
+
fullSessions;
|
|
19
|
+
partialSessions;
|
|
20
|
+
/**
|
|
21
|
+
* Serialises every reconcile call. The trigger sources (L2BlockStream events, the
|
|
22
|
+
* periodic tick, JSON-RPC `startProof`) run independently, so without this queue two
|
|
23
|
+
* reconciles could interleave on the `await session.cancel(...)` step and orphan a
|
|
24
|
+
* freshly-constructed session.
|
|
25
|
+
*/ reconcileQueue;
|
|
26
|
+
/** Cached L1 constants, populated on first read. */ cachedL1Constants;
|
|
27
|
+
/** Test-only hooks applied to every session this manager constructs. */ sessionHooks;
|
|
28
|
+
/** Periodic tick that nudges reconcile to pick up newly-complete epochs. Started by `start()`. */ epochTicker;
|
|
29
|
+
constructor(deps){
|
|
30
|
+
this.deps = deps;
|
|
31
|
+
this.fullSessions = new Map();
|
|
32
|
+
this.partialSessions = new Map();
|
|
33
|
+
this.reconcileQueue = new SerialQueue();
|
|
34
|
+
this.log = createLogger('prover-node:session-manager', deps.bindings);
|
|
35
|
+
this.reconcileQueue.start();
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Starts the periodic tick. Separated from the constructor so tests can drive `onTick()`
|
|
39
|
+
* manually without the background ticker interleaving. Idempotent.
|
|
40
|
+
*/ start() {
|
|
41
|
+
if (this.epochTicker) {
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
this.epochTicker = new RunningPromise(()=>this.onTick(), this.log, this.deps.config.tickIntervalMs);
|
|
45
|
+
this.epochTicker.start();
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Installs hooks applied to every session constructed from now on. Used by the e2e
|
|
49
|
+
* harness to interpose around top-tree proving (gate it, override it, observe it)
|
|
50
|
+
* without monkey-patching the orchestrator factory.
|
|
51
|
+
*/ setSessionHooks(hooks) {
|
|
52
|
+
this.sessionHooks = hooks;
|
|
53
|
+
}
|
|
54
|
+
// ---------------- read-only views ----------------
|
|
55
|
+
/** Every live (non-terminal) session. */ allSessions() {
|
|
56
|
+
return [
|
|
57
|
+
...this.fullSessions.values(),
|
|
58
|
+
...this.partialSessions.values()
|
|
59
|
+
];
|
|
60
|
+
}
|
|
61
|
+
/** Returns the full session for `epoch`, if any. */ getFullSession(epoch) {
|
|
62
|
+
return this.fullSessions.get(epoch);
|
|
63
|
+
}
|
|
64
|
+
/** Returns the partial session for `spec`, if any. */ getPartialSession(spec) {
|
|
65
|
+
return this.partialSessions.get(specKey(spec));
|
|
66
|
+
}
|
|
67
|
+
/** Observability summary used by the prover-node API. */ getJobs() {
|
|
68
|
+
return this.allSessions().map((s)=>({
|
|
69
|
+
uuid: s.getId(),
|
|
70
|
+
status: s.getState(),
|
|
71
|
+
epochNumber: s.getEpochNumber()
|
|
72
|
+
}));
|
|
73
|
+
}
|
|
74
|
+
// ---------------- event entry points ----------------
|
|
75
|
+
/** Called by ProverNode after a chain-checkpointed event has been added to the store. */ onCheckpointAdded(epoch) {
|
|
76
|
+
return this.scheduleReconcile({
|
|
77
|
+
kind: 'checkpoint',
|
|
78
|
+
epoch
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
/** Called by ProverNode after a chain-pruned event has flipped store provers to pruned. */ onPrune(affectedEpochs) {
|
|
82
|
+
return this.scheduleReconcile({
|
|
83
|
+
kind: 'prune',
|
|
84
|
+
affectedEpochs
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Called periodically by ProverNode's ticker. Picks up epochs that have become complete
|
|
89
|
+
* by time without a fresh checkpoint event (e.g. the epoch's last slots are empty), and
|
|
90
|
+
* advances to the next epoch once the previous one is proven on L1.
|
|
91
|
+
*/ onTick() {
|
|
92
|
+
return this.scheduleReconcile({
|
|
93
|
+
kind: 'tick'
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
// ---------------- public API ----------------
|
|
97
|
+
/**
|
|
98
|
+
* Schedules a proof attempt for the supplied epoch and returns the job id without waiting for
|
|
99
|
+
* the proof to complete — proving can far outlast an HTTP request, so callers poll `getJobs()`
|
|
100
|
+
* for the outcome. Every session — full or partial — begins at the epoch's first slot; the
|
|
101
|
+
* partial's spec stops at the last canonical slot, while the full's stops at the epoch's last
|
|
102
|
+
* slot. Dedupes against any existing session covering the same range, returning its id.
|
|
103
|
+
*/ async startProof(epoch) {
|
|
104
|
+
const canonical = await this.deps.checkpointStore.listForEpoch(epoch);
|
|
105
|
+
if (canonical.length === 0) {
|
|
106
|
+
throw new EmptyEpochError(epoch);
|
|
107
|
+
}
|
|
108
|
+
// Don't re-prove an epoch the L1 proven chain already encompasses — it was already proven
|
|
109
|
+
// (possibly by another prover node), so a fresh proof would be wasted work.
|
|
110
|
+
if (await this.isProvenChainEncompassing(canonical)) {
|
|
111
|
+
throw new EpochAlreadyProvenError(epoch);
|
|
112
|
+
}
|
|
113
|
+
const l1Constants = await this.getL1Constants();
|
|
114
|
+
const [fromSlot] = getSlotRangeForEpoch(epoch, l1Constants);
|
|
115
|
+
const toSlot = canonical[canonical.length - 1].slotNumber;
|
|
116
|
+
const spec = {
|
|
117
|
+
kind: 'partial',
|
|
118
|
+
epochNumber: epoch,
|
|
119
|
+
fromSlot,
|
|
120
|
+
toSlot
|
|
121
|
+
};
|
|
122
|
+
// Reuse a session already covering this exact range rather than scheduling a duplicate.
|
|
123
|
+
const existingFull = this.getFullSession(epoch);
|
|
124
|
+
if (existingFull && !existingFull.isTerminal() && existingFull.getSpec().fromSlot === fromSlot && existingFull.getSpec().toSlot === toSlot) {
|
|
125
|
+
return existingFull.getId();
|
|
126
|
+
}
|
|
127
|
+
const existingPartial = this.getPartialSession(spec);
|
|
128
|
+
if (existingPartial && !existingPartial.isTerminal()) {
|
|
129
|
+
return existingPartial.getId();
|
|
130
|
+
}
|
|
131
|
+
await this.scheduleReconcile({
|
|
132
|
+
kind: 'start-proof',
|
|
133
|
+
spec
|
|
134
|
+
});
|
|
135
|
+
const created = this.getPartialSession(spec);
|
|
136
|
+
if (!created) {
|
|
137
|
+
throw new Error(`Failed to schedule partial proof for epoch ${epoch}`);
|
|
138
|
+
}
|
|
139
|
+
return created.getId();
|
|
140
|
+
}
|
|
141
|
+
/** Stops the tick, drains the reconcile queue, and cancels every live session. */ async stop() {
|
|
142
|
+
await this.epochTicker?.stop();
|
|
143
|
+
await this.reconcileQueue.cancel();
|
|
144
|
+
const sessions = this.allSessions();
|
|
145
|
+
// A clean shutdown is just a restart, so preserve the in-flight broker jobs (abortJobs: false)
|
|
146
|
+
// for the restarted node to reuse rather than re-proving the epoch from scratch.
|
|
147
|
+
await Promise.allSettled(sessions.map((s)=>s.cancel('prover-node stopping', {
|
|
148
|
+
abortJobs: false
|
|
149
|
+
})));
|
|
150
|
+
}
|
|
151
|
+
// ---------------- reconcile ----------------
|
|
152
|
+
scheduleReconcile(trigger) {
|
|
153
|
+
return this.reconcileQueue.put(()=>this.reconcile(trigger));
|
|
154
|
+
}
|
|
155
|
+
async reconcile(trigger) {
|
|
156
|
+
this.log.debug(`Reconciling`, {
|
|
157
|
+
trigger
|
|
158
|
+
});
|
|
159
|
+
this.recreateInvalidSessions();
|
|
160
|
+
const implicatedEpochs = await this.epochsForTrigger(trigger);
|
|
161
|
+
for (const epoch of implicatedEpochs){
|
|
162
|
+
await this.openFullSessionIfReady(epoch);
|
|
163
|
+
}
|
|
164
|
+
if (trigger.kind === 'start-proof') {
|
|
165
|
+
this.openPartialSession(trigger.spec);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
recreateInvalidSessions() {
|
|
169
|
+
for (const [key, session] of Array.from(this.fullSessions.entries())){
|
|
170
|
+
const canonical = this.checkpointsForSpec(session.getSpec());
|
|
171
|
+
const contentChanged = !this.checkpointsMatch(session.getCheckpoints(), canonical);
|
|
172
|
+
if (session.isTerminal()) {
|
|
173
|
+
// A full session that failed on its own account is retained as a "do not re-prove" marker while
|
|
174
|
+
// its content is unchanged — this is what stops the tick re-proving a deterministically-failing
|
|
175
|
+
// epoch. When the content changes (a re-add), it is replaced so the epoch retries over the new
|
|
176
|
+
// provers. Any other terminal full session is simply dropped.
|
|
177
|
+
if (session.hasFailed() && !contentChanged) {
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
this.fullSessions.delete(key);
|
|
181
|
+
if (contentChanged && this.canBuildOver(canonical)) {
|
|
182
|
+
const newSession = this.constructSession(session.getSpec(), canonical);
|
|
183
|
+
this.fullSessions.set(key, newSession);
|
|
184
|
+
void this.runSession(newSession);
|
|
185
|
+
}
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
if (contentChanged) {
|
|
189
|
+
this.fireAndForgetCancel(session, 'canonical content changed');
|
|
190
|
+
this.fullSessions.delete(key);
|
|
191
|
+
if (this.canBuildOver(canonical)) {
|
|
192
|
+
const newSession = this.constructSession(session.getSpec(), canonical);
|
|
193
|
+
this.fullSessions.set(key, newSession);
|
|
194
|
+
void this.runSession(newSession);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
for (const [key, session] of Array.from(this.partialSessions.entries())){
|
|
199
|
+
if (session.isTerminal()) {
|
|
200
|
+
this.partialSessions.delete(key);
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
const canonical = this.checkpointsForSpec(session.getSpec());
|
|
204
|
+
if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) {
|
|
205
|
+
this.fireAndForgetCancel(session, 'canonical content changed');
|
|
206
|
+
this.partialSessions.delete(key);
|
|
207
|
+
if (this.canBuildOver(canonical)) {
|
|
208
|
+
const newSession = this.constructSession(session.getSpec(), canonical);
|
|
209
|
+
this.partialSessions.set(key, newSession);
|
|
210
|
+
void this.runSession(newSession);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
/** A session may be built over a checkpoint set only when it is non-empty and contains no failed prover. */ canBuildOver(canonical) {
|
|
216
|
+
return canonical.length > 0 && !this.hasFailedProver(canonical);
|
|
217
|
+
}
|
|
218
|
+
async openFullSessionIfReady(epoch) {
|
|
219
|
+
// A session present here already covers the epoch: either live, or a retained genuinely-failed
|
|
220
|
+
// session kept by `recreateInvalidSessions` as a "do not re-prove" marker. Either way, don't open
|
|
221
|
+
// another — the retained-failed one is replaced only when its canonical content changes.
|
|
222
|
+
if (this.fullSessions.has(epoch)) {
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
if (this.atMaxSessionLimit()) {
|
|
226
|
+
this.log.debug(`Skipping full-session open for epoch ${epoch}: max pending jobs reached`);
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
if (!await this.deps.l2BlockSource.isEpochComplete(epoch)) {
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
const l1Constants = await this.getL1Constants();
|
|
233
|
+
const archiverCps = await this.deps.l2BlockSource.getCheckpoints({
|
|
234
|
+
epoch
|
|
235
|
+
});
|
|
236
|
+
if (archiverCps.length === 0) {
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, l1Constants);
|
|
240
|
+
const canonical = this.deps.checkpointStore.listInSlotRange(fromSlot, toSlot);
|
|
241
|
+
if (!this.archiverFullyCovered(archiverCps, canonical)) {
|
|
242
|
+
this.log.debug(`Skipping full-session open for epoch ${epoch}: archiver checkpoints not all in store`, {
|
|
243
|
+
archiverCount: archiverCps.length,
|
|
244
|
+
storeCount: canonical.length
|
|
245
|
+
});
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
if (this.hasFailedProver(canonical)) {
|
|
249
|
+
// A checkpoint prover in the set has failed (a sub-tree fault or a prune-induced fork fault), so a
|
|
250
|
+
// session over it would fail immediately. Don't re-create it every tick — it recovers when a
|
|
251
|
+
// prune/re-add replaces the failed prover with a fresh one, or fails for good at expiry.
|
|
252
|
+
this.log.debug(`Skipping full-session open for epoch ${epoch}: a checkpoint prover has failed`, {
|
|
253
|
+
epoch
|
|
254
|
+
});
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
const spec = {
|
|
258
|
+
kind: 'full',
|
|
259
|
+
epochNumber: epoch,
|
|
260
|
+
fromSlot,
|
|
261
|
+
toSlot
|
|
262
|
+
};
|
|
263
|
+
const session = this.constructSession(spec, canonical);
|
|
264
|
+
this.fullSessions.set(epoch, session);
|
|
265
|
+
void this.runSession(session);
|
|
266
|
+
}
|
|
267
|
+
openPartialSession(spec) {
|
|
268
|
+
const canonical = this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot);
|
|
269
|
+
if (canonical.length === 0 || this.hasFailedProver(canonical)) {
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
// Reuse a live partial session for this epoch whose checkpoint set already matches the
|
|
273
|
+
// canonical content — e.g. a repeated `startProof` with no new checkpoints mined since the
|
|
274
|
+
// last one. Reconstructing would re-prove identical content and burn a pending-job slot.
|
|
275
|
+
const existing = Array.from(this.partialSessions.values()).find((s)=>s.getSpec().epochNumber === spec.epochNumber && !s.isTerminal() && this.checkpointsMatch(s.getCheckpoints(), canonical));
|
|
276
|
+
if (existing) {
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
if (this.atMaxSessionLimit()) {
|
|
280
|
+
throw new Error(`Maximum pending proving jobs ${this.deps.config.maxPendingJobs} reached.`);
|
|
281
|
+
}
|
|
282
|
+
const session = this.constructSession(spec, canonical);
|
|
283
|
+
this.partialSessions.set(specKey(spec), session);
|
|
284
|
+
void this.runSession(session);
|
|
285
|
+
}
|
|
286
|
+
// ---------------- session construction ----------------
|
|
287
|
+
constructSession(spec, checkpoints) {
|
|
288
|
+
return this.doConstructSession(spec, checkpoints, this.buildSessionDeps(spec.epochNumber), this.sessionHooks);
|
|
289
|
+
}
|
|
290
|
+
/** Extracted for test override. */ doConstructSession(spec, checkpoints, sessionDeps, hooks) {
|
|
291
|
+
return new EpochSession(spec, checkpoints, {
|
|
292
|
+
...sessionDeps,
|
|
293
|
+
hooks
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
buildSessionDeps(epochNumber) {
|
|
297
|
+
const config = {
|
|
298
|
+
finalizationDelayMs: this.deps.config.finalizationDelayMs
|
|
299
|
+
};
|
|
300
|
+
return {
|
|
301
|
+
proverFactory: this.deps.proverFactory,
|
|
302
|
+
proverId: this.deps.proverId,
|
|
303
|
+
publishingService: this.deps.publishingService,
|
|
304
|
+
metrics: this.deps.metrics,
|
|
305
|
+
dateProvider: this.deps.dateProvider,
|
|
306
|
+
deadline: this.computeDeadline(epochNumber),
|
|
307
|
+
config,
|
|
308
|
+
bindings: this.deps.bindings
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
computeDeadline(epochNumber) {
|
|
312
|
+
if (!this.cachedL1Constants) {
|
|
313
|
+
return undefined;
|
|
314
|
+
}
|
|
315
|
+
const ts = getProofSubmissionDeadlineTimestamp(epochNumber, this.cachedL1Constants);
|
|
316
|
+
return new Date(Number(ts) * 1000);
|
|
317
|
+
}
|
|
318
|
+
async runSession(session) {
|
|
319
|
+
// A reconcile may have cancelled this session before it starts (content-change
|
|
320
|
+
// recreation). Don't proceed — start() would build a TopTreeJob that should never run.
|
|
321
|
+
if (session.isTerminal()) {
|
|
322
|
+
this.log.debug(`Skipping start for ${session.getId()}: already terminal (${session.getState()})`);
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
const state = await session.start();
|
|
326
|
+
this.log.info(`Session ${session.getId()} exited with state ${state}`);
|
|
327
|
+
// A full session that failed on its own account (top-tree/submit failed with every prover healthy)
|
|
328
|
+
// is a genuine, race-free failure: upload its post-mortem once. `recreateInvalidSessions` retains
|
|
329
|
+
// the terminal session so this fires exactly once (it is never re-run over the same content). A
|
|
330
|
+
// `stopped` session (a prover under it failed) is not uploaded — it may be a prune, and recovers on
|
|
331
|
+
// re-add.
|
|
332
|
+
if (session.getKind() === 'full' && session.hasFailed() && this.deps.onSessionFailed) {
|
|
333
|
+
try {
|
|
334
|
+
await this.deps.onSessionFailed(session);
|
|
335
|
+
} catch (err) {
|
|
336
|
+
this.log.error(`Error in onSessionFailed callback for epoch ${session.getEpochNumber()}`, err);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Builds the EpochProvingJobData snapshot for a post-mortem failure upload from a set of
|
|
342
|
+
* checkpoint provers. Includes every checkpoint regardless of whether sub-tree proving
|
|
343
|
+
* completed — partial state is still useful for post-mortem analysis.
|
|
344
|
+
*/ static async buildProvingData(checkpoints) {
|
|
345
|
+
if (checkpoints.length === 0) {
|
|
346
|
+
throw new Error('Cannot build proving data from an empty checkpoint set');
|
|
347
|
+
}
|
|
348
|
+
// Provers no longer cache their txs; re-fetch each checkpoint's txs from the tx pool (concurrently)
|
|
349
|
+
// for the snapshot. The pool retains them past the proving window (A-1274), so this is durable.
|
|
350
|
+
const perCheckpoint = await Promise.all(checkpoints.map(async (c)=>({
|
|
351
|
+
c,
|
|
352
|
+
txs: await c.getTxsForUpload()
|
|
353
|
+
})));
|
|
354
|
+
const txs = new Map();
|
|
355
|
+
const l1ToL2Messages = {};
|
|
356
|
+
for (const { c, txs: checkpointTxs } of perCheckpoint){
|
|
357
|
+
for (const [hash, tx] of checkpointTxs){
|
|
358
|
+
txs.set(hash, tx);
|
|
359
|
+
}
|
|
360
|
+
l1ToL2Messages[c.checkpoint.number] = c.l1ToL2Messages;
|
|
361
|
+
}
|
|
362
|
+
return {
|
|
363
|
+
epochNumber: checkpoints[0].epochNumber,
|
|
364
|
+
checkpoints: checkpoints.map((c)=>c.checkpoint),
|
|
365
|
+
txs,
|
|
366
|
+
l1ToL2Messages,
|
|
367
|
+
previousBlockHeader: checkpoints[0].previousBlockHeader,
|
|
368
|
+
attestations: []
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
// ---------------- reconcile helpers ----------------
|
|
372
|
+
atMaxSessionLimit() {
|
|
373
|
+
const { maxPendingJobs: max } = this.deps.config;
|
|
374
|
+
if (!max || max <= 0) {
|
|
375
|
+
return false;
|
|
376
|
+
}
|
|
377
|
+
const live = this.allSessions().filter((s)=>!s.isTerminal()).length;
|
|
378
|
+
return live >= max;
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* Maps a reconcile trigger to the epochs whose full session should be (re)opened.
|
|
382
|
+
*
|
|
383
|
+
* The periodic `tick` returns the next unproven epoch every time; it does not track prior attempts.
|
|
384
|
+
* `openFullSessionIfReady` is what keeps this from re-proving a doomed epoch: it refuses to build a
|
|
385
|
+
* session when any checkpoint prover in the set has failed, so a stuck epoch is cheaply skipped each
|
|
386
|
+
* tick rather than re-proved. `checkpoint` and `prune` fire when an epoch's canonical content changes
|
|
387
|
+
* (a checkpoint arrives, or a reorg prunes/replaces one) — which is what installs a fresh prover in
|
|
388
|
+
* place of a failed one, letting the next open succeed and recovering a pruned-then-re-added epoch.
|
|
389
|
+
*/ async epochsForTrigger(trigger) {
|
|
390
|
+
switch(trigger.kind){
|
|
391
|
+
case 'checkpoint':
|
|
392
|
+
return [
|
|
393
|
+
trigger.epoch
|
|
394
|
+
];
|
|
395
|
+
case 'prune':
|
|
396
|
+
return trigger.affectedEpochs;
|
|
397
|
+
case 'tick':
|
|
398
|
+
{
|
|
399
|
+
const epoch = await this.nextUnprovenEpoch();
|
|
400
|
+
return epoch === undefined ? [] : [
|
|
401
|
+
epoch
|
|
402
|
+
];
|
|
403
|
+
}
|
|
404
|
+
case 'start-proof':
|
|
405
|
+
return [];
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* The next epoch to prove: the epoch containing the first block after the proven tip.
|
|
410
|
+
* Returns undefined when that block has not been mined yet (e.g. nothing new to prove).
|
|
411
|
+
* Subsequent ticks advance only once the chain's proven height moves forward, so epochs
|
|
412
|
+
* are proven in order rather than all at once.
|
|
413
|
+
*/ async nextUnprovenEpoch() {
|
|
414
|
+
const lastProven = await this.deps.l2BlockSource.getBlockNumber({
|
|
415
|
+
tag: 'proven'
|
|
416
|
+
}) ?? BlockNumber.ZERO;
|
|
417
|
+
const firstToProve = BlockNumber(lastProven + 1);
|
|
418
|
+
const header = (await this.deps.l2BlockSource.getBlockData({
|
|
419
|
+
number: firstToProve
|
|
420
|
+
}))?.header;
|
|
421
|
+
if (!header) {
|
|
422
|
+
return undefined;
|
|
423
|
+
}
|
|
424
|
+
return getEpochAtSlot(header.getSlot(), await this.getL1Constants());
|
|
425
|
+
}
|
|
426
|
+
checkpointsForSpec(spec) {
|
|
427
|
+
return this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot);
|
|
428
|
+
}
|
|
429
|
+
/**
|
|
430
|
+
* True if any prover in the set has failed. The epoch cannot be proven over a failed prover (it can
|
|
431
|
+
* never produce its block proofs), so a session must not be built or rebuilt over it until a prune/re-add
|
|
432
|
+
* has replaced it with a fresh prover.
|
|
433
|
+
*/ hasFailedProver(checkpoints) {
|
|
434
|
+
return checkpoints.some((c)=>c.isFailed());
|
|
435
|
+
}
|
|
436
|
+
fireAndForgetCancel(session, reason) {
|
|
437
|
+
void session.cancel(reason).catch((err)=>this.log.warn(`Error cancelling session ${session.getId()}`, err));
|
|
438
|
+
}
|
|
439
|
+
checkpointsMatch(a, b) {
|
|
440
|
+
if (a.length !== b.length) {
|
|
441
|
+
return false;
|
|
442
|
+
}
|
|
443
|
+
for(let i = 0; i < a.length; i++){
|
|
444
|
+
if (a[i].id !== b[i].id || a[i].isCancelled()) {
|
|
445
|
+
return false;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
return true;
|
|
449
|
+
}
|
|
450
|
+
archiverFullyCovered(archiverCps, storeCps) {
|
|
451
|
+
if (storeCps.length < archiverCps.length) {
|
|
452
|
+
return false;
|
|
453
|
+
}
|
|
454
|
+
// Compare by content-addressed id (number, slot, archive root) rather than checkpoint number:
|
|
455
|
+
// a reorg can keep the number while changing the checkpoint's post-state archive root.
|
|
456
|
+
const storeIds = new Set(storeCps.map((p)=>p.id));
|
|
457
|
+
return archiverCps.every((cp)=>storeIds.has(CheckpointProver.idFor(cp.checkpoint)));
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Returns true if the L1 proven tip already covers every canonical checkpoint in the set — i.e.
|
|
461
|
+
* the epoch has already been fully proven, so there is no point starting a new proof for it.
|
|
462
|
+
* Conservatively returns false when nothing is proven yet.
|
|
463
|
+
*/ async isProvenChainEncompassing(canonical) {
|
|
464
|
+
const provenBlock = await this.deps.l2BlockSource.getBlockNumber({
|
|
465
|
+
tag: 'proven'
|
|
466
|
+
});
|
|
467
|
+
if (!provenBlock || provenBlock <= 0) {
|
|
468
|
+
return false;
|
|
469
|
+
}
|
|
470
|
+
const lastCheckpoint = canonical[canonical.length - 1].checkpoint;
|
|
471
|
+
const lastBlock = lastCheckpoint.blocks[lastCheckpoint.blocks.length - 1].number;
|
|
472
|
+
return provenBlock >= lastBlock;
|
|
473
|
+
}
|
|
474
|
+
async getL1Constants() {
|
|
475
|
+
if (!this.cachedL1Constants) {
|
|
476
|
+
this.cachedL1Constants = await this.deps.l2BlockSource.getL1Constants();
|
|
477
|
+
}
|
|
478
|
+
return this.cachedL1Constants;
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
class EmptyEpochError extends Error {
|
|
482
|
+
constructor(epochNumber){
|
|
483
|
+
super(`No blocks found for epoch ${epochNumber}`);
|
|
484
|
+
this.name = 'EmptyEpochError';
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
class EpochAlreadyProvenError extends Error {
|
|
488
|
+
constructor(epochNumber){
|
|
489
|
+
super(`Epoch ${epochNumber} is already proven on L1`);
|
|
490
|
+
this.name = 'EpochAlreadyProvenError';
|
|
491
|
+
}
|
|
492
|
+
}
|
package/dest/test/index.d.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
import type { EpochProverFactory } from '@aztec/prover-client';
|
|
1
2
|
import type { EpochProverManager } from '@aztec/stdlib/interfaces/server';
|
|
2
|
-
import type {
|
|
3
|
-
import type { ProverNodePublisher } from '../prover-node-publisher.js';
|
|
3
|
+
import type { ProofPublishingService } from '../proof-publishing-service.js';
|
|
4
4
|
import { ProverNode } from '../prover-node.js';
|
|
5
|
+
import type { SessionManager } from '../session-manager.js';
|
|
5
6
|
declare abstract class TestProverNodeClass extends ProverNode {
|
|
6
|
-
prover: EpochProverManager;
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
prover: EpochProverManager & EpochProverFactory;
|
|
8
|
+
publishingService: ProofPublishingService;
|
|
9
|
+
sessionManager: SessionManager;
|
|
9
10
|
}
|
|
10
11
|
export type TestProverNode = TestProverNodeClass;
|
|
11
12
|
export {};
|
|
12
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
13
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy90ZXN0L2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDL0QsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUUxRSxPQUFPLEtBQUssRUFBRSxzQkFBc0IsRUFBRSxNQUFNLGdDQUFnQyxDQUFDO0FBQzdFLE9BQU8sRUFBRSxVQUFVLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQztBQUMvQyxPQUFPLEtBQUssRUFBRSxjQUFjLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUU1RCx1QkFBZSxtQkFBb0IsU0FBUSxVQUFVO0lBQ3BDLE1BQU0sRUFBRSxrQkFBa0IsR0FBRyxrQkFBa0IsQ0FBQztJQUNoRCxpQkFBaUIsRUFBRSxzQkFBc0IsQ0FBQztJQUMxQyxjQUFjLEVBQUUsY0FBYyxDQUFDO0NBQy9DO0FBRUQsTUFBTSxNQUFNLGNBQWMsR0FBRyxtQkFBbUIsQ0FBQyJ9
|
package/dest/test/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/test/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/test/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAE1E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAE5D,uBAAe,mBAAoB,SAAQ,UAAU;IACpC,MAAM,EAAE,kBAAkB,GAAG,kBAAkB,CAAC;IAChD,iBAAiB,EAAE,sBAAsB,CAAC;IAC1C,cAAc,EAAE,cAAc,CAAC;CAC/C;AAED,MAAM,MAAM,cAAc,GAAG,mBAAmB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/prover-node",
|
|
3
|
-
"version": "0.0.1-commit.
|
|
3
|
+
"version": "0.0.1-commit.9a89641",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./dest/index.js",
|
|
@@ -56,28 +56,29 @@
|
|
|
56
56
|
]
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
|
-
"@aztec/archiver": "0.0.1-commit.
|
|
60
|
-
"@aztec/bb-prover": "0.0.1-commit.
|
|
61
|
-
"@aztec/
|
|
62
|
-
"@aztec/blob-
|
|
63
|
-
"@aztec/
|
|
64
|
-
"@aztec/
|
|
65
|
-
"@aztec/
|
|
66
|
-
"@aztec/
|
|
67
|
-
"@aztec/
|
|
68
|
-
"@aztec/
|
|
69
|
-
"@aztec/
|
|
70
|
-
"@aztec/
|
|
71
|
-
"@aztec/node-
|
|
72
|
-
"@aztec/
|
|
73
|
-
"@aztec/
|
|
74
|
-
"@aztec/
|
|
75
|
-
"@aztec/
|
|
76
|
-
"@aztec/
|
|
77
|
-
"@aztec/
|
|
78
|
-
"@aztec/
|
|
79
|
-
"@aztec/
|
|
80
|
-
"@aztec/
|
|
59
|
+
"@aztec/archiver": "0.0.1-commit.9a89641",
|
|
60
|
+
"@aztec/bb-prover": "0.0.1-commit.9a89641",
|
|
61
|
+
"@aztec/bb.js": "6.0.0-nightly.20260807",
|
|
62
|
+
"@aztec/blob-client": "0.0.1-commit.9a89641",
|
|
63
|
+
"@aztec/blob-lib": "0.0.1-commit.9a89641",
|
|
64
|
+
"@aztec/constants": "0.0.1-commit.9a89641",
|
|
65
|
+
"@aztec/epoch-cache": "0.0.1-commit.9a89641",
|
|
66
|
+
"@aztec/ethereum": "0.0.1-commit.9a89641",
|
|
67
|
+
"@aztec/foundation": "0.0.1-commit.9a89641",
|
|
68
|
+
"@aztec/kv-store": "0.0.1-commit.9a89641",
|
|
69
|
+
"@aztec/l1-artifacts": "6.0.0-nightly.20260807",
|
|
70
|
+
"@aztec/native": "0.0.1-commit.9a89641",
|
|
71
|
+
"@aztec/node-keystore": "0.0.1-commit.9a89641",
|
|
72
|
+
"@aztec/node-lib": "0.0.1-commit.9a89641",
|
|
73
|
+
"@aztec/noir-protocol-circuits-types": "0.0.1-commit.9a89641",
|
|
74
|
+
"@aztec/p2p": "0.0.1-commit.9a89641",
|
|
75
|
+
"@aztec/protocol-contracts": "0.0.1-commit.9a89641",
|
|
76
|
+
"@aztec/prover-client": "0.0.1-commit.9a89641",
|
|
77
|
+
"@aztec/sequencer-client": "0.0.1-commit.9a89641",
|
|
78
|
+
"@aztec/simulator": "0.0.1-commit.9a89641",
|
|
79
|
+
"@aztec/stdlib": "0.0.1-commit.9a89641",
|
|
80
|
+
"@aztec/telemetry-client": "0.0.1-commit.9a89641",
|
|
81
|
+
"@aztec/world-state": "0.0.1-commit.9a89641",
|
|
81
82
|
"source-map-support": "^0.5.21",
|
|
82
83
|
"tslib": "^2.4.0",
|
|
83
84
|
"viem": "npm:@aztec/viem@2.38.2"
|