@aztec/prover-node 0.0.1-commit.b655e406 → 0.0.1-commit.b8a057fa

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/README.md +518 -0
  2. package/dest/actions/download-epoch-proving-job.d.ts +4 -4
  3. package/dest/actions/download-epoch-proving-job.js +1 -1
  4. package/dest/actions/index.d.ts +1 -1
  5. package/dest/actions/rerun-epoch-proving-job.d.ts +6 -4
  6. package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
  7. package/dest/actions/rerun-epoch-proving-job.js +191 -28
  8. package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
  9. package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
  10. package/dest/bin/run-failed-epoch.d.ts +1 -1
  11. package/dest/bin/run-failed-epoch.js +6 -5
  12. package/dest/checkpoint-store.d.ts +87 -0
  13. package/dest/checkpoint-store.d.ts.map +1 -0
  14. package/dest/checkpoint-store.js +186 -0
  15. package/dest/config.d.ts +10 -10
  16. package/dest/config.d.ts.map +1 -1
  17. package/dest/config.js +27 -22
  18. package/dest/factory.d.ts +23 -16
  19. package/dest/factory.d.ts.map +1 -1
  20. package/dest/factory.js +49 -68
  21. package/dest/index.d.ts +3 -1
  22. package/dest/index.d.ts.map +1 -1
  23. package/dest/index.js +2 -0
  24. package/dest/job/checkpoint-prover.d.ts +134 -0
  25. package/dest/job/checkpoint-prover.d.ts.map +1 -0
  26. package/dest/job/checkpoint-prover.js +353 -0
  27. package/dest/job/epoch-proving-job-data.d.ts +8 -6
  28. package/dest/job/epoch-proving-job-data.d.ts.map +1 -1
  29. package/dest/job/epoch-proving-job-data.js +25 -18
  30. package/dest/job/epoch-session.d.ts +146 -0
  31. package/dest/job/epoch-session.d.ts.map +1 -0
  32. package/dest/job/epoch-session.js +720 -0
  33. package/dest/job/top-tree-job.d.ts +82 -0
  34. package/dest/job/top-tree-job.d.ts.map +1 -0
  35. package/dest/job/top-tree-job.js +152 -0
  36. package/dest/metrics.d.ts +42 -4
  37. package/dest/metrics.d.ts.map +1 -1
  38. package/dest/metrics.js +128 -99
  39. package/dest/monitors/epoch-monitor.d.ts +3 -2
  40. package/dest/monitors/epoch-monitor.d.ts.map +1 -1
  41. package/dest/monitors/epoch-monitor.js +14 -20
  42. package/dest/monitors/index.d.ts +1 -1
  43. package/dest/proof-publishing-service.d.ts +161 -0
  44. package/dest/proof-publishing-service.d.ts.map +1 -0
  45. package/dest/proof-publishing-service.js +335 -0
  46. package/dest/prover-node-publisher.d.ts +36 -23
  47. package/dest/prover-node-publisher.d.ts.map +1 -1
  48. package/dest/prover-node-publisher.js +244 -98
  49. package/dest/prover-node.d.ts +122 -71
  50. package/dest/prover-node.d.ts.map +1 -1
  51. package/dest/prover-node.js +872 -226
  52. package/dest/prover-publisher-factory.d.ts +12 -6
  53. package/dest/prover-publisher-factory.d.ts.map +1 -1
  54. package/dest/prover-publisher-factory.js +8 -5
  55. package/dest/session-manager.d.ts +158 -0
  56. package/dest/session-manager.d.ts.map +1 -0
  57. package/dest/session-manager.js +452 -0
  58. package/dest/test/index.d.ts +7 -6
  59. package/dest/test/index.d.ts.map +1 -1
  60. package/package.json +28 -25
  61. package/src/actions/download-epoch-proving-job.ts +1 -1
  62. package/src/actions/rerun-epoch-proving-job.ts +120 -30
  63. package/src/actions/upload-epoch-proof-failure.ts +1 -1
  64. package/src/bin/run-failed-epoch.ts +5 -3
  65. package/src/checkpoint-store.ts +218 -0
  66. package/src/config.ts +39 -34
  67. package/src/factory.ts +86 -111
  68. package/src/index.ts +2 -0
  69. package/src/job/checkpoint-prover.ts +468 -0
  70. package/src/job/epoch-proving-job-data.ts +31 -25
  71. package/src/job/epoch-session.ts +436 -0
  72. package/src/job/top-tree-job.ts +227 -0
  73. package/src/metrics.ts +146 -84
  74. package/src/monitors/epoch-monitor.ts +10 -17
  75. package/src/proof-publishing-service.ts +427 -0
  76. package/src/prover-node-publisher.ts +300 -129
  77. package/src/prover-node.ts +569 -254
  78. package/src/prover-publisher-factory.ts +22 -11
  79. package/src/session-manager.ts +552 -0
  80. package/src/test/index.ts +6 -6
  81. package/dest/job/epoch-proving-job.d.ts +0 -69
  82. package/dest/job/epoch-proving-job.d.ts.map +0 -1
  83. package/dest/job/epoch-proving-job.js +0 -354
  84. package/src/job/epoch-proving-job.ts +0 -420
@@ -1,24 +1,409 @@
1
- function _ts_decorate(decorators, target, key, desc) {
2
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
- else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1
+ function applyDecs2203RFactory() {
2
+ function createAddInitializerMethod(initializers, decoratorFinishedRef) {
3
+ return function addInitializer(initializer) {
4
+ assertNotFinished(decoratorFinishedRef, "addInitializer");
5
+ assertCallable(initializer, "An initializer");
6
+ initializers.push(initializer);
7
+ };
8
+ }
9
+ function memberDec(dec, name, desc, initializers, kind, isStatic, isPrivate, metadata, value) {
10
+ var kindStr;
11
+ switch(kind){
12
+ case 1:
13
+ kindStr = "accessor";
14
+ break;
15
+ case 2:
16
+ kindStr = "method";
17
+ break;
18
+ case 3:
19
+ kindStr = "getter";
20
+ break;
21
+ case 4:
22
+ kindStr = "setter";
23
+ break;
24
+ default:
25
+ kindStr = "field";
26
+ }
27
+ var ctx = {
28
+ kind: kindStr,
29
+ name: isPrivate ? "#" + name : name,
30
+ static: isStatic,
31
+ private: isPrivate,
32
+ metadata: metadata
33
+ };
34
+ var decoratorFinishedRef = {
35
+ v: false
36
+ };
37
+ ctx.addInitializer = createAddInitializerMethod(initializers, decoratorFinishedRef);
38
+ var get, set;
39
+ if (kind === 0) {
40
+ if (isPrivate) {
41
+ get = desc.get;
42
+ set = desc.set;
43
+ } else {
44
+ get = function() {
45
+ return this[name];
46
+ };
47
+ set = function(v) {
48
+ this[name] = v;
49
+ };
50
+ }
51
+ } else if (kind === 2) {
52
+ get = function() {
53
+ return desc.value;
54
+ };
55
+ } else {
56
+ if (kind === 1 || kind === 3) {
57
+ get = function() {
58
+ return desc.get.call(this);
59
+ };
60
+ }
61
+ if (kind === 1 || kind === 4) {
62
+ set = function(v) {
63
+ desc.set.call(this, v);
64
+ };
65
+ }
66
+ }
67
+ ctx.access = get && set ? {
68
+ get: get,
69
+ set: set
70
+ } : get ? {
71
+ get: get
72
+ } : {
73
+ set: set
74
+ };
75
+ try {
76
+ return dec(value, ctx);
77
+ } finally{
78
+ decoratorFinishedRef.v = true;
79
+ }
80
+ }
81
+ function assertNotFinished(decoratorFinishedRef, fnName) {
82
+ if (decoratorFinishedRef.v) {
83
+ throw new Error("attempted to call " + fnName + " after decoration was finished");
84
+ }
85
+ }
86
+ function assertCallable(fn, hint) {
87
+ if (typeof fn !== "function") {
88
+ throw new TypeError(hint + " must be a function");
89
+ }
90
+ }
91
+ function assertValidReturnValue(kind, value) {
92
+ var type = typeof value;
93
+ if (kind === 1) {
94
+ if (type !== "object" || value === null) {
95
+ throw new TypeError("accessor decorators must return an object with get, set, or init properties or void 0");
96
+ }
97
+ if (value.get !== undefined) {
98
+ assertCallable(value.get, "accessor.get");
99
+ }
100
+ if (value.set !== undefined) {
101
+ assertCallable(value.set, "accessor.set");
102
+ }
103
+ if (value.init !== undefined) {
104
+ assertCallable(value.init, "accessor.init");
105
+ }
106
+ } else if (type !== "function") {
107
+ var hint;
108
+ if (kind === 0) {
109
+ hint = "field";
110
+ } else if (kind === 10) {
111
+ hint = "class";
112
+ } else {
113
+ hint = "method";
114
+ }
115
+ throw new TypeError(hint + " decorators must return a function or void 0");
116
+ }
117
+ }
118
+ function applyMemberDec(ret, base, decInfo, name, kind, isStatic, isPrivate, initializers, metadata) {
119
+ var decs = decInfo[0];
120
+ var desc, init, value;
121
+ if (isPrivate) {
122
+ if (kind === 0 || kind === 1) {
123
+ desc = {
124
+ get: decInfo[3],
125
+ set: decInfo[4]
126
+ };
127
+ } else if (kind === 3) {
128
+ desc = {
129
+ get: decInfo[3]
130
+ };
131
+ } else if (kind === 4) {
132
+ desc = {
133
+ set: decInfo[3]
134
+ };
135
+ } else {
136
+ desc = {
137
+ value: decInfo[3]
138
+ };
139
+ }
140
+ } else if (kind !== 0) {
141
+ desc = Object.getOwnPropertyDescriptor(base, name);
142
+ }
143
+ if (kind === 1) {
144
+ value = {
145
+ get: desc.get,
146
+ set: desc.set
147
+ };
148
+ } else if (kind === 2) {
149
+ value = desc.value;
150
+ } else if (kind === 3) {
151
+ value = desc.get;
152
+ } else if (kind === 4) {
153
+ value = desc.set;
154
+ }
155
+ var newValue, get, set;
156
+ if (typeof decs === "function") {
157
+ newValue = memberDec(decs, name, desc, initializers, kind, isStatic, isPrivate, metadata, value);
158
+ if (newValue !== void 0) {
159
+ assertValidReturnValue(kind, newValue);
160
+ if (kind === 0) {
161
+ init = newValue;
162
+ } else if (kind === 1) {
163
+ init = newValue.init;
164
+ get = newValue.get || value.get;
165
+ set = newValue.set || value.set;
166
+ value = {
167
+ get: get,
168
+ set: set
169
+ };
170
+ } else {
171
+ value = newValue;
172
+ }
173
+ }
174
+ } else {
175
+ for(var i = decs.length - 1; i >= 0; i--){
176
+ var dec = decs[i];
177
+ newValue = memberDec(dec, name, desc, initializers, kind, isStatic, isPrivate, metadata, value);
178
+ if (newValue !== void 0) {
179
+ assertValidReturnValue(kind, newValue);
180
+ var newInit;
181
+ if (kind === 0) {
182
+ newInit = newValue;
183
+ } else if (kind === 1) {
184
+ newInit = newValue.init;
185
+ get = newValue.get || value.get;
186
+ set = newValue.set || value.set;
187
+ value = {
188
+ get: get,
189
+ set: set
190
+ };
191
+ } else {
192
+ value = newValue;
193
+ }
194
+ if (newInit !== void 0) {
195
+ if (init === void 0) {
196
+ init = newInit;
197
+ } else if (typeof init === "function") {
198
+ init = [
199
+ init,
200
+ newInit
201
+ ];
202
+ } else {
203
+ init.push(newInit);
204
+ }
205
+ }
206
+ }
207
+ }
208
+ }
209
+ if (kind === 0 || kind === 1) {
210
+ if (init === void 0) {
211
+ init = function(instance, init) {
212
+ return init;
213
+ };
214
+ } else if (typeof init !== "function") {
215
+ var ownInitializers = init;
216
+ init = function(instance, init) {
217
+ var value = init;
218
+ for(var i = 0; i < ownInitializers.length; i++){
219
+ value = ownInitializers[i].call(instance, value);
220
+ }
221
+ return value;
222
+ };
223
+ } else {
224
+ var originalInitializer = init;
225
+ init = function(instance, init) {
226
+ return originalInitializer.call(instance, init);
227
+ };
228
+ }
229
+ ret.push(init);
230
+ }
231
+ if (kind !== 0) {
232
+ if (kind === 1) {
233
+ desc.get = value.get;
234
+ desc.set = value.set;
235
+ } else if (kind === 2) {
236
+ desc.value = value;
237
+ } else if (kind === 3) {
238
+ desc.get = value;
239
+ } else if (kind === 4) {
240
+ desc.set = value;
241
+ }
242
+ if (isPrivate) {
243
+ if (kind === 1) {
244
+ ret.push(function(instance, args) {
245
+ return value.get.call(instance, args);
246
+ });
247
+ ret.push(function(instance, args) {
248
+ return value.set.call(instance, args);
249
+ });
250
+ } else if (kind === 2) {
251
+ ret.push(value);
252
+ } else {
253
+ ret.push(function(instance, args) {
254
+ return value.call(instance, args);
255
+ });
256
+ }
257
+ } else {
258
+ Object.defineProperty(base, name, desc);
259
+ }
260
+ }
261
+ }
262
+ function applyMemberDecs(Class, decInfos, metadata) {
263
+ var ret = [];
264
+ var protoInitializers;
265
+ var staticInitializers;
266
+ var existingProtoNonFields = new Map();
267
+ var existingStaticNonFields = new Map();
268
+ for(var i = 0; i < decInfos.length; i++){
269
+ var decInfo = decInfos[i];
270
+ if (!Array.isArray(decInfo)) continue;
271
+ var kind = decInfo[1];
272
+ var name = decInfo[2];
273
+ var isPrivate = decInfo.length > 3;
274
+ var isStatic = kind >= 5;
275
+ var base;
276
+ var initializers;
277
+ if (isStatic) {
278
+ base = Class;
279
+ kind = kind - 5;
280
+ staticInitializers = staticInitializers || [];
281
+ initializers = staticInitializers;
282
+ } else {
283
+ base = Class.prototype;
284
+ protoInitializers = protoInitializers || [];
285
+ initializers = protoInitializers;
286
+ }
287
+ if (kind !== 0 && !isPrivate) {
288
+ var existingNonFields = isStatic ? existingStaticNonFields : existingProtoNonFields;
289
+ var existingKind = existingNonFields.get(name) || 0;
290
+ if (existingKind === true || existingKind === 3 && kind !== 4 || existingKind === 4 && kind !== 3) {
291
+ throw new Error("Attempted to decorate a public method/accessor that has the same name as a previously decorated public method/accessor. This is not currently supported by the decorators plugin. Property name was: " + name);
292
+ } else if (!existingKind && kind > 2) {
293
+ existingNonFields.set(name, kind);
294
+ } else {
295
+ existingNonFields.set(name, true);
296
+ }
297
+ }
298
+ applyMemberDec(ret, base, decInfo, name, kind, isStatic, isPrivate, initializers, metadata);
299
+ }
300
+ pushInitializers(ret, protoInitializers);
301
+ pushInitializers(ret, staticInitializers);
302
+ return ret;
303
+ }
304
+ function pushInitializers(ret, initializers) {
305
+ if (initializers) {
306
+ ret.push(function(instance) {
307
+ for(var i = 0; i < initializers.length; i++){
308
+ initializers[i].call(instance);
309
+ }
310
+ return instance;
311
+ });
312
+ }
313
+ }
314
+ function applyClassDecs(targetClass, classDecs, metadata) {
315
+ if (classDecs.length > 0) {
316
+ var initializers = [];
317
+ var newClass = targetClass;
318
+ var name = targetClass.name;
319
+ for(var i = classDecs.length - 1; i >= 0; i--){
320
+ var decoratorFinishedRef = {
321
+ v: false
322
+ };
323
+ try {
324
+ var nextNewClass = classDecs[i](newClass, {
325
+ kind: "class",
326
+ name: name,
327
+ addInitializer: createAddInitializerMethod(initializers, decoratorFinishedRef),
328
+ metadata
329
+ });
330
+ } finally{
331
+ decoratorFinishedRef.v = true;
332
+ }
333
+ if (nextNewClass !== undefined) {
334
+ assertValidReturnValue(10, nextNewClass);
335
+ newClass = nextNewClass;
336
+ }
337
+ }
338
+ return [
339
+ defineMetadata(newClass, metadata),
340
+ function() {
341
+ for(var i = 0; i < initializers.length; i++){
342
+ initializers[i].call(newClass);
343
+ }
344
+ }
345
+ ];
346
+ }
347
+ }
348
+ function defineMetadata(Class, metadata) {
349
+ return Object.defineProperty(Class, Symbol.metadata || Symbol.for("Symbol.metadata"), {
350
+ configurable: true,
351
+ enumerable: true,
352
+ value: metadata
353
+ });
354
+ }
355
+ return function applyDecs2203R(targetClass, memberDecs, classDecs, parentClass) {
356
+ if (parentClass !== void 0) {
357
+ var parentMetadata = parentClass[Symbol.metadata || Symbol.for("Symbol.metadata")];
358
+ }
359
+ var metadata = Object.create(parentMetadata === void 0 ? null : parentMetadata);
360
+ var e = applyMemberDecs(targetClass, memberDecs, metadata);
361
+ if (!classDecs.length) defineMetadata(targetClass, metadata);
362
+ return {
363
+ e: e,
364
+ get c () {
365
+ return applyClassDecs(targetClass, classDecs, metadata);
366
+ }
367
+ };
368
+ };
369
+ }
370
+ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
371
+ return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
6
372
  }
7
- import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
373
+ var _initProto;
374
+ import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
375
+ import { assertRequired, compact, pick } from '@aztec/foundation/collection';
8
376
  import { memoize } from '@aztec/foundation/decorators';
9
377
  import { createLogger } from '@aztec/foundation/log';
10
- import { DateProvider } from '@aztec/foundation/timer';
378
+ import { RunningPromise } from '@aztec/foundation/running-promise';
379
+ import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
380
+ import { getLastSiblingPath } from '@aztec/prover-client/helpers';
381
+ import { ChonkCache } from '@aztec/prover-client/orchestrator';
11
382
  import { PublicProcessorFactory } from '@aztec/simulator/server';
12
- import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
383
+ import { EventDrivenL2BlockStream, L2TipsMemoryStore } from '@aztec/stdlib/block';
384
+ import { getEpochAtSlot, getProofSubmissionDeadlineEpoch } from '@aztec/stdlib/epoch-helpers';
13
385
  import { EpochProvingJobTerminalState, tryStop } from '@aztec/stdlib/interfaces/server';
14
- import { Attributes, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
386
+ import { MerkleTreeId } from '@aztec/stdlib/trees';
387
+ import { getTelemetryClient } from '@aztec/telemetry-client';
15
388
  import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
16
- import { EpochProvingJob } from './job/epoch-proving-job.js';
389
+ import { CheckpointStore } from './checkpoint-store.js';
17
390
  import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
391
+ import { ProofPublishingService } from './proof-publishing-service.js';
392
+ import { SessionManager } from './session-manager.js';
393
+ /**
394
+ * Grace period for the proof-publishing service to settle during shutdown. The service waits for
395
+ * any in-flight L1 proof-submission tx to finish; that tx can take a long time to mine, so we cap
396
+ * the wait rather than letting `stop()` hang indefinitely.
397
+ */ const PUBLISHING_SERVICE_STOP_TIMEOUT_MS = 30_000;
18
398
  /**
19
- * An Aztec Prover Node is a standalone process that monitors the unfinalized chain on L1 for unproven epochs,
20
- * fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
21
- * proof for the epoch, and submits it to L1.
399
+ * An Aztec Prover Node is a standalone process that monitors the chain for new checkpoints,
400
+ * starts proving them optimistically as they arrive, and submits epoch proofs to L1 once
401
+ * complete.
402
+ *
403
+ * The class is intentionally thin: it owns the long-lived collections (`CheckpointStore`,
404
+ * `ChonkCache`, `SessionManager`), the L2BlockStream, and a periodic ticker that nudges the
405
+ * manager to pick up newly-complete epochs. Every session lifecycle decision is delegated to
406
+ * the `SessionManager`. Each chain event is translated here into a single method call on it.
22
407
  */ export class ProverNode {
23
408
  prover;
24
409
  publisherFactory;
@@ -27,19 +412,46 @@ import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
27
412
  contractDataSource;
28
413
  worldState;
29
414
  p2pClient;
30
- epochsMonitor;
31
415
  rollupContract;
32
416
  l1Metrics;
417
+ avmSimulator;
33
418
  telemetryClient;
34
- log;
419
+ delayer;
35
420
  dateProvider;
36
- jobs;
421
+ static{
422
+ ({ e: [_initProto] } = _apply_decs_2203_r(this, [
423
+ [
424
+ memoize,
425
+ 2,
426
+ "getL1Constants"
427
+ ]
428
+ ], []));
429
+ }
430
+ log;
431
+ checkpointStore;
432
+ chonkCache;
433
+ sessionManager;
37
434
  config;
38
435
  jobMetrics;
39
436
  rewardsMetrics;
437
+ /** In-memory store for the L2BlockStream's local data provider. */ tipsStore;
438
+ /** Block stream for checkpoint and reorg detection. */ blockStream;
439
+ /**
440
+ * Highest epoch whose proof-submission window has passed. Monotonic high-water mark.
441
+ * Seeded from the last fully-proven epoch at start(); advanced on every block-stream
442
+ * event by comparing the archiver's latest synced L2 slot against each epoch's
443
+ * submission deadline. Protected so tests can verify the start() seeding.
444
+ */ lastExpiredEpoch;
445
+ /**
446
+ * Highest checkpoint number whose proving-side handling has completed (or that was legitimately skipped).
447
+ * The catch-up loop walks from here to each `chain-checkpointed` tip event. Seeded at start() from the last
448
+ * checkpoint of the last fully-proven epoch (or 0), so a restart reprocesses the partially-proven epoch rather
449
+ * than trusting a checkpointed tip that may sit ahead of unproven checkpoints. Clamped down on a prune.
450
+ */ lastProcessedCheckpoint;
451
+ /** Periodic tick that runs the epoch-expiry sweep during idle periods when no block-stream events arrive. */ expiryTicker;
40
452
  tracer;
41
- publisher;
42
- constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, epochsMonitor, rollupContract, l1Metrics, config = {}, telemetryClient = getTelemetryClient()){
453
+ publishingService;
454
+ constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, rollupContract, l1Metrics, avmSimulator, config = {}, telemetryClient = getTelemetryClient(), delayer, dateProvider = new DateProvider()){
43
455
  this.prover = prover;
44
456
  this.publisherFactory = publisherFactory;
45
457
  this.l2BlockSource = l2BlockSource;
@@ -47,17 +459,18 @@ import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
47
459
  this.contractDataSource = contractDataSource;
48
460
  this.worldState = worldState;
49
461
  this.p2pClient = p2pClient;
50
- this.epochsMonitor = epochsMonitor;
51
462
  this.rollupContract = rollupContract;
52
463
  this.l1Metrics = l1Metrics;
464
+ this.avmSimulator = avmSimulator;
53
465
  this.telemetryClient = telemetryClient;
54
- this.log = createLogger('prover-node');
55
- this.dateProvider = new DateProvider();
56
- this.jobs = new Map();
466
+ this.delayer = delayer;
467
+ this.dateProvider = dateProvider;
468
+ this.log = (_initProto(this), createLogger('prover-node'));
469
+ this.lastProcessedCheckpoint = CheckpointNumber.ZERO;
57
470
  this.config = {
58
471
  proverNodePollingIntervalMs: 1_000,
59
472
  proverNodeMaxPendingJobs: 100,
60
- proverNodeMaxParallelBlocksPerEpoch: 32,
473
+ proverNodeMaxParallelBlocksPerEpoch: 0,
61
474
  txGatheringIntervalMs: 1_000,
62
475
  txGatheringBatchSize: 10,
63
476
  txGatheringMaxParallelRequestsPerNode: 100,
@@ -71,6 +484,22 @@ import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
71
484
  this.tracer = telemetryClient.getTracer('ProverNode');
72
485
  this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
73
486
  this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
487
+ this.tipsStore = new L2TipsMemoryStore(this.l2BlockSource.getGenesisBlockHash());
488
+ this.chonkCache = new ChonkCache(this.log.getBindings());
489
+ this.checkpointStore = new CheckpointStore(this.l2BlockSource, {
490
+ proverFactory: this.prover,
491
+ chonkCache: this.chonkCache,
492
+ publicProcessorFactory: new PublicProcessorFactory(this.contractDataSource, this.avmSimulator, this.dateProvider, this.telemetryClient, this.log.getBindings()),
493
+ dbProvider: this.worldState,
494
+ txProvider: this.p2pClient.getTxProvider(),
495
+ dateProvider: this.dateProvider,
496
+ proverId: this.prover.getProverId(),
497
+ metrics: this.jobMetrics,
498
+ txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
499
+ deadline: undefined
500
+ }, {
501
+ slotWatcherPollIntervalMs: this.config.proverNodePollingIntervalMs
502
+ }, this.log.getBindings());
74
503
  }
75
504
  getProverId() {
76
505
  return this.prover.getProverId();
@@ -78,252 +507,469 @@ import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
78
507
  getP2P() {
79
508
  return this.p2pClient;
80
509
  }
510
+ /** Test-only: the shared L1 tx delayer, if enabled. */ getDelayer() {
511
+ return this.delayer;
512
+ }
513
+ /** Observability summary for the ProverNodeApi. */ getJobs() {
514
+ return Promise.resolve(this.sessionManager?.getJobs() ?? []);
515
+ }
516
+ /** Tests inspect this when validating reconcile behaviour. */ getCheckpointStore() {
517
+ return this.checkpointStore;
518
+ }
519
+ /** Tests inspect this to verify chonk-cache release semantics. */ getChonkCache() {
520
+ return this.chonkCache;
521
+ }
522
+ /** Tests inspect this when looking up live sessions. */ getSessionManager() {
523
+ if (!this.sessionManager) {
524
+ throw new Error('SessionManager not yet constructed — start() must be called first.');
525
+ }
526
+ return this.sessionManager;
527
+ }
528
+ /** Returns the underlying prover instance. */ getProver() {
529
+ return this.prover;
530
+ }
531
+ // ---------------- L2BlockStream handler ----------------
532
+ async handleBlockStreamEvent(event) {
533
+ switch(event.type){
534
+ case 'chain-checkpointed':
535
+ await this.processCheckpointJump(event.checkpoint.number);
536
+ break;
537
+ case 'chain-pruned':
538
+ await this.handlePruneEvent(event.block);
539
+ break;
540
+ case 'chain-proven':
541
+ this.publishingService?.onChainProven(BlockNumber(event.block.number));
542
+ break;
543
+ // The proposed tip drives only the tips store's walk-back history (recorded below); the prover-node
544
+ // tracks checkpoints, not proposed blocks. `blocks-added` is never emitted in tips-only mode, and
545
+ // `chain-finalized` carries nothing the prover-node acts on.
546
+ case 'chain-proposed':
547
+ case 'chain-finalized':
548
+ case 'blocks-added':
549
+ break;
550
+ default:
551
+ {
552
+ const _ = event;
553
+ break;
554
+ }
555
+ }
556
+ // Expiry is driven by the archiver's latest synced L2 slot
557
+ await this.checkEpochExpiry();
558
+ // Advance the local tips store only after the proving-side handling has succeeded. Any
559
+ // failure above propagates to the L2BlockStream (which logs and stops this poll pass) and
560
+ // skips this update, so the event is re-emitted on the next poll rather than skipped (A-1041).
561
+ await this.tipsStore.handleBlockStreamEvent(event);
562
+ }
81
563
  /**
82
- * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
83
- * @param epochNumber - The epoch number that was just completed.
84
- * @returns false if there is an error, true otherwise
85
- */ async handleEpochReadyToProve(epochNumber) {
86
- try {
87
- this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
88
- jobs: Array.from(this.jobs.values()).map((job)=>`${job.getEpochNumber()}:${job.getId()}`)
564
+ * Walks every checkpoint between the local cursor and the newly-reported checkpointed tip, registering
565
+ * each one that belongs to an epoch that can still be proven. The block stream now delivers a single thin
566
+ * `chain-checkpointed` tip event per pass rather than one fat event per checkpoint, so this drives the
567
+ * catch-up itself: light metadata first (`getCheckpointsData`) to decide relevance per epoch, then a heavy
568
+ * `getCheckpoints` fetch only for checkpoints in provable epochs.
569
+ *
570
+ * The cursor advances one checkpoint at a time and only after that checkpoint's proving-side handling has
571
+ * fully succeeded, preserving the A-1041 at-least-once semantics: a mid-jump failure leaves the cursor
572
+ * behind so the next pass retries from the first checkpoint that did not complete.
573
+ */ async processCheckpointJump(targetCheckpoint) {
574
+ if (targetCheckpoint <= this.lastProcessedCheckpoint) {
575
+ return;
576
+ }
577
+ const l1Constants = await this.getL1Constants();
578
+ // Cap the catch-up at the `(proofSubmissionEpochs + 1) * epochDuration` most recent checkpoints.
579
+ // When the cursor is much further behind (e.g. resyncing after a long time offline), fetching the whole gap could
580
+ // load thousands of checkpoints we cannot act on: anything older than the last two epochs is already past
581
+ // its proof-submission window, so we skip it and jump the cursor forward to the start of the capped range.
582
+ const maxCheckpoints = (l1Constants.proofSubmissionEpochs + 1) * l1Constants.epochDuration;
583
+ let from = CheckpointNumber(this.lastProcessedCheckpoint + 1);
584
+ if (Number(targetCheckpoint - from) + 1 > maxCheckpoints) {
585
+ const cappedFrom = CheckpointNumber(targetCheckpoint - maxCheckpoints + 1);
586
+ this.log.warn(`Skipping unprovable checkpoints during catch-up; the prover node is far behind`, {
587
+ from,
588
+ cappedFrom,
589
+ targetCheckpoint,
590
+ maxCheckpoints
89
591
  });
90
- const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
91
- if (activeJobs.length > 0) {
92
- this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
93
- activeJobs: activeJobs.map((job)=>job.uuid)
94
- });
95
- return true;
592
+ // Advance the cursor past the skipped checkpoints so they are never retried.
593
+ this.lastProcessedCheckpoint = CheckpointNumber(cappedFrom - 1);
594
+ from = cappedFrom;
595
+ }
596
+ const limit = Number(targetCheckpoint - from) + 1;
597
+ const metadatas = await this.l2BlockSource.getCheckpointsData({
598
+ from,
599
+ limit
600
+ });
601
+ // Per-epoch relevance is cached so a multi-checkpoint epoch resolves it once. Skipping is whole-epoch
602
+ // only: the SessionManager requires an epoch's checkpoints fully covered before it opens a session, so we
603
+ // never drop an individual checkpoint inside an epoch we will prove.
604
+ const epochSkippable = new Map();
605
+ for (const metadata of metadatas){
606
+ const epochNumber = getEpochAtSlot(metadata.header.slotNumber, l1Constants);
607
+ let skippable = epochSkippable.get(epochNumber);
608
+ if (skippable === undefined) {
609
+ skippable = await this.isEpochFullyProven(epochNumber, l1Constants) || await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants);
610
+ epochSkippable.set(epochNumber, skippable);
96
611
  }
97
- await this.startProof(epochNumber);
98
- return true;
99
- } catch (err) {
100
- if (err instanceof EmptyEpochError) {
101
- this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
612
+ if (skippable) {
613
+ this.log.debug(`Skipping checkpoint ${metadata.checkpointNumber} for unprovable epoch ${epochNumber}`);
102
614
  } else {
103
- this.log.error(`Error handling epoch completed`, err);
615
+ await this.registerCheckpoint(metadata.checkpointNumber, epochNumber);
616
+ }
617
+ // Advance only after the checkpoint's handling succeeded (or it was legitimately skipped). registerCheckpoint
618
+ // throws on failure, which leaves the cursor here for the next pass to retry (A-1041).
619
+ this.lastProcessedCheckpoint = metadata.checkpointNumber;
620
+ }
621
+ }
622
+ /** Heavy-fetch a single checkpoint, register it with the store, and notify the session manager. */ async registerCheckpoint(checkpointNumber, epochNumber) {
623
+ const published = await this.l2BlockSource.getCheckpoint({
624
+ number: checkpointNumber
625
+ });
626
+ if (!published) {
627
+ throw new Error(`Checkpoint ${checkpointNumber} not found in block source during catch-up`);
628
+ }
629
+ const checkpoint = published.checkpoint;
630
+ this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
631
+ checkpointNumber: checkpoint.number,
632
+ epochNumber,
633
+ slotNumber: checkpoint.header.slotNumber
634
+ });
635
+ const registerData = await this.collectRegisterData(checkpoint, published.attestations);
636
+ await this.checkpointStore.addOrUpdate(checkpoint, registerData);
637
+ await this.sessionManager?.onCheckpointAdded(epochNumber);
638
+ // Tips-only mode delivers no blocks, so record one witness per checkpointed block: a reorg into the checkpoint's
639
+ // range then prunes at the true divergence instead of the nearest sparse tip anchor.
640
+ await this.tipsStore.recordBlockHashes(await Promise.all(checkpoint.blocks.map(async (block)=>({
641
+ number: block.number,
642
+ hash: (await block.header.hash()).toString()
643
+ }))));
644
+ }
645
+ /**
646
+ * Gathers register-time data for a checkpoint: previous block header, L1-to-L2 messages,
647
+ * and the archive sibling path.
648
+ */ async collectRegisterData(checkpoint, attestations) {
649
+ const previousBlockNumber = BlockNumber(checkpoint.blocks[0].number - 1);
650
+ const previousBlockHeader = await this.gatherPreviousBlockHeader(previousBlockNumber);
651
+ const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpoint.number);
652
+ const lastBlock = checkpoint.blocks.at(-1);
653
+ const lastBlockHash = await lastBlock.header.hash();
654
+ await this.worldState.syncImmediate(lastBlock.number, lastBlockHash);
655
+ const previousArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, this.worldState.getSnapshot(previousBlockNumber));
656
+ return {
657
+ attestations,
658
+ previousBlockHeader,
659
+ l1ToL2Messages,
660
+ previousArchiveSiblingPath
661
+ };
662
+ }
663
+ /**
664
+ * Marks every prover orphaned by the prune as pruned, clamps the catch-up cursor below the prune target's
665
+ * checkpoint, and notifies the session manager. Keyed off the prune target block (the highest surviving block)
666
+ * rather than the source's checkpointed tip, which can sit above the target after a re-checkpoint and would leave
667
+ * orphaned provers canonical. Throws (rather than warning) if the cursor floor cannot be resolved, so the pass
668
+ * fails and the prune is retried next iteration.
669
+ */ async handlePruneEvent(prunedToBlock) {
670
+ this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, {
671
+ prunedToBlock
672
+ });
673
+ // Resolve the cursor floor BEFORE marking provers: markPrunedAboveBlock returns only newly-marked provers, so a
674
+ // throw after marking would leave a retry pass with nothing to act on. Resolving first means a throw leaves
675
+ // everything untouched and the next pass retries the whole handler (the tips cursor only advances on success).
676
+ let cursorFloor;
677
+ if (prunedToBlock.number === 0) {
678
+ cursorFloor = CheckpointNumber.ZERO;
679
+ } else {
680
+ const targetData = await this.l2BlockSource.getBlockData({
681
+ number: prunedToBlock.number
682
+ });
683
+ if (targetData === undefined) {
684
+ throw new Error(`No block data found for prune target block ${prunedToBlock.number}; cannot clamp checkpoint cursor`);
104
685
  }
686
+ // Clamp to `cpAtTarget - 1`: a mid-checkpoint target leaves that checkpoint partially orphaned and it must be
687
+ // reprocessed. Over-clamping merely re-registers a checkpoint (at-least-once by design — A-1041); under-clamping
688
+ // would permanently skip a rebuilt same-number checkpoint.
689
+ cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1));
690
+ }
691
+ const affected = this.checkpointStore.markPrunedAboveBlock(prunedToBlock.number);
692
+ if (this.lastProcessedCheckpoint > cursorFloor) {
693
+ this.lastProcessedCheckpoint = cursorFloor;
694
+ }
695
+ if (affected.length === 0) {
696
+ return;
697
+ }
698
+ const l1Constants = await this.getL1Constants();
699
+ const affectedEpochs = Array.from(new Set(affected.map((p)=>Number(getEpochAtSlot(p.slotNumber, l1Constants))))).map((n)=>EpochNumber(n));
700
+ // The session manager cancels every affected session, which in turn calls
701
+ // publishingService.withdraw(uuid) for each candidate; no separate notification to the
702
+ // publishing service is needed.
703
+ await this.sessionManager?.onPrune(affectedEpochs);
704
+ }
705
+ /**
706
+ * Returns true once the chain has advanced past the given epoch's proof-submission window.
707
+ * Used to ignore checkpoints whose epoch can no longer be proven in time — chiefly while the
708
+ * archiver replays old blocks after a restart. Compares the archiver's latest synced L2 slot
709
+ * against the epoch's submission-deadline epoch; conservatively returns false if the slot can't
710
+ * be read yet.
711
+ */ async isEpochPastProofSubmissionWindow(epochNumber, l1Constants) {
712
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
713
+ if (latestSlot === undefined) {
105
714
  return false;
106
715
  }
716
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
717
+ return latestEpoch >= getProofSubmissionDeadlineEpoch(epochNumber, l1Constants);
718
+ }
719
+ /**
720
+ * Compares the archiver's latest synced L2 slot against `lastExpiredEpoch` and, for each
721
+ * newly-expired epoch, releases the chonk-cache entries for its blocks and reaps any
722
+ * CheckpointProvers in the store. An epoch E is expired once the chain reaches the start
723
+ * of epoch `E + proofSubmissionEpochs + 1`. Silently no-ops if nothing has expired since
724
+ * the last check or the archiver's slot can't be read.
725
+ */ async checkEpochExpiry() {
726
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
727
+ if (latestSlot === undefined) {
728
+ return;
729
+ }
730
+ const l1Constants = await this.getL1Constants();
731
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
732
+ const offset = l1Constants.proofSubmissionEpochs + 1;
733
+ if (latestEpoch < offset) {
734
+ return;
735
+ }
736
+ const newlyExpiredUpTo = EpochNumber(latestEpoch - offset);
737
+ const from = this.lastExpiredEpoch === undefined ? EpochNumber(0) : EpochNumber(this.lastExpiredEpoch + 1);
738
+ if (newlyExpiredUpTo < from) {
739
+ return;
740
+ }
741
+ for(let e = from; e <= newlyExpiredUpTo; e = EpochNumber(e + 1)){
742
+ await this.expireEpoch(e);
743
+ }
744
+ this.lastExpiredEpoch = newlyExpiredUpTo;
107
745
  }
108
746
  /**
109
- * Starts the prover node so it periodically checks for unproven epochs in the unfinalized chain from L1 and
110
- * starts proving jobs for them.
111
- */ async start() {
112
- this.epochsMonitor.start(this);
747
+ * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and
748
+ * reaps every CheckpointProver in the store whose epoch number matches.
749
+ */ async expireEpoch(epoch) {
750
+ try {
751
+ const blocks = await this.l2BlockSource.getBlocks({
752
+ epoch,
753
+ onlyCheckpointed: true
754
+ });
755
+ if (blocks.length > 0) {
756
+ this.chonkCache.releaseForBlocks(blocks);
757
+ }
758
+ } catch (err) {
759
+ this.log.warn(`Could not release chonk-cache entries for expired epoch ${epoch}`, err);
760
+ }
761
+ this.checkpointStore.reapExpired(epoch);
762
+ }
763
+ // ---------------- public API ----------------
764
+ /**
765
+ * Schedules proving for the given epoch and returns the job id without waiting for completion.
766
+ */ async startProof(epochNumber) {
767
+ if (!this.sessionManager) {
768
+ throw new Error('ProverNode not started');
769
+ }
770
+ return await this.sessionManager.startProof(epochNumber);
771
+ }
772
+ // ---------------- Service lifecycle ----------------
773
+ async start() {
774
+ await this.checkpointStore.start();
113
775
  await this.publisherFactory.start();
114
- this.publisher = await this.publisherFactory.create();
776
+ this.publishingService = new ProofPublishingService({
777
+ publisherFactory: this.publisherFactory,
778
+ l2BlockSource: this.l2BlockSource,
779
+ dateProvider: this.dateProvider,
780
+ config: {
781
+ skipSubmitProof: !!this.config.proverNodeDisableProofPublish
782
+ },
783
+ bindings: this.log.getBindings()
784
+ });
785
+ this.sessionManager = this.createSessionManager(this.publishingService);
786
+ // SessionManager owns its own periodic tick; start it here so it begins picking up
787
+ // epochs that become complete by time (no fresh checkpoint event) and advances once
788
+ // the previous epoch is proven on L1.
789
+ this.sessionManager.start();
790
+ // Now that the store + manager exist, arm the live-state observable gauges.
791
+ this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
792
+ const { lastFullyProvenEpoch } = await this.resolveLastFullyProvenEpoch();
793
+ this.lastExpiredEpoch = lastFullyProvenEpoch;
794
+ this.lastProcessedCheckpoint = await this.computeStartingCheckpoint(lastFullyProvenEpoch);
795
+ this.blockStream = new EventDrivenL2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
796
+ pollIntervalMS: this.config.proverNodePollingIntervalMs,
797
+ tipsOnly: true
798
+ });
799
+ this.blockStream.start();
800
+ // With thin once-per-pass tip events, the expiry sweep no longer fires once per checkpoint; drive it
801
+ // from a periodic tick so epochs still expire during idle/no-event periods.
802
+ this.expiryTicker = new RunningPromise(()=>this.checkEpochExpiry(), this.log, this.config.proverNodePollingIntervalMs);
803
+ this.expiryTicker.start();
115
804
  await this.rewardsMetrics.start();
116
805
  this.l1Metrics.start();
117
806
  this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
118
807
  }
119
- /**
120
- * Stops the prover node and all its dependencies.
121
- */ async stop() {
808
+ async stop() {
122
809
  this.log.info('Stopping ProverNode');
123
- await this.epochsMonitor.stop();
810
+ this.jobMetrics.stopObservingState();
811
+ await this.blockStream?.stop();
812
+ await this.expiryTicker?.stop();
813
+ if (this.sessionManager) {
814
+ await this.sessionManager.stop();
815
+ }
816
+ if (this.publishingService) {
817
+ // Bound the wait: the publishing service blocks until any in-flight L1 proof-submission tx
818
+ // settles, which can outlast a reasonable shutdown window. On timeout we log and move on —
819
+ // the tx may still mine, but shutdown must not hang on it.
820
+ const publishingService = this.publishingService;
821
+ await executeTimeout(()=>publishingService.stop(), PUBLISHING_SERVICE_STOP_TIMEOUT_MS, 'prover-node publishing-service stop').catch((err)=>this.log.warn(`Timed out stopping proof publishing service`, err));
822
+ }
823
+ await this.checkpointStore.stop();
824
+ this.chonkCache.stop();
124
825
  await this.prover.stop();
125
- await tryStop(this.p2pClient);
126
- await tryStop(this.l2BlockSource);
127
826
  await tryStop(this.publisherFactory);
128
- this.publisher?.interrupt();
129
- await Promise.all(Array.from(this.jobs.values()).map((job)=>job.stop()));
130
- await this.worldState.stop();
131
827
  this.rewardsMetrics.stop();
132
828
  this.l1Metrics.stop();
133
829
  await this.telemetryClient.stop();
134
830
  this.log.info('Stopped ProverNode');
135
831
  }
136
- /** Returns world state status. */ async getWorldStateSyncStatus() {
137
- const { syncSummary } = await this.worldState.status();
138
- return syncSummary;
139
- }
140
- /** Returns archiver status. */ getL2Tips() {
141
- return this.l2BlockSource.getL2Tips();
142
- }
143
832
  /**
144
- * Starts a proving process and returns immediately.
145
- */ async startProof(epochNumber) {
146
- const job = await this.createProvingJob(BigInt(epochNumber), {
147
- skipEpochCheck: true
833
+ * Constructs the session manager. Extracted so subclasses (test harness) can swap
834
+ * the implementation. Wired to `tryUploadSessionFailure` so failed sessions get
835
+ * their proving data uploaded.
836
+ */ createSessionManager(publishingService) {
837
+ return new SessionManager({
838
+ checkpointStore: this.checkpointStore,
839
+ l2BlockSource: this.l2BlockSource,
840
+ proverFactory: this.prover,
841
+ proverId: this.prover.getProverId(),
842
+ publishingService,
843
+ metrics: this.jobMetrics,
844
+ dateProvider: this.dateProvider,
845
+ config: {
846
+ maxPendingJobs: this.config.proverNodeMaxPendingJobs,
847
+ tickIntervalMs: this.config.proverNodePollingIntervalMs,
848
+ finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs
849
+ },
850
+ onSessionFailed: async (session)=>{
851
+ await this.tryUploadSessionFailure(session);
852
+ },
853
+ bindings: this.log.getBindings()
148
854
  });
149
- void this.runJob(job);
150
855
  }
151
- async runJob(job) {
152
- const epochNumber = job.getEpochNumber();
153
- const ctx = {
154
- id: job.getId(),
155
- epochNumber,
156
- state: undefined
157
- };
158
- try {
159
- await job.run();
160
- const state = job.getState();
161
- ctx.state = state;
162
- if (state === 'reorg') {
163
- this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
164
- await this.createProvingJob(epochNumber);
165
- } else if (state === 'failed') {
166
- this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
167
- await this.tryUploadEpochFailure(job);
168
- } else {
169
- this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
170
- }
171
- } catch (err) {
172
- this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
173
- } finally{
174
- this.jobs.delete(job.getId());
856
+ /**
857
+ * Installs session hooks for the e2e harness to interpose around top-tree proving
858
+ * (gate, override, or observe it) without monkey-patching the orchestrator factory.
859
+ * Applies to every session constructed after this call.
860
+ */ setSessionHooks(hooks) {
861
+ if (!this.sessionManager) {
862
+ throw new Error('ProverNode not started; call start() before setting session hooks.');
175
863
  }
864
+ this.sessionManager.setSessionHooks(hooks);
176
865
  }
177
- async tryUploadEpochFailure(job) {
178
- if (this.config.proverNodeFailedEpochStore) {
179
- return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, job.getId(), job.getProvingData(), this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
866
+ /** Uploads failure snapshots when sessions exit with `failed`. Exposed as a method so tests can spy on it. */ async tryUploadSessionFailure(session) {
867
+ if (!this.config.proverNodeFailedEpochStore) {
868
+ return undefined;
180
869
  }
870
+ const data = SessionManager.buildSessionProvingData(session);
871
+ return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, session.getId(), data, this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
181
872
  }
182
- /**
183
- * Returns the prover instance.
184
- */ getProver() {
185
- return this.prover;
873
+ // ---------------- helpers ----------------
874
+ getL1Constants() {
875
+ return this.l2BlockSource.getL1Constants();
186
876
  }
187
877
  /**
188
- * Returns an array of jobs being processed.
189
- */ getJobs() {
190
- return Promise.resolve(Array.from(this.jobs.entries()).map(([uuid, job])=>({
191
- uuid,
192
- status: job.getState(),
193
- epochNumber: Number(job.getEpochNumber())
194
- })));
195
- }
196
- async getActiveJobsForEpoch(epochBigInt) {
197
- const jobs = await this.getJobs();
198
- const epochNumber = Number(epochBigInt);
199
- return jobs.filter((job)=>job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
200
- }
201
- checkMaximumPendingJobs() {
202
- const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
203
- if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
204
- throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
205
- }
206
- }
207
- async createProvingJob(epochNumber, opts = {}) {
208
- this.checkMaximumPendingJobs();
209
- this.publisher = await this.publisherFactory.create();
210
- // Gather all data for this epoch
211
- const epochData = await this.gatherEpochData(epochNumber);
212
- const fromBlock = epochData.blocks[0].number;
213
- const toBlock = epochData.blocks.at(-1).number;
214
- this.log.verbose(`Creating proving job for epoch ${epochNumber} for block range ${fromBlock} to ${toBlock}`);
215
- // Fast forward world state to right before the target block and get a fork
216
- await this.worldState.syncImmediate(toBlock);
217
- // Create a processor factory
218
- const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.dateProvider, this.telemetryClient);
219
- // Set deadline for this job to run. It will abort if it takes too long.
220
- const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
221
- const deadline = new Date(Number(deadlineTs) * 1000);
222
- const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
223
- this.jobs.set(job.getId(), job);
224
- return job;
878
+ * Returns true if every block in the given epoch is proven on L1. An epoch is only
879
+ * fully proven when its *last* block is proven. Protected for direct unit-test access.
880
+ */ async isEpochFullyProven(epochNumber, l1Constants) {
881
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
882
+ tag: 'proven'
883
+ });
884
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
885
+ return false;
886
+ }
887
+ const provenHeader = (await this.l2BlockSource.getBlockData({
888
+ number: BlockNumber(provenBlockNumber)
889
+ }))?.header;
890
+ if (!provenHeader) {
891
+ return false;
892
+ }
893
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
894
+ if (epochNumber < provenEpoch) {
895
+ return true;
896
+ }
897
+ if (epochNumber > provenEpoch) {
898
+ return false;
899
+ }
900
+ return this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants);
225
901
  }
226
- getL1Constants() {
227
- return this.l2BlockSource.getL1Constants();
902
+ /** Protected for direct unit-test access. */ async isProvenBlockLastOfItsEpoch(provenBlockNumber, provenEpoch, l1Constants) {
903
+ const nextHeader = (await this.l2BlockSource.getBlockData({
904
+ number: BlockNumber(provenBlockNumber + 1)
905
+ }))?.header;
906
+ if (nextHeader) {
907
+ return getEpochAtSlot(nextHeader.getSlot(), l1Constants) > provenEpoch;
908
+ }
909
+ return this.l2BlockSource.isEpochComplete(provenEpoch);
228
910
  }
229
- async gatherEpochData(epochNumber) {
230
- const blocks = await this.gatherBlocks(epochNumber);
231
- const txArray = await this.gatherTxs(epochNumber, blocks);
232
- const txs = new Map(txArray.map((tx)=>[
233
- tx.getTxHash().toString(),
234
- tx
235
- ]));
236
- const l1ToL2Messages = await this.gatherMessages(epochNumber, blocks);
237
- const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, blocks[0]);
238
- const [lastBlock] = await this.l2BlockSource.getPublishedBlocks(blocks.at(-1).number, 1);
239
- const attestations = lastBlock?.attestations ?? [];
911
+ /**
912
+ * Resolves the last fully-proven epoch from L1 proven state, used to seed the catch-up cursor (via
913
+ * `computeStartingCheckpoint`) and `lastExpiredEpoch`. The fully-proven epoch is `provenEpoch` when the
914
+ * proven tip is the last block of its epoch, otherwise `provenEpoch - 1`, or `undefined` if no block is
915
+ * proven yet (so a restart reprocesses the partially-proven epoch rather than trusting a stale tip).
916
+ */ async resolveLastFullyProvenEpoch() {
917
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
918
+ tag: 'proven'
919
+ });
920
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
921
+ return {
922
+ lastFullyProvenEpoch: undefined
923
+ };
924
+ }
925
+ const l1Constants = await this.getL1Constants();
926
+ const provenHeader = (await this.l2BlockSource.getBlockData({
927
+ number: BlockNumber(provenBlockNumber)
928
+ }))?.header;
929
+ if (!provenHeader) {
930
+ return {
931
+ lastFullyProvenEpoch: undefined
932
+ };
933
+ }
934
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
935
+ if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
936
+ return {
937
+ lastFullyProvenEpoch: provenEpoch
938
+ };
939
+ }
940
+ const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
240
941
  return {
241
- blocks,
242
- txs,
243
- l1ToL2Messages,
244
- epochNumber,
245
- previousBlockHeader,
246
- attestations
942
+ lastFullyProvenEpoch
247
943
  };
248
944
  }
249
- async gatherBlocks(epochNumber) {
250
- const blocks = await this.l2BlockSource.getBlocksForEpoch(epochNumber);
251
- if (blocks.length === 0) {
252
- throw new EmptyEpochError(epochNumber);
253
- }
254
- return blocks;
255
- }
256
- async gatherTxs(epochNumber, blocks) {
257
- const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
258
- const txProvider = this.p2pClient.getTxProvider();
259
- const txsByBlock = await Promise.all(blocks.map((block)=>txProvider.getTxsForBlock(block, {
260
- deadline
261
- })));
262
- const txs = txsByBlock.map(({ txs })=>txs).flat();
263
- const missingTxs = txsByBlock.map(({ missingTxs })=>missingTxs).flat();
264
- if (missingTxs.length === 0) {
265
- this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, {
266
- epochNumber
267
- });
268
- return txs;
945
+ /**
946
+ * Resolves the catch-up cursor seed: the last checkpoint of the last fully-proven epoch, or 0 if none. Seeding
947
+ * from a checkpoint (rather than a checkpointed tip) guarantees a restart reprocesses every checkpoint of the
948
+ * partially-proven epoch, since the checkpointed tip can sit ahead of the last fully-proven checkpoint.
949
+ */ async computeStartingCheckpoint(lastFullyProvenEpoch) {
950
+ if (lastFullyProvenEpoch === undefined) {
951
+ return CheckpointNumber.ZERO;
269
952
  }
270
- throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxs.map((hash)=>hash.toString()).join(', ')}`);
271
- }
272
- async gatherMessages(epochNumber, blocks) {
273
- const messages = await Promise.all(blocks.map((b)=>this.l1ToL2MessageSource.getL1ToL2Messages(b.number)));
274
- const messageCount = sum(messages.map((m)=>m.length));
275
- this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, {
276
- epochNumber
277
- });
278
- const messagesByBlock = {};
279
- for(let i = 0; i < blocks.length; i++){
280
- messagesByBlock[blocks[i].number] = messages[i];
281
- }
282
- return messagesByBlock;
283
- }
284
- async gatherPreviousBlockHeader(epochNumber, initialBlock) {
285
- const previousBlockNumber = initialBlock.number - 1;
286
- const header = await (previousBlockNumber === 0 ? this.worldState.getCommitted().getInitialHeader() : this.l2BlockSource.getBlockHeader(previousBlockNumber));
287
- if (!header) {
288
- throw new Error(`Previous block header ${initialBlock.number} not found for proving epoch ${epochNumber}`);
289
- }
290
- this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
291
- return header;
292
- }
293
- /** Extracted for testing purposes. */ doCreateEpochProvingJob(data, deadline, publicProcessorFactory, publisher, opts = {}) {
294
- const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config;
295
- return new EpochProvingJob(data, this.worldState, this.prover.createEpochProver(), publicProcessorFactory, publisher, this.l2BlockSource, this.jobMetrics, deadline, {
296
- parallelBlockLimit,
297
- skipSubmitProof: proverNodeDisableProofPublish,
298
- ...opts
953
+ const checkpoints = await this.l2BlockSource.getCheckpointsData({
954
+ epoch: lastFullyProvenEpoch
299
955
  });
956
+ return checkpoints.at(-1)?.checkpointNumber ?? CheckpointNumber.ZERO;
300
957
  }
301
- /** Extracted for testing purposes. */ async triggerMonitors() {
302
- await this.epochsMonitor.work();
958
+ async gatherPreviousBlockHeader(previousBlockNumber) {
959
+ const data = await this.l2BlockSource.getBlockData({
960
+ number: BlockNumber(previousBlockNumber)
961
+ });
962
+ if (!data?.header) {
963
+ throw new Error(`Previous block header ${previousBlockNumber} not found`);
964
+ }
965
+ return data.header;
303
966
  }
304
967
  validateConfig() {
305
- if (this.config.proverNodeFailedEpochStore && (!this.config.dataDirectory || !this.config.l1ChainId || !this.config.rollupVersion)) {
968
+ if (this.config.proverNodeFailedEpochStore && (!this.config.dataDirectory || !this.config.l1ChainId || this.config.rollupVersion === undefined)) {
306
969
  this.log.warn(`Invalid prover-node config (missing dataDirectory, l1ChainId, or rollupVersion)`, pick(this.config, 'proverNodeFailedEpochStore', 'dataDirectory', 'l1ChainId', 'rollupVersion'));
307
970
  throw new Error('All of dataDirectory, l1ChainId, and rollupVersion are required if proverNodeFailedEpochStore is set.');
308
971
  }
309
972
  }
310
973
  }
311
- _ts_decorate([
312
- trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
313
- [Attributes.EPOCH_NUMBER]: Number(epochNumber)
314
- }))
315
- ], ProverNode.prototype, "createProvingJob", null);
316
- _ts_decorate([
317
- memoize
318
- ], ProverNode.prototype, "getL1Constants", null);
319
- _ts_decorate([
320
- trackSpan('ProverNode.gatherEpochData', (epochNumber)=>({
321
- [Attributes.EPOCH_NUMBER]: Number(epochNumber)
322
- }))
323
- ], ProverNode.prototype, "gatherEpochData", null);
324
- class EmptyEpochError extends Error {
325
- constructor(epochNumber){
326
- super(`No blocks found for epoch ${epochNumber}`);
327
- this.name = 'EmptyEpochError';
328
- }
329
- }
974
+ // Re-export so handlers can compare states externally.
975
+ export { EpochProvingJobTerminalState };