@aztec/prover-node 0.0.1-commit.d3ec352c → 0.0.1-commit.d58ff9d0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +518 -0
  2. package/dest/actions/download-epoch-proving-job.d.ts +1 -1
  3. package/dest/actions/download-epoch-proving-job.js +1 -1
  4. package/dest/actions/rerun-epoch-proving-job.d.ts +6 -4
  5. package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
  6. package/dest/actions/rerun-epoch-proving-job.js +191 -28
  7. package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
  8. package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
  9. package/dest/bin/run-failed-epoch.js +6 -5
  10. package/dest/checkpoint-store.d.ts +87 -0
  11. package/dest/checkpoint-store.d.ts.map +1 -0
  12. package/dest/checkpoint-store.js +186 -0
  13. package/dest/config.d.ts +10 -10
  14. package/dest/config.d.ts.map +1 -1
  15. package/dest/config.js +27 -22
  16. package/dest/factory.d.ts +23 -16
  17. package/dest/factory.d.ts.map +1 -1
  18. package/dest/factory.js +49 -68
  19. package/dest/index.d.ts +3 -1
  20. package/dest/index.d.ts.map +1 -1
  21. package/dest/index.js +2 -0
  22. package/dest/job/checkpoint-prover.d.ts +134 -0
  23. package/dest/job/checkpoint-prover.d.ts.map +1 -0
  24. package/dest/job/checkpoint-prover.js +353 -0
  25. package/dest/job/epoch-proving-job-data.d.ts +2 -2
  26. package/dest/job/epoch-proving-job-data.d.ts.map +1 -1
  27. package/dest/job/epoch-proving-job-data.js +1 -1
  28. package/dest/job/epoch-session.d.ts +146 -0
  29. package/dest/job/epoch-session.d.ts.map +1 -0
  30. package/dest/job/epoch-session.js +720 -0
  31. package/dest/job/top-tree-job.d.ts +82 -0
  32. package/dest/job/top-tree-job.d.ts.map +1 -0
  33. package/dest/job/top-tree-job.js +152 -0
  34. package/dest/metrics.d.ts +41 -4
  35. package/dest/metrics.d.ts.map +1 -1
  36. package/dest/metrics.js +125 -102
  37. package/dest/monitors/epoch-monitor.d.ts +1 -1
  38. package/dest/monitors/epoch-monitor.d.ts.map +1 -1
  39. package/dest/monitors/epoch-monitor.js +12 -19
  40. package/dest/proof-publishing-service.d.ts +161 -0
  41. package/dest/proof-publishing-service.d.ts.map +1 -0
  42. package/dest/proof-publishing-service.js +335 -0
  43. package/dest/prover-node-publisher.d.ts +31 -19
  44. package/dest/prover-node-publisher.d.ts.map +1 -1
  45. package/dest/prover-node-publisher.js +222 -78
  46. package/dest/prover-node.d.ts +122 -72
  47. package/dest/prover-node.d.ts.map +1 -1
  48. package/dest/prover-node.js +872 -229
  49. package/dest/prover-publisher-factory.d.ts +12 -6
  50. package/dest/prover-publisher-factory.d.ts.map +1 -1
  51. package/dest/prover-publisher-factory.js +8 -5
  52. package/dest/session-manager.d.ts +158 -0
  53. package/dest/session-manager.d.ts.map +1 -0
  54. package/dest/session-manager.js +452 -0
  55. package/dest/test/index.d.ts +7 -6
  56. package/dest/test/index.d.ts.map +1 -1
  57. package/package.json +25 -23
  58. package/src/actions/download-epoch-proving-job.ts +1 -1
  59. package/src/actions/rerun-epoch-proving-job.ts +120 -30
  60. package/src/actions/upload-epoch-proof-failure.ts +1 -1
  61. package/src/bin/run-failed-epoch.ts +5 -3
  62. package/src/checkpoint-store.ts +218 -0
  63. package/src/config.ts +39 -34
  64. package/src/factory.ts +86 -113
  65. package/src/index.ts +2 -0
  66. package/src/job/checkpoint-prover.ts +468 -0
  67. package/src/job/epoch-proving-job-data.ts +1 -1
  68. package/src/job/epoch-session.ts +436 -0
  69. package/src/job/top-tree-job.ts +227 -0
  70. package/src/metrics.ts +143 -93
  71. package/src/monitors/epoch-monitor.ts +6 -14
  72. package/src/proof-publishing-service.ts +427 -0
  73. package/src/prover-node-publisher.ts +264 -98
  74. package/src/prover-node.ts +568 -259
  75. package/src/prover-publisher-factory.ts +22 -11
  76. package/src/session-manager.ts +552 -0
  77. package/src/test/index.ts +6 -6
  78. package/dest/job/epoch-proving-job.d.ts +0 -62
  79. package/dest/job/epoch-proving-job.d.ts.map +0 -1
  80. package/dest/job/epoch-proving-job.js +0 -363
  81. package/src/job/epoch-proving-job.ts +0 -426
@@ -1,25 +1,409 @@
1
- function _ts_decorate(decorators, target, key, desc) {
2
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
- else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1
+ function applyDecs2203RFactory() {
2
+ function createAddInitializerMethod(initializers, decoratorFinishedRef) {
3
+ return function addInitializer(initializer) {
4
+ assertNotFinished(decoratorFinishedRef, "addInitializer");
5
+ assertCallable(initializer, "An initializer");
6
+ initializers.push(initializer);
7
+ };
8
+ }
9
+ function memberDec(dec, name, desc, initializers, kind, isStatic, isPrivate, metadata, value) {
10
+ var kindStr;
11
+ switch(kind){
12
+ case 1:
13
+ kindStr = "accessor";
14
+ break;
15
+ case 2:
16
+ kindStr = "method";
17
+ break;
18
+ case 3:
19
+ kindStr = "getter";
20
+ break;
21
+ case 4:
22
+ kindStr = "setter";
23
+ break;
24
+ default:
25
+ kindStr = "field";
26
+ }
27
+ var ctx = {
28
+ kind: kindStr,
29
+ name: isPrivate ? "#" + name : name,
30
+ static: isStatic,
31
+ private: isPrivate,
32
+ metadata: metadata
33
+ };
34
+ var decoratorFinishedRef = {
35
+ v: false
36
+ };
37
+ ctx.addInitializer = createAddInitializerMethod(initializers, decoratorFinishedRef);
38
+ var get, set;
39
+ if (kind === 0) {
40
+ if (isPrivate) {
41
+ get = desc.get;
42
+ set = desc.set;
43
+ } else {
44
+ get = function() {
45
+ return this[name];
46
+ };
47
+ set = function(v) {
48
+ this[name] = v;
49
+ };
50
+ }
51
+ } else if (kind === 2) {
52
+ get = function() {
53
+ return desc.value;
54
+ };
55
+ } else {
56
+ if (kind === 1 || kind === 3) {
57
+ get = function() {
58
+ return desc.get.call(this);
59
+ };
60
+ }
61
+ if (kind === 1 || kind === 4) {
62
+ set = function(v) {
63
+ desc.set.call(this, v);
64
+ };
65
+ }
66
+ }
67
+ ctx.access = get && set ? {
68
+ get: get,
69
+ set: set
70
+ } : get ? {
71
+ get: get
72
+ } : {
73
+ set: set
74
+ };
75
+ try {
76
+ return dec(value, ctx);
77
+ } finally{
78
+ decoratorFinishedRef.v = true;
79
+ }
80
+ }
81
+ function assertNotFinished(decoratorFinishedRef, fnName) {
82
+ if (decoratorFinishedRef.v) {
83
+ throw new Error("attempted to call " + fnName + " after decoration was finished");
84
+ }
85
+ }
86
+ function assertCallable(fn, hint) {
87
+ if (typeof fn !== "function") {
88
+ throw new TypeError(hint + " must be a function");
89
+ }
90
+ }
91
+ function assertValidReturnValue(kind, value) {
92
+ var type = typeof value;
93
+ if (kind === 1) {
94
+ if (type !== "object" || value === null) {
95
+ throw new TypeError("accessor decorators must return an object with get, set, or init properties or void 0");
96
+ }
97
+ if (value.get !== undefined) {
98
+ assertCallable(value.get, "accessor.get");
99
+ }
100
+ if (value.set !== undefined) {
101
+ assertCallable(value.set, "accessor.set");
102
+ }
103
+ if (value.init !== undefined) {
104
+ assertCallable(value.init, "accessor.init");
105
+ }
106
+ } else if (type !== "function") {
107
+ var hint;
108
+ if (kind === 0) {
109
+ hint = "field";
110
+ } else if (kind === 10) {
111
+ hint = "class";
112
+ } else {
113
+ hint = "method";
114
+ }
115
+ throw new TypeError(hint + " decorators must return a function or void 0");
116
+ }
117
+ }
118
+ function applyMemberDec(ret, base, decInfo, name, kind, isStatic, isPrivate, initializers, metadata) {
119
+ var decs = decInfo[0];
120
+ var desc, init, value;
121
+ if (isPrivate) {
122
+ if (kind === 0 || kind === 1) {
123
+ desc = {
124
+ get: decInfo[3],
125
+ set: decInfo[4]
126
+ };
127
+ } else if (kind === 3) {
128
+ desc = {
129
+ get: decInfo[3]
130
+ };
131
+ } else if (kind === 4) {
132
+ desc = {
133
+ set: decInfo[3]
134
+ };
135
+ } else {
136
+ desc = {
137
+ value: decInfo[3]
138
+ };
139
+ }
140
+ } else if (kind !== 0) {
141
+ desc = Object.getOwnPropertyDescriptor(base, name);
142
+ }
143
+ if (kind === 1) {
144
+ value = {
145
+ get: desc.get,
146
+ set: desc.set
147
+ };
148
+ } else if (kind === 2) {
149
+ value = desc.value;
150
+ } else if (kind === 3) {
151
+ value = desc.get;
152
+ } else if (kind === 4) {
153
+ value = desc.set;
154
+ }
155
+ var newValue, get, set;
156
+ if (typeof decs === "function") {
157
+ newValue = memberDec(decs, name, desc, initializers, kind, isStatic, isPrivate, metadata, value);
158
+ if (newValue !== void 0) {
159
+ assertValidReturnValue(kind, newValue);
160
+ if (kind === 0) {
161
+ init = newValue;
162
+ } else if (kind === 1) {
163
+ init = newValue.init;
164
+ get = newValue.get || value.get;
165
+ set = newValue.set || value.set;
166
+ value = {
167
+ get: get,
168
+ set: set
169
+ };
170
+ } else {
171
+ value = newValue;
172
+ }
173
+ }
174
+ } else {
175
+ for(var i = decs.length - 1; i >= 0; i--){
176
+ var dec = decs[i];
177
+ newValue = memberDec(dec, name, desc, initializers, kind, isStatic, isPrivate, metadata, value);
178
+ if (newValue !== void 0) {
179
+ assertValidReturnValue(kind, newValue);
180
+ var newInit;
181
+ if (kind === 0) {
182
+ newInit = newValue;
183
+ } else if (kind === 1) {
184
+ newInit = newValue.init;
185
+ get = newValue.get || value.get;
186
+ set = newValue.set || value.set;
187
+ value = {
188
+ get: get,
189
+ set: set
190
+ };
191
+ } else {
192
+ value = newValue;
193
+ }
194
+ if (newInit !== void 0) {
195
+ if (init === void 0) {
196
+ init = newInit;
197
+ } else if (typeof init === "function") {
198
+ init = [
199
+ init,
200
+ newInit
201
+ ];
202
+ } else {
203
+ init.push(newInit);
204
+ }
205
+ }
206
+ }
207
+ }
208
+ }
209
+ if (kind === 0 || kind === 1) {
210
+ if (init === void 0) {
211
+ init = function(instance, init) {
212
+ return init;
213
+ };
214
+ } else if (typeof init !== "function") {
215
+ var ownInitializers = init;
216
+ init = function(instance, init) {
217
+ var value = init;
218
+ for(var i = 0; i < ownInitializers.length; i++){
219
+ value = ownInitializers[i].call(instance, value);
220
+ }
221
+ return value;
222
+ };
223
+ } else {
224
+ var originalInitializer = init;
225
+ init = function(instance, init) {
226
+ return originalInitializer.call(instance, init);
227
+ };
228
+ }
229
+ ret.push(init);
230
+ }
231
+ if (kind !== 0) {
232
+ if (kind === 1) {
233
+ desc.get = value.get;
234
+ desc.set = value.set;
235
+ } else if (kind === 2) {
236
+ desc.value = value;
237
+ } else if (kind === 3) {
238
+ desc.get = value;
239
+ } else if (kind === 4) {
240
+ desc.set = value;
241
+ }
242
+ if (isPrivate) {
243
+ if (kind === 1) {
244
+ ret.push(function(instance, args) {
245
+ return value.get.call(instance, args);
246
+ });
247
+ ret.push(function(instance, args) {
248
+ return value.set.call(instance, args);
249
+ });
250
+ } else if (kind === 2) {
251
+ ret.push(value);
252
+ } else {
253
+ ret.push(function(instance, args) {
254
+ return value.call(instance, args);
255
+ });
256
+ }
257
+ } else {
258
+ Object.defineProperty(base, name, desc);
259
+ }
260
+ }
261
+ }
262
+ function applyMemberDecs(Class, decInfos, metadata) {
263
+ var ret = [];
264
+ var protoInitializers;
265
+ var staticInitializers;
266
+ var existingProtoNonFields = new Map();
267
+ var existingStaticNonFields = new Map();
268
+ for(var i = 0; i < decInfos.length; i++){
269
+ var decInfo = decInfos[i];
270
+ if (!Array.isArray(decInfo)) continue;
271
+ var kind = decInfo[1];
272
+ var name = decInfo[2];
273
+ var isPrivate = decInfo.length > 3;
274
+ var isStatic = kind >= 5;
275
+ var base;
276
+ var initializers;
277
+ if (isStatic) {
278
+ base = Class;
279
+ kind = kind - 5;
280
+ staticInitializers = staticInitializers || [];
281
+ initializers = staticInitializers;
282
+ } else {
283
+ base = Class.prototype;
284
+ protoInitializers = protoInitializers || [];
285
+ initializers = protoInitializers;
286
+ }
287
+ if (kind !== 0 && !isPrivate) {
288
+ var existingNonFields = isStatic ? existingStaticNonFields : existingProtoNonFields;
289
+ var existingKind = existingNonFields.get(name) || 0;
290
+ if (existingKind === true || existingKind === 3 && kind !== 4 || existingKind === 4 && kind !== 3) {
291
+ throw new Error("Attempted to decorate a public method/accessor that has the same name as a previously decorated public method/accessor. This is not currently supported by the decorators plugin. Property name was: " + name);
292
+ } else if (!existingKind && kind > 2) {
293
+ existingNonFields.set(name, kind);
294
+ } else {
295
+ existingNonFields.set(name, true);
296
+ }
297
+ }
298
+ applyMemberDec(ret, base, decInfo, name, kind, isStatic, isPrivate, initializers, metadata);
299
+ }
300
+ pushInitializers(ret, protoInitializers);
301
+ pushInitializers(ret, staticInitializers);
302
+ return ret;
303
+ }
304
+ function pushInitializers(ret, initializers) {
305
+ if (initializers) {
306
+ ret.push(function(instance) {
307
+ for(var i = 0; i < initializers.length; i++){
308
+ initializers[i].call(instance);
309
+ }
310
+ return instance;
311
+ });
312
+ }
313
+ }
314
+ function applyClassDecs(targetClass, classDecs, metadata) {
315
+ if (classDecs.length > 0) {
316
+ var initializers = [];
317
+ var newClass = targetClass;
318
+ var name = targetClass.name;
319
+ for(var i = classDecs.length - 1; i >= 0; i--){
320
+ var decoratorFinishedRef = {
321
+ v: false
322
+ };
323
+ try {
324
+ var nextNewClass = classDecs[i](newClass, {
325
+ kind: "class",
326
+ name: name,
327
+ addInitializer: createAddInitializerMethod(initializers, decoratorFinishedRef),
328
+ metadata
329
+ });
330
+ } finally{
331
+ decoratorFinishedRef.v = true;
332
+ }
333
+ if (nextNewClass !== undefined) {
334
+ assertValidReturnValue(10, nextNewClass);
335
+ newClass = nextNewClass;
336
+ }
337
+ }
338
+ return [
339
+ defineMetadata(newClass, metadata),
340
+ function() {
341
+ for(var i = 0; i < initializers.length; i++){
342
+ initializers[i].call(newClass);
343
+ }
344
+ }
345
+ ];
346
+ }
347
+ }
348
+ function defineMetadata(Class, metadata) {
349
+ return Object.defineProperty(Class, Symbol.metadata || Symbol.for("Symbol.metadata"), {
350
+ configurable: true,
351
+ enumerable: true,
352
+ value: metadata
353
+ });
354
+ }
355
+ return function applyDecs2203R(targetClass, memberDecs, classDecs, parentClass) {
356
+ if (parentClass !== void 0) {
357
+ var parentMetadata = parentClass[Symbol.metadata || Symbol.for("Symbol.metadata")];
358
+ }
359
+ var metadata = Object.create(parentMetadata === void 0 ? null : parentMetadata);
360
+ var e = applyMemberDecs(targetClass, memberDecs, metadata);
361
+ if (!classDecs.length) defineMetadata(targetClass, metadata);
362
+ return {
363
+ e: e,
364
+ get c () {
365
+ return applyClassDecs(targetClass, classDecs, metadata);
366
+ }
367
+ };
368
+ };
369
+ }
370
+ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
371
+ return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
6
372
  }
7
- import { BlockNumber } from '@aztec/foundation/branded-types';
8
- import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
373
+ var _initProto;
374
+ import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
375
+ import { assertRequired, compact, pick } from '@aztec/foundation/collection';
9
376
  import { memoize } from '@aztec/foundation/decorators';
10
377
  import { createLogger } from '@aztec/foundation/log';
11
- import { DateProvider } from '@aztec/foundation/timer';
378
+ import { RunningPromise } from '@aztec/foundation/running-promise';
379
+ import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
380
+ import { getLastSiblingPath } from '@aztec/prover-client/helpers';
381
+ import { ChonkCache } from '@aztec/prover-client/orchestrator';
12
382
  import { PublicProcessorFactory } from '@aztec/simulator/server';
13
- import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
383
+ import { EventDrivenL2BlockStream, L2TipsMemoryStore } from '@aztec/stdlib/block';
384
+ import { getEpochAtSlot, getProofSubmissionDeadlineEpoch } from '@aztec/stdlib/epoch-helpers';
14
385
  import { EpochProvingJobTerminalState, tryStop } from '@aztec/stdlib/interfaces/server';
15
- import { Attributes, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
386
+ import { MerkleTreeId } from '@aztec/stdlib/trees';
387
+ import { getTelemetryClient } from '@aztec/telemetry-client';
16
388
  import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
17
- import { EpochProvingJob } from './job/epoch-proving-job.js';
389
+ import { CheckpointStore } from './checkpoint-store.js';
18
390
  import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
391
+ import { ProofPublishingService } from './proof-publishing-service.js';
392
+ import { SessionManager } from './session-manager.js';
393
+ /**
394
+ * Grace period for the proof-publishing service to settle during shutdown. The service waits for
395
+ * any in-flight L1 proof-submission tx to finish; that tx can take a long time to mine, so we cap
396
+ * the wait rather than letting `stop()` hang indefinitely.
397
+ */ const PUBLISHING_SERVICE_STOP_TIMEOUT_MS = 30_000;
19
398
  /**
20
- * An Aztec Prover Node is a standalone process that monitors the unfinalized chain on L1 for unproven epochs,
21
- * fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
22
- * proof for the epoch, and submits it to L1.
399
+ * An Aztec Prover Node is a standalone process that monitors the chain for new checkpoints,
400
+ * starts proving them optimistically as they arrive, and submits epoch proofs to L1 once
401
+ * complete.
402
+ *
403
+ * The class is intentionally thin: it owns the long-lived collections (`CheckpointStore`,
404
+ * `ChonkCache`, `SessionManager`), the L2BlockStream, and a periodic ticker that nudges the
405
+ * manager to pick up newly-complete epochs. Every session lifecycle decision is delegated to
406
+ * the `SessionManager`. Each chain event is translated here into a single method call on it.
23
407
  */ export class ProverNode {
24
408
  prover;
25
409
  publisherFactory;
@@ -28,19 +412,46 @@ import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
28
412
  contractDataSource;
29
413
  worldState;
30
414
  p2pClient;
31
- epochsMonitor;
32
415
  rollupContract;
33
416
  l1Metrics;
417
+ avmSimulator;
34
418
  telemetryClient;
35
- log;
419
+ delayer;
36
420
  dateProvider;
37
- jobs;
421
+ static{
422
+ ({ e: [_initProto] } = _apply_decs_2203_r(this, [
423
+ [
424
+ memoize,
425
+ 2,
426
+ "getL1Constants"
427
+ ]
428
+ ], []));
429
+ }
430
+ log;
431
+ checkpointStore;
432
+ chonkCache;
433
+ sessionManager;
38
434
  config;
39
435
  jobMetrics;
40
436
  rewardsMetrics;
437
+ /** In-memory store for the L2BlockStream's local data provider. */ tipsStore;
438
+ /** Block stream for checkpoint and reorg detection. */ blockStream;
439
+ /**
440
+ * Highest epoch whose proof-submission window has passed. Monotonic high-water mark.
441
+ * Seeded from the last fully-proven epoch at start(); advanced on every block-stream
442
+ * event by comparing the archiver's latest synced L2 slot against each epoch's
443
+ * submission deadline. Protected so tests can verify the start() seeding.
444
+ */ lastExpiredEpoch;
445
+ /**
446
+ * Highest checkpoint number whose proving-side handling has completed (or that was legitimately skipped).
447
+ * The catch-up loop walks from here to each `chain-checkpointed` tip event. Seeded at start() from the last
448
+ * checkpoint of the last fully-proven epoch (or 0), so a restart reprocesses the partially-proven epoch rather
449
+ * than trusting a checkpointed tip that may sit ahead of unproven checkpoints. Clamped down on a prune.
450
+ */ lastProcessedCheckpoint;
451
+ /** Periodic tick that runs the epoch-expiry sweep during idle periods when no block-stream events arrive. */ expiryTicker;
41
452
  tracer;
42
- publisher;
43
- constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, epochsMonitor, rollupContract, l1Metrics, config = {}, telemetryClient = getTelemetryClient()){
453
+ publishingService;
454
+ constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, rollupContract, l1Metrics, avmSimulator, config = {}, telemetryClient = getTelemetryClient(), delayer, dateProvider = new DateProvider()){
44
455
  this.prover = prover;
45
456
  this.publisherFactory = publisherFactory;
46
457
  this.l2BlockSource = l2BlockSource;
@@ -48,17 +459,18 @@ import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
48
459
  this.contractDataSource = contractDataSource;
49
460
  this.worldState = worldState;
50
461
  this.p2pClient = p2pClient;
51
- this.epochsMonitor = epochsMonitor;
52
462
  this.rollupContract = rollupContract;
53
463
  this.l1Metrics = l1Metrics;
464
+ this.avmSimulator = avmSimulator;
54
465
  this.telemetryClient = telemetryClient;
55
- this.log = createLogger('prover-node');
56
- this.dateProvider = new DateProvider();
57
- this.jobs = new Map();
466
+ this.delayer = delayer;
467
+ this.dateProvider = dateProvider;
468
+ this.log = (_initProto(this), createLogger('prover-node'));
469
+ this.lastProcessedCheckpoint = CheckpointNumber.ZERO;
58
470
  this.config = {
59
471
  proverNodePollingIntervalMs: 1_000,
60
472
  proverNodeMaxPendingJobs: 100,
61
- proverNodeMaxParallelBlocksPerEpoch: 32,
473
+ proverNodeMaxParallelBlocksPerEpoch: 0,
62
474
  txGatheringIntervalMs: 1_000,
63
475
  txGatheringBatchSize: 10,
64
476
  txGatheringMaxParallelRequestsPerNode: 100,
@@ -72,6 +484,22 @@ import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
72
484
  this.tracer = telemetryClient.getTracer('ProverNode');
73
485
  this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
74
486
  this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
487
+ this.tipsStore = new L2TipsMemoryStore(this.l2BlockSource.getGenesisBlockHash());
488
+ this.chonkCache = new ChonkCache(this.log.getBindings());
489
+ this.checkpointStore = new CheckpointStore(this.l2BlockSource, {
490
+ proverFactory: this.prover,
491
+ chonkCache: this.chonkCache,
492
+ publicProcessorFactory: new PublicProcessorFactory(this.contractDataSource, this.avmSimulator, this.dateProvider, this.telemetryClient, this.log.getBindings()),
493
+ dbProvider: this.worldState,
494
+ txProvider: this.p2pClient.getTxProvider(),
495
+ dateProvider: this.dateProvider,
496
+ proverId: this.prover.getProverId(),
497
+ metrics: this.jobMetrics,
498
+ txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
499
+ deadline: undefined
500
+ }, {
501
+ slotWatcherPollIntervalMs: this.config.proverNodePollingIntervalMs
502
+ }, this.log.getBindings());
75
503
  }
76
504
  getProverId() {
77
505
  return this.prover.getProverId();
@@ -79,254 +507,469 @@ import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
79
507
  getP2P() {
80
508
  return this.p2pClient;
81
509
  }
510
+ /** Test-only: the shared L1 tx delayer, if enabled. */ getDelayer() {
511
+ return this.delayer;
512
+ }
513
+ /** Observability summary for the ProverNodeApi. */ getJobs() {
514
+ return Promise.resolve(this.sessionManager?.getJobs() ?? []);
515
+ }
516
+ /** Tests inspect this when validating reconcile behaviour. */ getCheckpointStore() {
517
+ return this.checkpointStore;
518
+ }
519
+ /** Tests inspect this to verify chonk-cache release semantics. */ getChonkCache() {
520
+ return this.chonkCache;
521
+ }
522
+ /** Tests inspect this when looking up live sessions. */ getSessionManager() {
523
+ if (!this.sessionManager) {
524
+ throw new Error('SessionManager not yet constructed — start() must be called first.');
525
+ }
526
+ return this.sessionManager;
527
+ }
528
+ /** Returns the underlying prover instance. */ getProver() {
529
+ return this.prover;
530
+ }
531
+ // ---------------- L2BlockStream handler ----------------
532
+ async handleBlockStreamEvent(event) {
533
+ switch(event.type){
534
+ case 'chain-checkpointed':
535
+ await this.processCheckpointJump(event.checkpoint.number);
536
+ break;
537
+ case 'chain-pruned':
538
+ await this.handlePruneEvent(event.block);
539
+ break;
540
+ case 'chain-proven':
541
+ this.publishingService?.onChainProven(BlockNumber(event.block.number));
542
+ break;
543
+ // The proposed tip drives only the tips store's walk-back history (recorded below); the prover-node
544
+ // tracks checkpoints, not proposed blocks. `blocks-added` is never emitted in tips-only mode, and
545
+ // `chain-finalized` carries nothing the prover-node acts on.
546
+ case 'chain-proposed':
547
+ case 'chain-finalized':
548
+ case 'blocks-added':
549
+ break;
550
+ default:
551
+ {
552
+ const _ = event;
553
+ break;
554
+ }
555
+ }
556
+ // Expiry is driven by the archiver's latest synced L2 slot
557
+ await this.checkEpochExpiry();
558
+ // Advance the local tips store only after the proving-side handling has succeeded. Any
559
+ // failure above propagates to the L2BlockStream (which logs and stops this poll pass) and
560
+ // skips this update, so the event is re-emitted on the next poll rather than skipped (A-1041).
561
+ await this.tipsStore.handleBlockStreamEvent(event);
562
+ }
82
563
  /**
83
- * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
84
- * @param epochNumber - The epoch number that was just completed.
85
- * @returns false if there is an error, true otherwise
86
- */ async handleEpochReadyToProve(epochNumber) {
87
- try {
88
- this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
89
- jobs: Array.from(this.jobs.values()).map((job)=>`${job.getEpochNumber()}:${job.getId()}`)
564
+ * Walks every checkpoint between the local cursor and the newly-reported checkpointed tip, registering
565
+ * each one that belongs to an epoch that can still be proven. The block stream now delivers a single thin
566
+ * `chain-checkpointed` tip event per pass rather than one fat event per checkpoint, so this drives the
567
+ * catch-up itself: light metadata first (`getCheckpointsData`) to decide relevance per epoch, then a heavy
568
+ * `getCheckpoints` fetch only for checkpoints in provable epochs.
569
+ *
570
+ * The cursor advances one checkpoint at a time and only after that checkpoint's proving-side handling has
571
+ * fully succeeded, preserving the A-1041 at-least-once semantics: a mid-jump failure leaves the cursor
572
+ * behind so the next pass retries from the first checkpoint that did not complete.
573
+ */ async processCheckpointJump(targetCheckpoint) {
574
+ if (targetCheckpoint <= this.lastProcessedCheckpoint) {
575
+ return;
576
+ }
577
+ const l1Constants = await this.getL1Constants();
578
+ // Cap the catch-up at the `(proofSubmissionEpochs + 1) * epochDuration` most recent checkpoints.
579
+ // When the cursor is much further behind (e.g. resyncing after a long time offline), fetching the whole gap could
580
+ // load thousands of checkpoints we cannot act on: anything older than the last two epochs is already past
581
+ // its proof-submission window, so we skip it and jump the cursor forward to the start of the capped range.
582
+ const maxCheckpoints = (l1Constants.proofSubmissionEpochs + 1) * l1Constants.epochDuration;
583
+ let from = CheckpointNumber(this.lastProcessedCheckpoint + 1);
584
+ if (Number(targetCheckpoint - from) + 1 > maxCheckpoints) {
585
+ const cappedFrom = CheckpointNumber(targetCheckpoint - maxCheckpoints + 1);
586
+ this.log.warn(`Skipping unprovable checkpoints during catch-up; the prover node is far behind`, {
587
+ from,
588
+ cappedFrom,
589
+ targetCheckpoint,
590
+ maxCheckpoints
90
591
  });
91
- const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
92
- if (activeJobs.length > 0) {
93
- this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
94
- activeJobs: activeJobs.map((job)=>job.uuid)
95
- });
96
- return true;
592
+ // Advance the cursor past the skipped checkpoints so they are never retried.
593
+ this.lastProcessedCheckpoint = CheckpointNumber(cappedFrom - 1);
594
+ from = cappedFrom;
595
+ }
596
+ const limit = Number(targetCheckpoint - from) + 1;
597
+ const metadatas = await this.l2BlockSource.getCheckpointsData({
598
+ from,
599
+ limit
600
+ });
601
+ // Per-epoch relevance is cached so a multi-checkpoint epoch resolves it once. Skipping is whole-epoch
602
+ // only: the SessionManager requires an epoch's checkpoints fully covered before it opens a session, so we
603
+ // never drop an individual checkpoint inside an epoch we will prove.
604
+ const epochSkippable = new Map();
605
+ for (const metadata of metadatas){
606
+ const epochNumber = getEpochAtSlot(metadata.header.slotNumber, l1Constants);
607
+ let skippable = epochSkippable.get(epochNumber);
608
+ if (skippable === undefined) {
609
+ skippable = await this.isEpochFullyProven(epochNumber, l1Constants) || await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants);
610
+ epochSkippable.set(epochNumber, skippable);
97
611
  }
98
- await this.startProof(epochNumber);
99
- return true;
100
- } catch (err) {
101
- if (err instanceof EmptyEpochError) {
102
- this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
612
+ if (skippable) {
613
+ this.log.debug(`Skipping checkpoint ${metadata.checkpointNumber} for unprovable epoch ${epochNumber}`);
103
614
  } else {
104
- this.log.error(`Error handling epoch completed`, err);
615
+ await this.registerCheckpoint(metadata.checkpointNumber, epochNumber);
616
+ }
617
+ // Advance only after the checkpoint's handling succeeded (or it was legitimately skipped). registerCheckpoint
618
+ // throws on failure, which leaves the cursor here for the next pass to retry (A-1041).
619
+ this.lastProcessedCheckpoint = metadata.checkpointNumber;
620
+ }
621
+ }
622
+ /** Heavy-fetch a single checkpoint, register it with the store, and notify the session manager. */ async registerCheckpoint(checkpointNumber, epochNumber) {
623
+ const published = await this.l2BlockSource.getCheckpoint({
624
+ number: checkpointNumber
625
+ });
626
+ if (!published) {
627
+ throw new Error(`Checkpoint ${checkpointNumber} not found in block source during catch-up`);
628
+ }
629
+ const checkpoint = published.checkpoint;
630
+ this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
631
+ checkpointNumber: checkpoint.number,
632
+ epochNumber,
633
+ slotNumber: checkpoint.header.slotNumber
634
+ });
635
+ const registerData = await this.collectRegisterData(checkpoint, published.attestations);
636
+ await this.checkpointStore.addOrUpdate(checkpoint, registerData);
637
+ await this.sessionManager?.onCheckpointAdded(epochNumber);
638
+ // Tips-only mode delivers no blocks, so record one witness per checkpointed block: a reorg into the checkpoint's
639
+ // range then prunes at the true divergence instead of the nearest sparse tip anchor.
640
+ await this.tipsStore.recordBlockHashes(await Promise.all(checkpoint.blocks.map(async (block)=>({
641
+ number: block.number,
642
+ hash: (await block.header.hash()).toString()
643
+ }))));
644
+ }
645
+ /**
646
+ * Gathers register-time data for a checkpoint: previous block header, L1-to-L2 messages,
647
+ * and the archive sibling path.
648
+ */ async collectRegisterData(checkpoint, attestations) {
649
+ const previousBlockNumber = BlockNumber(checkpoint.blocks[0].number - 1);
650
+ const previousBlockHeader = await this.gatherPreviousBlockHeader(previousBlockNumber);
651
+ const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpoint.number);
652
+ const lastBlock = checkpoint.blocks.at(-1);
653
+ const lastBlockHash = await lastBlock.header.hash();
654
+ await this.worldState.syncImmediate(lastBlock.number, lastBlockHash);
655
+ const previousArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, this.worldState.getSnapshot(previousBlockNumber));
656
+ return {
657
+ attestations,
658
+ previousBlockHeader,
659
+ l1ToL2Messages,
660
+ previousArchiveSiblingPath
661
+ };
662
+ }
663
+ /**
664
+ * Marks every prover orphaned by the prune as pruned, clamps the catch-up cursor below the prune target's
665
+ * checkpoint, and notifies the session manager. Keyed off the prune target block (the highest surviving block)
666
+ * rather than the source's checkpointed tip, which can sit above the target after a re-checkpoint and would leave
667
+ * orphaned provers canonical. Throws (rather than warning) if the cursor floor cannot be resolved, so the pass
668
+ * fails and the prune is retried next iteration.
669
+ */ async handlePruneEvent(prunedToBlock) {
670
+ this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, {
671
+ prunedToBlock
672
+ });
673
+ // Resolve the cursor floor BEFORE marking provers: markPrunedAboveBlock returns only newly-marked provers, so a
674
+ // throw after marking would leave a retry pass with nothing to act on. Resolving first means a throw leaves
675
+ // everything untouched and the next pass retries the whole handler (the tips cursor only advances on success).
676
+ let cursorFloor;
677
+ if (prunedToBlock.number === 0) {
678
+ cursorFloor = CheckpointNumber.ZERO;
679
+ } else {
680
+ const targetData = await this.l2BlockSource.getBlockData({
681
+ number: prunedToBlock.number
682
+ });
683
+ if (targetData === undefined) {
684
+ throw new Error(`No block data found for prune target block ${prunedToBlock.number}; cannot clamp checkpoint cursor`);
105
685
  }
686
+ // Clamp to `cpAtTarget - 1`: a mid-checkpoint target leaves that checkpoint partially orphaned and it must be
687
+ // reprocessed. Over-clamping merely re-registers a checkpoint (at-least-once by design — A-1041); under-clamping
688
+ // would permanently skip a rebuilt same-number checkpoint.
689
+ cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1));
690
+ }
691
+ const affected = this.checkpointStore.markPrunedAboveBlock(prunedToBlock.number);
692
+ if (this.lastProcessedCheckpoint > cursorFloor) {
693
+ this.lastProcessedCheckpoint = cursorFloor;
694
+ }
695
+ if (affected.length === 0) {
696
+ return;
697
+ }
698
+ const l1Constants = await this.getL1Constants();
699
+ const affectedEpochs = Array.from(new Set(affected.map((p)=>Number(getEpochAtSlot(p.slotNumber, l1Constants))))).map((n)=>EpochNumber(n));
700
+ // The session manager cancels every affected session, which in turn calls
701
+ // publishingService.withdraw(uuid) for each candidate; no separate notification to the
702
+ // publishing service is needed.
703
+ await this.sessionManager?.onPrune(affectedEpochs);
704
+ }
705
+ /**
706
+ * Returns true once the chain has advanced past the given epoch's proof-submission window.
707
+ * Used to ignore checkpoints whose epoch can no longer be proven in time — chiefly while the
708
+ * archiver replays old blocks after a restart. Compares the archiver's latest synced L2 slot
709
+ * against the epoch's submission-deadline epoch; conservatively returns false if the slot can't
710
+ * be read yet.
711
+ */ async isEpochPastProofSubmissionWindow(epochNumber, l1Constants) {
712
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
713
+ if (latestSlot === undefined) {
106
714
  return false;
107
715
  }
716
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
717
+ return latestEpoch >= getProofSubmissionDeadlineEpoch(epochNumber, l1Constants);
718
+ }
719
+ /**
720
+ * Compares the archiver's latest synced L2 slot against `lastExpiredEpoch` and, for each
721
+ * newly-expired epoch, releases the chonk-cache entries for its blocks and reaps any
722
+ * CheckpointProvers in the store. An epoch E is expired once the chain reaches the start
723
+ * of epoch `E + proofSubmissionEpochs + 1`. Silently no-ops if nothing has expired since
724
+ * the last check or the archiver's slot can't be read.
725
+ */ async checkEpochExpiry() {
726
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
727
+ if (latestSlot === undefined) {
728
+ return;
729
+ }
730
+ const l1Constants = await this.getL1Constants();
731
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
732
+ const offset = l1Constants.proofSubmissionEpochs + 1;
733
+ if (latestEpoch < offset) {
734
+ return;
735
+ }
736
+ const newlyExpiredUpTo = EpochNumber(latestEpoch - offset);
737
+ const from = this.lastExpiredEpoch === undefined ? EpochNumber(0) : EpochNumber(this.lastExpiredEpoch + 1);
738
+ if (newlyExpiredUpTo < from) {
739
+ return;
740
+ }
741
+ for(let e = from; e <= newlyExpiredUpTo; e = EpochNumber(e + 1)){
742
+ await this.expireEpoch(e);
743
+ }
744
+ this.lastExpiredEpoch = newlyExpiredUpTo;
108
745
  }
109
746
  /**
110
- * Starts the prover node so it periodically checks for unproven epochs in the unfinalized chain from L1 and
111
- * starts proving jobs for them.
112
- */ async start() {
113
- this.epochsMonitor.start(this);
747
+ * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and
748
+ * reaps every CheckpointProver in the store whose epoch number matches.
749
+ */ async expireEpoch(epoch) {
750
+ try {
751
+ const blocks = await this.l2BlockSource.getBlocks({
752
+ epoch,
753
+ onlyCheckpointed: true
754
+ });
755
+ if (blocks.length > 0) {
756
+ this.chonkCache.releaseForBlocks(blocks);
757
+ }
758
+ } catch (err) {
759
+ this.log.warn(`Could not release chonk-cache entries for expired epoch ${epoch}`, err);
760
+ }
761
+ this.checkpointStore.reapExpired(epoch);
762
+ }
763
+ // ---------------- public API ----------------
764
+ /**
765
+ * Schedules proving for the given epoch and returns the job id without waiting for completion.
766
+ */ async startProof(epochNumber) {
767
+ if (!this.sessionManager) {
768
+ throw new Error('ProverNode not started');
769
+ }
770
+ return await this.sessionManager.startProof(epochNumber);
771
+ }
772
+ // ---------------- Service lifecycle ----------------
773
+ async start() {
774
+ await this.checkpointStore.start();
114
775
  await this.publisherFactory.start();
115
- this.publisher = await this.publisherFactory.create();
776
+ this.publishingService = new ProofPublishingService({
777
+ publisherFactory: this.publisherFactory,
778
+ l2BlockSource: this.l2BlockSource,
779
+ dateProvider: this.dateProvider,
780
+ config: {
781
+ skipSubmitProof: !!this.config.proverNodeDisableProofPublish
782
+ },
783
+ bindings: this.log.getBindings()
784
+ });
785
+ this.sessionManager = this.createSessionManager(this.publishingService);
786
+ // SessionManager owns its own periodic tick; start it here so it begins picking up
787
+ // epochs that become complete by time (no fresh checkpoint event) and advances once
788
+ // the previous epoch is proven on L1.
789
+ this.sessionManager.start();
790
+ // Now that the store + manager exist, arm the live-state observable gauges.
791
+ this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
792
+ const { lastFullyProvenEpoch } = await this.resolveLastFullyProvenEpoch();
793
+ this.lastExpiredEpoch = lastFullyProvenEpoch;
794
+ this.lastProcessedCheckpoint = await this.computeStartingCheckpoint(lastFullyProvenEpoch);
795
+ this.blockStream = new EventDrivenL2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
796
+ pollIntervalMS: this.config.proverNodePollingIntervalMs,
797
+ tipsOnly: true
798
+ });
799
+ this.blockStream.start();
800
+ // With thin once-per-pass tip events, the expiry sweep no longer fires once per checkpoint; drive it
801
+ // from a periodic tick so epochs still expire during idle/no-event periods.
802
+ this.expiryTicker = new RunningPromise(()=>this.checkEpochExpiry(), this.log, this.config.proverNodePollingIntervalMs);
803
+ this.expiryTicker.start();
116
804
  await this.rewardsMetrics.start();
117
805
  this.l1Metrics.start();
118
806
  this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
119
807
  }
120
- /**
121
- * Stops the prover node and all its dependencies.
122
- */ async stop() {
808
+ async stop() {
123
809
  this.log.info('Stopping ProverNode');
124
- await this.epochsMonitor.stop();
810
+ this.jobMetrics.stopObservingState();
811
+ await this.blockStream?.stop();
812
+ await this.expiryTicker?.stop();
813
+ if (this.sessionManager) {
814
+ await this.sessionManager.stop();
815
+ }
816
+ if (this.publishingService) {
817
+ // Bound the wait: the publishing service blocks until any in-flight L1 proof-submission tx
818
+ // settles, which can outlast a reasonable shutdown window. On timeout we log and move on —
819
+ // the tx may still mine, but shutdown must not hang on it.
820
+ const publishingService = this.publishingService;
821
+ await executeTimeout(()=>publishingService.stop(), PUBLISHING_SERVICE_STOP_TIMEOUT_MS, 'prover-node publishing-service stop').catch((err)=>this.log.warn(`Timed out stopping proof publishing service`, err));
822
+ }
823
+ await this.checkpointStore.stop();
824
+ this.chonkCache.stop();
125
825
  await this.prover.stop();
126
- await tryStop(this.p2pClient);
127
- await tryStop(this.l2BlockSource);
128
826
  await tryStop(this.publisherFactory);
129
- this.publisher?.interrupt();
130
- await Promise.all(Array.from(this.jobs.values()).map((job)=>job.stop()));
131
- await this.worldState.stop();
132
827
  this.rewardsMetrics.stop();
133
828
  this.l1Metrics.stop();
134
829
  await this.telemetryClient.stop();
135
830
  this.log.info('Stopped ProverNode');
136
831
  }
137
- /** Returns world state status. */ async getWorldStateSyncStatus() {
138
- const { syncSummary } = await this.worldState.status();
139
- return syncSummary;
140
- }
141
- /** Returns archiver status. */ getL2Tips() {
142
- return this.l2BlockSource.getL2Tips();
143
- }
144
832
  /**
145
- * Starts a proving process and returns immediately.
146
- */ async startProof(epochNumber) {
147
- const job = await this.createProvingJob(epochNumber, {
148
- skipEpochCheck: true
833
+ * Constructs the session manager. Extracted so subclasses (test harness) can swap
834
+ * the implementation. Wired to `tryUploadSessionFailure` so failed sessions get
835
+ * their proving data uploaded.
836
+ */ createSessionManager(publishingService) {
837
+ return new SessionManager({
838
+ checkpointStore: this.checkpointStore,
839
+ l2BlockSource: this.l2BlockSource,
840
+ proverFactory: this.prover,
841
+ proverId: this.prover.getProverId(),
842
+ publishingService,
843
+ metrics: this.jobMetrics,
844
+ dateProvider: this.dateProvider,
845
+ config: {
846
+ maxPendingJobs: this.config.proverNodeMaxPendingJobs,
847
+ tickIntervalMs: this.config.proverNodePollingIntervalMs,
848
+ finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs
849
+ },
850
+ onSessionFailed: async (session)=>{
851
+ await this.tryUploadSessionFailure(session);
852
+ },
853
+ bindings: this.log.getBindings()
149
854
  });
150
- void this.runJob(job);
151
855
  }
152
- async runJob(job) {
153
- const epochNumber = job.getEpochNumber();
154
- const ctx = {
155
- id: job.getId(),
156
- epochNumber,
157
- state: undefined
158
- };
159
- try {
160
- await job.run();
161
- const state = job.getState();
162
- ctx.state = state;
163
- if (state === 'reorg') {
164
- this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
165
- await this.createProvingJob(epochNumber);
166
- } else if (state === 'failed') {
167
- this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
168
- await this.tryUploadEpochFailure(job);
169
- } else {
170
- this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
171
- }
172
- } catch (err) {
173
- this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
174
- } finally{
175
- this.jobs.delete(job.getId());
856
+ /**
857
+ * Installs session hooks for the e2e harness to interpose around top-tree proving
858
+ * (gate, override, or observe it) without monkey-patching the orchestrator factory.
859
+ * Applies to every session constructed after this call.
860
+ */ setSessionHooks(hooks) {
861
+ if (!this.sessionManager) {
862
+ throw new Error('ProverNode not started; call start() before setting session hooks.');
176
863
  }
864
+ this.sessionManager.setSessionHooks(hooks);
177
865
  }
178
- async tryUploadEpochFailure(job) {
179
- if (this.config.proverNodeFailedEpochStore) {
180
- return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, job.getId(), job.getProvingData(), this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
866
+ /** Uploads failure snapshots when sessions exit with `failed`. Exposed as a method so tests can spy on it. */ async tryUploadSessionFailure(session) {
867
+ if (!this.config.proverNodeFailedEpochStore) {
868
+ return undefined;
181
869
  }
870
+ const data = SessionManager.buildSessionProvingData(session);
871
+ return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, session.getId(), data, this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
182
872
  }
183
- /**
184
- * Returns the prover instance.
185
- */ getProver() {
186
- return this.prover;
873
+ // ---------------- helpers ----------------
874
+ getL1Constants() {
875
+ return this.l2BlockSource.getL1Constants();
187
876
  }
188
877
  /**
189
- * Returns an array of jobs being processed.
190
- */ getJobs() {
191
- return Promise.resolve(Array.from(this.jobs.entries()).map(([uuid, job])=>({
192
- uuid,
193
- status: job.getState(),
194
- epochNumber: job.getEpochNumber()
195
- })));
196
- }
197
- async getActiveJobsForEpoch(epochNumber) {
198
- const jobs = await this.getJobs();
199
- return jobs.filter((job)=>job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
200
- }
201
- checkMaximumPendingJobs() {
202
- const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
203
- if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
204
- throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
205
- }
206
- }
207
- async createProvingJob(epochNumber, opts = {}) {
208
- this.checkMaximumPendingJobs();
209
- this.publisher = await this.publisherFactory.create();
210
- // Gather all data for this epoch
211
- const epochData = await this.gatherEpochData(epochNumber);
212
- const fromCheckpoint = epochData.checkpoints[0].number;
213
- const toCheckpoint = epochData.checkpoints.at(-1).number;
214
- const fromBlock = epochData.checkpoints[0].blocks[0].number;
215
- const toBlock = epochData.checkpoints.at(-1).blocks.at(-1).number;
216
- this.log.verbose(`Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`);
217
- // Fast forward world state to right before the target block and get a fork
218
- await this.worldState.syncImmediate(toBlock);
219
- // Create a processor factory
220
- const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.dateProvider, this.telemetryClient);
221
- // Set deadline for this job to run. It will abort if it takes too long.
222
- const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
223
- const deadline = new Date(Number(deadlineTs) * 1000);
224
- const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
225
- this.jobs.set(job.getId(), job);
226
- return job;
878
+ * Returns true if every block in the given epoch is proven on L1. An epoch is only
879
+ * fully proven when its *last* block is proven. Protected for direct unit-test access.
880
+ */ async isEpochFullyProven(epochNumber, l1Constants) {
881
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
882
+ tag: 'proven'
883
+ });
884
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
885
+ return false;
886
+ }
887
+ const provenHeader = (await this.l2BlockSource.getBlockData({
888
+ number: BlockNumber(provenBlockNumber)
889
+ }))?.header;
890
+ if (!provenHeader) {
891
+ return false;
892
+ }
893
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
894
+ if (epochNumber < provenEpoch) {
895
+ return true;
896
+ }
897
+ if (epochNumber > provenEpoch) {
898
+ return false;
899
+ }
900
+ return this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants);
227
901
  }
228
- getL1Constants() {
229
- return this.l2BlockSource.getL1Constants();
902
+ /** Protected for direct unit-test access. */ async isProvenBlockLastOfItsEpoch(provenBlockNumber, provenEpoch, l1Constants) {
903
+ const nextHeader = (await this.l2BlockSource.getBlockData({
904
+ number: BlockNumber(provenBlockNumber + 1)
905
+ }))?.header;
906
+ if (nextHeader) {
907
+ return getEpochAtSlot(nextHeader.getSlot(), l1Constants) > provenEpoch;
908
+ }
909
+ return this.l2BlockSource.isEpochComplete(provenEpoch);
230
910
  }
231
- async gatherEpochData(epochNumber) {
232
- const checkpoints = await this.gatherCheckpoints(epochNumber);
233
- const txArray = await this.gatherTxs(epochNumber, checkpoints);
234
- const txs = new Map(txArray.map((tx)=>[
235
- tx.getTxHash().toString(),
236
- tx
237
- ]));
238
- const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
239
- const [firstBlock] = checkpoints[0].blocks;
240
- const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
241
- const [lastPublishedCheckpoint] = await this.l2BlockSource.getPublishedCheckpoints(checkpoints.at(-1).number, 1);
242
- const attestations = lastPublishedCheckpoint?.attestations ?? [];
911
+ /**
912
+ * Resolves the last fully-proven epoch from L1 proven state, used to seed the catch-up cursor (via
913
+ * `computeStartingCheckpoint`) and `lastExpiredEpoch`. The fully-proven epoch is `provenEpoch` when the
914
+ * proven tip is the last block of its epoch, otherwise `provenEpoch - 1`, or `undefined` if no block is
915
+ * proven yet (so a restart reprocesses the partially-proven epoch rather than trusting a stale tip).
916
+ */ async resolveLastFullyProvenEpoch() {
917
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
918
+ tag: 'proven'
919
+ });
920
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
921
+ return {
922
+ lastFullyProvenEpoch: undefined
923
+ };
924
+ }
925
+ const l1Constants = await this.getL1Constants();
926
+ const provenHeader = (await this.l2BlockSource.getBlockData({
927
+ number: BlockNumber(provenBlockNumber)
928
+ }))?.header;
929
+ if (!provenHeader) {
930
+ return {
931
+ lastFullyProvenEpoch: undefined
932
+ };
933
+ }
934
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
935
+ if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
936
+ return {
937
+ lastFullyProvenEpoch: provenEpoch
938
+ };
939
+ }
940
+ const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
243
941
  return {
244
- checkpoints,
245
- txs,
246
- l1ToL2Messages,
247
- epochNumber,
248
- previousBlockHeader,
249
- attestations
942
+ lastFullyProvenEpoch
250
943
  };
251
944
  }
252
- async gatherCheckpoints(epochNumber) {
253
- const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
254
- if (checkpoints.length === 0) {
255
- throw new EmptyEpochError(epochNumber);
256
- }
257
- return checkpoints;
258
- }
259
- async gatherTxs(epochNumber, checkpoints) {
260
- const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
261
- const txProvider = this.p2pClient.getTxProvider();
262
- const blocks = checkpoints.flatMap((checkpoint)=>checkpoint.blocks);
263
- const txsByBlock = await Promise.all(blocks.map((block)=>txProvider.getTxsForBlock(block, {
264
- deadline
265
- })));
266
- const txs = txsByBlock.map(({ txs })=>txs).flat();
267
- const missingTxs = txsByBlock.map(({ missingTxs })=>missingTxs).flat();
268
- if (missingTxs.length === 0) {
269
- this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, {
270
- epochNumber
271
- });
272
- return txs;
945
+ /**
946
+ * Resolves the catch-up cursor seed: the last checkpoint of the last fully-proven epoch, or 0 if none. Seeding
947
+ * from a checkpoint (rather than a checkpointed tip) guarantees a restart reprocesses every checkpoint of the
948
+ * partially-proven epoch, since the checkpointed tip can sit ahead of the last fully-proven checkpoint.
949
+ */ async computeStartingCheckpoint(lastFullyProvenEpoch) {
950
+ if (lastFullyProvenEpoch === undefined) {
951
+ return CheckpointNumber.ZERO;
273
952
  }
274
- throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxs.map((hash)=>hash.toString()).join(', ')}`);
275
- }
276
- async gatherMessages(epochNumber, checkpoints) {
277
- const messages = await Promise.all(checkpoints.map((c)=>this.l1ToL2MessageSource.getL1ToL2MessagesForCheckpoint(c.number)));
278
- const messageCount = sum(messages.map((m)=>m.length));
279
- this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, {
280
- epochNumber
281
- });
282
- const messagesByCheckpoint = {};
283
- for(let i = 0; i < checkpoints.length; i++){
284
- messagesByCheckpoint[checkpoints[i].number] = messages[i];
285
- }
286
- return messagesByCheckpoint;
287
- }
288
- async gatherPreviousBlockHeader(epochNumber, previousBlockNumber) {
289
- const header = await (previousBlockNumber === 0 ? this.worldState.getCommitted().getInitialHeader() : this.l2BlockSource.getBlockHeader(BlockNumber(previousBlockNumber)));
290
- if (!header) {
291
- throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
292
- }
293
- this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
294
- return header;
295
- }
296
- /** Extracted for testing purposes. */ doCreateEpochProvingJob(data, deadline, publicProcessorFactory, publisher, opts = {}) {
297
- const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config;
298
- return new EpochProvingJob(data, this.worldState, this.prover.createEpochProver(), publicProcessorFactory, publisher, this.l2BlockSource, this.jobMetrics, deadline, {
299
- parallelBlockLimit,
300
- skipSubmitProof: proverNodeDisableProofPublish,
301
- ...opts
953
+ const checkpoints = await this.l2BlockSource.getCheckpointsData({
954
+ epoch: lastFullyProvenEpoch
302
955
  });
956
+ return checkpoints.at(-1)?.checkpointNumber ?? CheckpointNumber.ZERO;
303
957
  }
304
- /** Extracted for testing purposes. */ async triggerMonitors() {
305
- await this.epochsMonitor.work();
958
+ async gatherPreviousBlockHeader(previousBlockNumber) {
959
+ const data = await this.l2BlockSource.getBlockData({
960
+ number: BlockNumber(previousBlockNumber)
961
+ });
962
+ if (!data?.header) {
963
+ throw new Error(`Previous block header ${previousBlockNumber} not found`);
964
+ }
965
+ return data.header;
306
966
  }
307
967
  validateConfig() {
308
- if (this.config.proverNodeFailedEpochStore && (!this.config.dataDirectory || !this.config.l1ChainId || !this.config.rollupVersion)) {
968
+ if (this.config.proverNodeFailedEpochStore && (!this.config.dataDirectory || !this.config.l1ChainId || this.config.rollupVersion === undefined)) {
309
969
  this.log.warn(`Invalid prover-node config (missing dataDirectory, l1ChainId, or rollupVersion)`, pick(this.config, 'proverNodeFailedEpochStore', 'dataDirectory', 'l1ChainId', 'rollupVersion'));
310
970
  throw new Error('All of dataDirectory, l1ChainId, and rollupVersion are required if proverNodeFailedEpochStore is set.');
311
971
  }
312
972
  }
313
973
  }
314
- _ts_decorate([
315
- trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
316
- [Attributes.EPOCH_NUMBER]: epochNumber
317
- }))
318
- ], ProverNode.prototype, "createProvingJob", null);
319
- _ts_decorate([
320
- memoize
321
- ], ProverNode.prototype, "getL1Constants", null);
322
- _ts_decorate([
323
- trackSpan('ProverNode.gatherEpochData', (epochNumber)=>({
324
- [Attributes.EPOCH_NUMBER]: epochNumber
325
- }))
326
- ], ProverNode.prototype, "gatherEpochData", null);
327
- class EmptyEpochError extends Error {
328
- constructor(epochNumber){
329
- super(`No blocks found for epoch ${epochNumber}`);
330
- this.name = 'EmptyEpochError';
331
- }
332
- }
974
+ // Re-export so handlers can compare states externally.
975
+ export { EpochProvingJobTerminalState };