@aztec-labs/prover-node 6.0.0-nightly.20260829
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +573 -0
- package/dest/actions/download-epoch-proving-job.d.ts +18 -0
- package/dest/actions/download-epoch-proving-job.d.ts.map +1 -0
- package/dest/actions/download-epoch-proving-job.js +37 -0
- package/dest/actions/index.d.ts +3 -0
- package/dest/actions/index.d.ts.map +1 -0
- package/dest/actions/index.js +2 -0
- package/dest/actions/rerun-epoch-proving-job.d.ts +23 -0
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -0
- package/dest/actions/rerun-epoch-proving-job.js +264 -0
- package/dest/actions/upload-epoch-proof-failure.d.ts +15 -0
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -0
- package/dest/actions/upload-epoch-proof-failure.js +78 -0
- package/dest/bin/run-failed-epoch.d.ts +2 -0
- package/dest/bin/run-failed-epoch.d.ts.map +1 -0
- package/dest/bin/run-failed-epoch.js +68 -0
- package/dest/checkpoint-store.d.ts +95 -0
- package/dest/checkpoint-store.d.ts.map +1 -0
- package/dest/checkpoint-store.js +178 -0
- package/dest/config.d.ts +31 -0
- package/dest/config.d.ts.map +1 -0
- package/dest/config.js +143 -0
- package/dest/factory.d.ts +36 -0
- package/dest/factory.d.ts.map +1 -0
- package/dest/factory.js +105 -0
- package/dest/index.d.ts +8 -0
- package/dest/index.d.ts.map +1 -0
- package/dest/index.js +7 -0
- package/dest/job/checkpoint-prover.d.ts +169 -0
- package/dest/job/checkpoint-prover.d.ts.map +1 -0
- package/dest/job/checkpoint-prover.js +413 -0
- package/dest/job/epoch-proving-job-data.d.ts +20 -0
- package/dest/job/epoch-proving-job-data.d.ts.map +1 -0
- package/dest/job/epoch-proving-job-data.js +61 -0
- package/dest/job/epoch-session.d.ts +160 -0
- package/dest/job/epoch-session.d.ts.map +1 -0
- package/dest/job/epoch-session.js +744 -0
- package/dest/job/top-tree-job.d.ts +82 -0
- package/dest/job/top-tree-job.d.ts.map +1 -0
- package/dest/job/top-tree-job.js +152 -0
- package/dest/metrics.d.ts +88 -0
- package/dest/metrics.d.ts.map +1 -0
- package/dest/metrics.js +264 -0
- package/dest/monitors/epoch-monitor.d.ts +43 -0
- package/dest/monitors/epoch-monitor.d.ts.map +1 -0
- package/dest/monitors/epoch-monitor.js +111 -0
- package/dest/monitors/index.d.ts +2 -0
- package/dest/monitors/index.d.ts.map +1 -0
- package/dest/monitors/index.js +1 -0
- package/dest/proof-publishing-service.d.ts +161 -0
- package/dest/proof-publishing-service.d.ts.map +1 -0
- package/dest/proof-publishing-service.js +335 -0
- package/dest/prover-node-publisher.d.ts +71 -0
- package/dest/prover-node-publisher.d.ts.map +1 -0
- package/dest/prover-node-publisher.js +346 -0
- package/dest/prover-node.d.ts +185 -0
- package/dest/prover-node.d.ts.map +1 -0
- package/dest/prover-node.js +1056 -0
- package/dest/prover-publisher-factory.d.ts +27 -0
- package/dest/prover-publisher-factory.d.ts.map +1 -0
- package/dest/prover-publisher-factory.js +29 -0
- package/dest/session-manager.d.ts +158 -0
- package/dest/session-manager.d.ts.map +1 -0
- package/dest/session-manager.js +493 -0
- package/dest/test/index.d.ts +13 -0
- package/dest/test/index.d.ts.map +1 -0
- package/dest/test/index.js +3 -0
- package/package.json +102 -0
- package/src/actions/download-epoch-proving-job.ts +43 -0
- package/src/actions/index.ts +2 -0
- package/src/actions/rerun-epoch-proving-job.ts +224 -0
- package/src/actions/upload-epoch-proof-failure.ts +87 -0
- package/src/bin/run-failed-epoch.ts +78 -0
- package/src/checkpoint-store.ts +212 -0
- package/src/config.ts +208 -0
- package/src/factory.ts +191 -0
- package/src/index.ts +7 -0
- package/src/job/checkpoint-prover.ts +551 -0
- package/src/job/epoch-proving-job-data.ts +94 -0
- package/src/job/epoch-session.ts +466 -0
- package/src/job/top-tree-job.ts +227 -0
- package/src/metrics.ts +323 -0
- package/src/monitors/epoch-monitor.ts +110 -0
- package/src/monitors/index.ts +1 -0
- package/src/proof-publishing-service.ts +427 -0
- package/src/prover-node-publisher.ts +473 -0
- package/src/prover-node.ts +830 -0
- package/src/prover-publisher-factory.ts +48 -0
- package/src/session-manager.ts +593 -0
- package/src/test/index.ts +14 -0
|
@@ -0,0 +1,1056 @@
|
|
|
1
|
+
function applyDecs2203RFactory() {
|
|
2
|
+
function createAddInitializerMethod(initializers, decoratorFinishedRef) {
|
|
3
|
+
return function addInitializer(initializer) {
|
|
4
|
+
assertNotFinished(decoratorFinishedRef, "addInitializer");
|
|
5
|
+
assertCallable(initializer, "An initializer");
|
|
6
|
+
initializers.push(initializer);
|
|
7
|
+
};
|
|
8
|
+
}
|
|
9
|
+
function memberDec(dec, name, desc, initializers, kind, isStatic, isPrivate, metadata, value) {
|
|
10
|
+
var kindStr;
|
|
11
|
+
switch(kind){
|
|
12
|
+
case 1:
|
|
13
|
+
kindStr = "accessor";
|
|
14
|
+
break;
|
|
15
|
+
case 2:
|
|
16
|
+
kindStr = "method";
|
|
17
|
+
break;
|
|
18
|
+
case 3:
|
|
19
|
+
kindStr = "getter";
|
|
20
|
+
break;
|
|
21
|
+
case 4:
|
|
22
|
+
kindStr = "setter";
|
|
23
|
+
break;
|
|
24
|
+
default:
|
|
25
|
+
kindStr = "field";
|
|
26
|
+
}
|
|
27
|
+
var ctx = {
|
|
28
|
+
kind: kindStr,
|
|
29
|
+
name: isPrivate ? "#" + name : name,
|
|
30
|
+
static: isStatic,
|
|
31
|
+
private: isPrivate,
|
|
32
|
+
metadata: metadata
|
|
33
|
+
};
|
|
34
|
+
var decoratorFinishedRef = {
|
|
35
|
+
v: false
|
|
36
|
+
};
|
|
37
|
+
ctx.addInitializer = createAddInitializerMethod(initializers, decoratorFinishedRef);
|
|
38
|
+
var get, set;
|
|
39
|
+
if (kind === 0) {
|
|
40
|
+
if (isPrivate) {
|
|
41
|
+
get = desc.get;
|
|
42
|
+
set = desc.set;
|
|
43
|
+
} else {
|
|
44
|
+
get = function() {
|
|
45
|
+
return this[name];
|
|
46
|
+
};
|
|
47
|
+
set = function(v) {
|
|
48
|
+
this[name] = v;
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
} else if (kind === 2) {
|
|
52
|
+
get = function() {
|
|
53
|
+
return desc.value;
|
|
54
|
+
};
|
|
55
|
+
} else {
|
|
56
|
+
if (kind === 1 || kind === 3) {
|
|
57
|
+
get = function() {
|
|
58
|
+
return desc.get.call(this);
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
if (kind === 1 || kind === 4) {
|
|
62
|
+
set = function(v) {
|
|
63
|
+
desc.set.call(this, v);
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
ctx.access = get && set ? {
|
|
68
|
+
get: get,
|
|
69
|
+
set: set
|
|
70
|
+
} : get ? {
|
|
71
|
+
get: get
|
|
72
|
+
} : {
|
|
73
|
+
set: set
|
|
74
|
+
};
|
|
75
|
+
try {
|
|
76
|
+
return dec(value, ctx);
|
|
77
|
+
} finally{
|
|
78
|
+
decoratorFinishedRef.v = true;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
function assertNotFinished(decoratorFinishedRef, fnName) {
|
|
82
|
+
if (decoratorFinishedRef.v) {
|
|
83
|
+
throw new Error("attempted to call " + fnName + " after decoration was finished");
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
function assertCallable(fn, hint) {
|
|
87
|
+
if (typeof fn !== "function") {
|
|
88
|
+
throw new TypeError(hint + " must be a function");
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
function assertValidReturnValue(kind, value) {
|
|
92
|
+
var type = typeof value;
|
|
93
|
+
if (kind === 1) {
|
|
94
|
+
if (type !== "object" || value === null) {
|
|
95
|
+
throw new TypeError("accessor decorators must return an object with get, set, or init properties or void 0");
|
|
96
|
+
}
|
|
97
|
+
if (value.get !== undefined) {
|
|
98
|
+
assertCallable(value.get, "accessor.get");
|
|
99
|
+
}
|
|
100
|
+
if (value.set !== undefined) {
|
|
101
|
+
assertCallable(value.set, "accessor.set");
|
|
102
|
+
}
|
|
103
|
+
if (value.init !== undefined) {
|
|
104
|
+
assertCallable(value.init, "accessor.init");
|
|
105
|
+
}
|
|
106
|
+
} else if (type !== "function") {
|
|
107
|
+
var hint;
|
|
108
|
+
if (kind === 0) {
|
|
109
|
+
hint = "field";
|
|
110
|
+
} else if (kind === 10) {
|
|
111
|
+
hint = "class";
|
|
112
|
+
} else {
|
|
113
|
+
hint = "method";
|
|
114
|
+
}
|
|
115
|
+
throw new TypeError(hint + " decorators must return a function or void 0");
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
function applyMemberDec(ret, base, decInfo, name, kind, isStatic, isPrivate, initializers, metadata) {
|
|
119
|
+
var decs = decInfo[0];
|
|
120
|
+
var desc, init, value;
|
|
121
|
+
if (isPrivate) {
|
|
122
|
+
if (kind === 0 || kind === 1) {
|
|
123
|
+
desc = {
|
|
124
|
+
get: decInfo[3],
|
|
125
|
+
set: decInfo[4]
|
|
126
|
+
};
|
|
127
|
+
} else if (kind === 3) {
|
|
128
|
+
desc = {
|
|
129
|
+
get: decInfo[3]
|
|
130
|
+
};
|
|
131
|
+
} else if (kind === 4) {
|
|
132
|
+
desc = {
|
|
133
|
+
set: decInfo[3]
|
|
134
|
+
};
|
|
135
|
+
} else {
|
|
136
|
+
desc = {
|
|
137
|
+
value: decInfo[3]
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
} else if (kind !== 0) {
|
|
141
|
+
desc = Object.getOwnPropertyDescriptor(base, name);
|
|
142
|
+
}
|
|
143
|
+
if (kind === 1) {
|
|
144
|
+
value = {
|
|
145
|
+
get: desc.get,
|
|
146
|
+
set: desc.set
|
|
147
|
+
};
|
|
148
|
+
} else if (kind === 2) {
|
|
149
|
+
value = desc.value;
|
|
150
|
+
} else if (kind === 3) {
|
|
151
|
+
value = desc.get;
|
|
152
|
+
} else if (kind === 4) {
|
|
153
|
+
value = desc.set;
|
|
154
|
+
}
|
|
155
|
+
var newValue, get, set;
|
|
156
|
+
if (typeof decs === "function") {
|
|
157
|
+
newValue = memberDec(decs, name, desc, initializers, kind, isStatic, isPrivate, metadata, value);
|
|
158
|
+
if (newValue !== void 0) {
|
|
159
|
+
assertValidReturnValue(kind, newValue);
|
|
160
|
+
if (kind === 0) {
|
|
161
|
+
init = newValue;
|
|
162
|
+
} else if (kind === 1) {
|
|
163
|
+
init = newValue.init;
|
|
164
|
+
get = newValue.get || value.get;
|
|
165
|
+
set = newValue.set || value.set;
|
|
166
|
+
value = {
|
|
167
|
+
get: get,
|
|
168
|
+
set: set
|
|
169
|
+
};
|
|
170
|
+
} else {
|
|
171
|
+
value = newValue;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
} else {
|
|
175
|
+
for(var i = decs.length - 1; i >= 0; i--){
|
|
176
|
+
var dec = decs[i];
|
|
177
|
+
newValue = memberDec(dec, name, desc, initializers, kind, isStatic, isPrivate, metadata, value);
|
|
178
|
+
if (newValue !== void 0) {
|
|
179
|
+
assertValidReturnValue(kind, newValue);
|
|
180
|
+
var newInit;
|
|
181
|
+
if (kind === 0) {
|
|
182
|
+
newInit = newValue;
|
|
183
|
+
} else if (kind === 1) {
|
|
184
|
+
newInit = newValue.init;
|
|
185
|
+
get = newValue.get || value.get;
|
|
186
|
+
set = newValue.set || value.set;
|
|
187
|
+
value = {
|
|
188
|
+
get: get,
|
|
189
|
+
set: set
|
|
190
|
+
};
|
|
191
|
+
} else {
|
|
192
|
+
value = newValue;
|
|
193
|
+
}
|
|
194
|
+
if (newInit !== void 0) {
|
|
195
|
+
if (init === void 0) {
|
|
196
|
+
init = newInit;
|
|
197
|
+
} else if (typeof init === "function") {
|
|
198
|
+
init = [
|
|
199
|
+
init,
|
|
200
|
+
newInit
|
|
201
|
+
];
|
|
202
|
+
} else {
|
|
203
|
+
init.push(newInit);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
if (kind === 0 || kind === 1) {
|
|
210
|
+
if (init === void 0) {
|
|
211
|
+
init = function(instance, init) {
|
|
212
|
+
return init;
|
|
213
|
+
};
|
|
214
|
+
} else if (typeof init !== "function") {
|
|
215
|
+
var ownInitializers = init;
|
|
216
|
+
init = function(instance, init) {
|
|
217
|
+
var value = init;
|
|
218
|
+
for(var i = 0; i < ownInitializers.length; i++){
|
|
219
|
+
value = ownInitializers[i].call(instance, value);
|
|
220
|
+
}
|
|
221
|
+
return value;
|
|
222
|
+
};
|
|
223
|
+
} else {
|
|
224
|
+
var originalInitializer = init;
|
|
225
|
+
init = function(instance, init) {
|
|
226
|
+
return originalInitializer.call(instance, init);
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
ret.push(init);
|
|
230
|
+
}
|
|
231
|
+
if (kind !== 0) {
|
|
232
|
+
if (kind === 1) {
|
|
233
|
+
desc.get = value.get;
|
|
234
|
+
desc.set = value.set;
|
|
235
|
+
} else if (kind === 2) {
|
|
236
|
+
desc.value = value;
|
|
237
|
+
} else if (kind === 3) {
|
|
238
|
+
desc.get = value;
|
|
239
|
+
} else if (kind === 4) {
|
|
240
|
+
desc.set = value;
|
|
241
|
+
}
|
|
242
|
+
if (isPrivate) {
|
|
243
|
+
if (kind === 1) {
|
|
244
|
+
ret.push(function(instance, args) {
|
|
245
|
+
return value.get.call(instance, args);
|
|
246
|
+
});
|
|
247
|
+
ret.push(function(instance, args) {
|
|
248
|
+
return value.set.call(instance, args);
|
|
249
|
+
});
|
|
250
|
+
} else if (kind === 2) {
|
|
251
|
+
ret.push(value);
|
|
252
|
+
} else {
|
|
253
|
+
ret.push(function(instance, args) {
|
|
254
|
+
return value.call(instance, args);
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
} else {
|
|
258
|
+
Object.defineProperty(base, name, desc);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
function applyMemberDecs(Class, decInfos, metadata) {
|
|
263
|
+
var ret = [];
|
|
264
|
+
var protoInitializers;
|
|
265
|
+
var staticInitializers;
|
|
266
|
+
var existingProtoNonFields = new Map();
|
|
267
|
+
var existingStaticNonFields = new Map();
|
|
268
|
+
for(var i = 0; i < decInfos.length; i++){
|
|
269
|
+
var decInfo = decInfos[i];
|
|
270
|
+
if (!Array.isArray(decInfo)) continue;
|
|
271
|
+
var kind = decInfo[1];
|
|
272
|
+
var name = decInfo[2];
|
|
273
|
+
var isPrivate = decInfo.length > 3;
|
|
274
|
+
var isStatic = kind >= 5;
|
|
275
|
+
var base;
|
|
276
|
+
var initializers;
|
|
277
|
+
if (isStatic) {
|
|
278
|
+
base = Class;
|
|
279
|
+
kind = kind - 5;
|
|
280
|
+
staticInitializers = staticInitializers || [];
|
|
281
|
+
initializers = staticInitializers;
|
|
282
|
+
} else {
|
|
283
|
+
base = Class.prototype;
|
|
284
|
+
protoInitializers = protoInitializers || [];
|
|
285
|
+
initializers = protoInitializers;
|
|
286
|
+
}
|
|
287
|
+
if (kind !== 0 && !isPrivate) {
|
|
288
|
+
var existingNonFields = isStatic ? existingStaticNonFields : existingProtoNonFields;
|
|
289
|
+
var existingKind = existingNonFields.get(name) || 0;
|
|
290
|
+
if (existingKind === true || existingKind === 3 && kind !== 4 || existingKind === 4 && kind !== 3) {
|
|
291
|
+
throw new Error("Attempted to decorate a public method/accessor that has the same name as a previously decorated public method/accessor. This is not currently supported by the decorators plugin. Property name was: " + name);
|
|
292
|
+
} else if (!existingKind && kind > 2) {
|
|
293
|
+
existingNonFields.set(name, kind);
|
|
294
|
+
} else {
|
|
295
|
+
existingNonFields.set(name, true);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
applyMemberDec(ret, base, decInfo, name, kind, isStatic, isPrivate, initializers, metadata);
|
|
299
|
+
}
|
|
300
|
+
pushInitializers(ret, protoInitializers);
|
|
301
|
+
pushInitializers(ret, staticInitializers);
|
|
302
|
+
return ret;
|
|
303
|
+
}
|
|
304
|
+
function pushInitializers(ret, initializers) {
|
|
305
|
+
if (initializers) {
|
|
306
|
+
ret.push(function(instance) {
|
|
307
|
+
for(var i = 0; i < initializers.length; i++){
|
|
308
|
+
initializers[i].call(instance);
|
|
309
|
+
}
|
|
310
|
+
return instance;
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
function applyClassDecs(targetClass, classDecs, metadata) {
|
|
315
|
+
if (classDecs.length > 0) {
|
|
316
|
+
var initializers = [];
|
|
317
|
+
var newClass = targetClass;
|
|
318
|
+
var name = targetClass.name;
|
|
319
|
+
for(var i = classDecs.length - 1; i >= 0; i--){
|
|
320
|
+
var decoratorFinishedRef = {
|
|
321
|
+
v: false
|
|
322
|
+
};
|
|
323
|
+
try {
|
|
324
|
+
var nextNewClass = classDecs[i](newClass, {
|
|
325
|
+
kind: "class",
|
|
326
|
+
name: name,
|
|
327
|
+
addInitializer: createAddInitializerMethod(initializers, decoratorFinishedRef),
|
|
328
|
+
metadata
|
|
329
|
+
});
|
|
330
|
+
} finally{
|
|
331
|
+
decoratorFinishedRef.v = true;
|
|
332
|
+
}
|
|
333
|
+
if (nextNewClass !== undefined) {
|
|
334
|
+
assertValidReturnValue(10, nextNewClass);
|
|
335
|
+
newClass = nextNewClass;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
return [
|
|
339
|
+
defineMetadata(newClass, metadata),
|
|
340
|
+
function() {
|
|
341
|
+
for(var i = 0; i < initializers.length; i++){
|
|
342
|
+
initializers[i].call(newClass);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
];
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
function defineMetadata(Class, metadata) {
|
|
349
|
+
return Object.defineProperty(Class, Symbol.metadata || Symbol.for("Symbol.metadata"), {
|
|
350
|
+
configurable: true,
|
|
351
|
+
enumerable: true,
|
|
352
|
+
value: metadata
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
return function applyDecs2203R(targetClass, memberDecs, classDecs, parentClass) {
|
|
356
|
+
if (parentClass !== void 0) {
|
|
357
|
+
var parentMetadata = parentClass[Symbol.metadata || Symbol.for("Symbol.metadata")];
|
|
358
|
+
}
|
|
359
|
+
var metadata = Object.create(parentMetadata === void 0 ? null : parentMetadata);
|
|
360
|
+
var e = applyMemberDecs(targetClass, memberDecs, metadata);
|
|
361
|
+
if (!classDecs.length) defineMetadata(targetClass, metadata);
|
|
362
|
+
return {
|
|
363
|
+
e: e,
|
|
364
|
+
get c () {
|
|
365
|
+
return applyClassDecs(targetClass, classDecs, metadata);
|
|
366
|
+
}
|
|
367
|
+
};
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
|
|
371
|
+
return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
|
|
372
|
+
}
|
|
373
|
+
var _initProto;
|
|
374
|
+
import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec-labs/foundation/branded-types';
|
|
375
|
+
import { assertRequired, compact, pick } from '@aztec-labs/foundation/collection';
|
|
376
|
+
import { Fr } from '@aztec-labs/foundation/curves/bn254';
|
|
377
|
+
import { memoize } from '@aztec-labs/foundation/decorators';
|
|
378
|
+
import { createLogger } from '@aztec-labs/foundation/log';
|
|
379
|
+
import { RunningPromise } from '@aztec-labs/foundation/running-promise';
|
|
380
|
+
import { DateProvider, executeTimeout } from '@aztec-labs/foundation/timer';
|
|
381
|
+
import { getLastSiblingPath } from '@aztec-labs/prover-client/helpers';
|
|
382
|
+
import { ChonkCache } from '@aztec-labs/prover-client/orchestrator';
|
|
383
|
+
import { PublicProcessorFactory } from '@aztec-labs/simulator/server';
|
|
384
|
+
import { EventDrivenL2BlockStream, L2TipsMemoryStore } from '@aztec-labs/stdlib/block';
|
|
385
|
+
import { getEpochAtSlot, getProofSubmissionDeadlineEpoch } from '@aztec-labs/stdlib/epoch-helpers';
|
|
386
|
+
import { EpochProvingJobTerminalState, tryStop } from '@aztec-labs/stdlib/interfaces/server';
|
|
387
|
+
import { MerkleTreeId } from '@aztec-labs/stdlib/trees';
|
|
388
|
+
import { getTelemetryClient } from '@aztec-labs/telemetry-client';
|
|
389
|
+
import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
|
|
390
|
+
import { CheckpointStore } from './checkpoint-store.js';
|
|
391
|
+
import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
|
|
392
|
+
import { ProofPublishingService } from './proof-publishing-service.js';
|
|
393
|
+
import { SessionManager } from './session-manager.js';
|
|
394
|
+
/**
|
|
395
|
+
* Grace period for the proof-publishing service to settle during shutdown. The service waits for
|
|
396
|
+
* any in-flight L1 proof-submission tx to finish; that tx can take a long time to mine, so we cap
|
|
397
|
+
* the wait rather than letting `stop()` hang indefinitely.
|
|
398
|
+
*/ const PUBLISHING_SERVICE_STOP_TIMEOUT_MS = 30_000;
|
|
399
|
+
/**
|
|
400
|
+
* An Aztec Prover Node is a standalone process that monitors the chain for new checkpoints,
|
|
401
|
+
* starts proving them optimistically as they arrive, and submits epoch proofs to L1 once
|
|
402
|
+
* complete.
|
|
403
|
+
*
|
|
404
|
+
* The class is intentionally thin: it owns the long-lived collections (`CheckpointStore`,
|
|
405
|
+
* `ChonkCache`, `SessionManager`), the L2BlockStream, and a periodic ticker that nudges the
|
|
406
|
+
* manager to pick up newly-complete epochs. Every session lifecycle decision is delegated to
|
|
407
|
+
* the `SessionManager`. Each chain event is translated here into a single method call on it.
|
|
408
|
+
*/ export class ProverNode {
|
|
409
|
+
prover;
|
|
410
|
+
publisherFactory;
|
|
411
|
+
l2BlockSource;
|
|
412
|
+
l1ToL2MessageSource;
|
|
413
|
+
contractDataSource;
|
|
414
|
+
worldState;
|
|
415
|
+
p2pClient;
|
|
416
|
+
rollupContract;
|
|
417
|
+
l1Metrics;
|
|
418
|
+
avmSimulator;
|
|
419
|
+
telemetryClient;
|
|
420
|
+
delayer;
|
|
421
|
+
dateProvider;
|
|
422
|
+
static{
|
|
423
|
+
({ e: [_initProto] } = _apply_decs_2203_r(this, [
|
|
424
|
+
[
|
|
425
|
+
memoize,
|
|
426
|
+
2,
|
|
427
|
+
"getL1Constants"
|
|
428
|
+
]
|
|
429
|
+
], []));
|
|
430
|
+
}
|
|
431
|
+
log;
|
|
432
|
+
checkpointStore;
|
|
433
|
+
chonkCache;
|
|
434
|
+
sessionManager;
|
|
435
|
+
config;
|
|
436
|
+
jobMetrics;
|
|
437
|
+
rewardsMetrics;
|
|
438
|
+
/** In-memory store for the L2BlockStream's local data provider. */ tipsStore;
|
|
439
|
+
/** Block stream for checkpoint and reorg detection. */ blockStream;
|
|
440
|
+
/**
|
|
441
|
+
* Highest epoch whose proof-submission window has passed. Monotonic high-water mark.
|
|
442
|
+
* Seeded from the last fully-proven epoch at start(); advanced on every block-stream
|
|
443
|
+
* event by comparing the archiver's latest synced L2 slot against each epoch's
|
|
444
|
+
* submission deadline. Protected so tests can verify the start() seeding.
|
|
445
|
+
*/ lastExpiredEpoch;
|
|
446
|
+
/**
|
|
447
|
+
* Highest checkpoint number whose proving-side handling has completed (or that was legitimately skipped).
|
|
448
|
+
* The catch-up loop walks from here to each `chain-checkpointed` tip event. Seeded at start() from the last
|
|
449
|
+
* checkpoint of the last fully-proven epoch (or 0), so a restart reprocesses the partially-proven epoch rather
|
|
450
|
+
* than trusting a checkpointed tip that may sit ahead of unproven checkpoints. Clamped down on a prune.
|
|
451
|
+
*/ lastProcessedCheckpoint;
|
|
452
|
+
/** Periodic tick that runs the epoch-expiry sweep during idle periods when no block-stream events arrive. */ expiryTicker;
|
|
453
|
+
tracer;
|
|
454
|
+
publishingService;
|
|
455
|
+
constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, rollupContract, l1Metrics, avmSimulator, config = {}, telemetryClient = getTelemetryClient(), delayer, dateProvider = new DateProvider()){
|
|
456
|
+
this.prover = prover;
|
|
457
|
+
this.publisherFactory = publisherFactory;
|
|
458
|
+
this.l2BlockSource = l2BlockSource;
|
|
459
|
+
this.l1ToL2MessageSource = l1ToL2MessageSource;
|
|
460
|
+
this.contractDataSource = contractDataSource;
|
|
461
|
+
this.worldState = worldState;
|
|
462
|
+
this.p2pClient = p2pClient;
|
|
463
|
+
this.rollupContract = rollupContract;
|
|
464
|
+
this.l1Metrics = l1Metrics;
|
|
465
|
+
this.avmSimulator = avmSimulator;
|
|
466
|
+
this.telemetryClient = telemetryClient;
|
|
467
|
+
this.delayer = delayer;
|
|
468
|
+
this.dateProvider = dateProvider;
|
|
469
|
+
this.log = (_initProto(this), createLogger('prover-node'));
|
|
470
|
+
this.lastProcessedCheckpoint = CheckpointNumber.ZERO;
|
|
471
|
+
this.config = {
|
|
472
|
+
proverNodePollingIntervalMs: 1_000,
|
|
473
|
+
proverNodeMaxPendingJobs: 100,
|
|
474
|
+
proverNodeMaxParallelBlocksPerEpoch: 0,
|
|
475
|
+
txGatheringIntervalMs: 1_000,
|
|
476
|
+
txGatheringBatchSize: 10,
|
|
477
|
+
txGatheringMaxParallelRequestsPerNode: 100,
|
|
478
|
+
txGatheringTimeoutMs: 120_000,
|
|
479
|
+
proverNodeFailedEpochStore: undefined,
|
|
480
|
+
proverNodeEpochProvingDelayMs: undefined,
|
|
481
|
+
...compact(config)
|
|
482
|
+
};
|
|
483
|
+
this.validateConfig();
|
|
484
|
+
const meter = telemetryClient.getMeter('ProverNode');
|
|
485
|
+
this.tracer = telemetryClient.getTracer('ProverNode');
|
|
486
|
+
this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
|
|
487
|
+
this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
|
|
488
|
+
this.tipsStore = new L2TipsMemoryStore(this.l2BlockSource.getGenesisBlockHash());
|
|
489
|
+
this.chonkCache = new ChonkCache(this.log.getBindings());
|
|
490
|
+
this.checkpointStore = new CheckpointStore(this.l2BlockSource, {
|
|
491
|
+
proverFactory: this.prover,
|
|
492
|
+
chonkCache: this.chonkCache,
|
|
493
|
+
publicProcessorFactory: new PublicProcessorFactory(this.contractDataSource, this.avmSimulator, this.dateProvider, this.telemetryClient, this.log.getBindings()),
|
|
494
|
+
dbProvider: this.worldState,
|
|
495
|
+
txProvider: this.p2pClient.getTxProvider(),
|
|
496
|
+
dateProvider: this.dateProvider,
|
|
497
|
+
proverId: this.prover.getProverId(),
|
|
498
|
+
metrics: this.jobMetrics,
|
|
499
|
+
txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
|
|
500
|
+
deadline: undefined,
|
|
501
|
+
// A checkpoint prover that fails (a sub-tree fault or a prune-induced fork fault) uploads a
|
|
502
|
+
// post-mortem for its own checkpoint, independently of any session. Fire-and-forget.
|
|
503
|
+
onFailed: (prover)=>void this.tryUploadCheckpointFailure(prover)
|
|
504
|
+
}, this.log.getBindings());
|
|
505
|
+
}
|
|
506
|
+
getProverId() {
|
|
507
|
+
return this.prover.getProverId();
|
|
508
|
+
}
|
|
509
|
+
getP2P() {
|
|
510
|
+
return this.p2pClient;
|
|
511
|
+
}
|
|
512
|
+
/** Test-only: the shared L1 tx delayer, if enabled. */ getDelayer() {
|
|
513
|
+
return this.delayer;
|
|
514
|
+
}
|
|
515
|
+
/** Observability summary for the ProverNodeApi. */ getJobs() {
|
|
516
|
+
return Promise.resolve(this.sessionManager?.getJobs() ?? []);
|
|
517
|
+
}
|
|
518
|
+
/** Tests inspect this when validating reconcile behaviour. */ getCheckpointStore() {
|
|
519
|
+
return this.checkpointStore;
|
|
520
|
+
}
|
|
521
|
+
/** Tests inspect this to verify chonk-cache release semantics. */ getChonkCache() {
|
|
522
|
+
return this.chonkCache;
|
|
523
|
+
}
|
|
524
|
+
/** Tests inspect this when looking up live sessions. */ getSessionManager() {
|
|
525
|
+
if (!this.sessionManager) {
|
|
526
|
+
throw new Error('SessionManager not yet constructed — start() must be called first.');
|
|
527
|
+
}
|
|
528
|
+
return this.sessionManager;
|
|
529
|
+
}
|
|
530
|
+
/** Returns the underlying prover instance. */ getProver() {
|
|
531
|
+
return this.prover;
|
|
532
|
+
}
|
|
533
|
+
// ---------------- L2BlockStream handler ----------------
|
|
534
|
+
async handleBlockStreamEvent(event) {
|
|
535
|
+
switch(event.type){
|
|
536
|
+
case 'chain-checkpointed':
|
|
537
|
+
await this.processCheckpointJump(event.checkpoint.number);
|
|
538
|
+
break;
|
|
539
|
+
case 'chain-pruned':
|
|
540
|
+
await this.handlePruneEvent(event.block);
|
|
541
|
+
break;
|
|
542
|
+
case 'chain-proven':
|
|
543
|
+
this.publishingService?.onChainProven(BlockNumber(event.block.number));
|
|
544
|
+
break;
|
|
545
|
+
// The proposed tip drives only the tips store's walk-back history (recorded below); the prover-node
|
|
546
|
+
// tracks checkpoints, not proposed blocks. `blocks-added` is never emitted in tips-only mode, and
|
|
547
|
+
// `chain-finalized` carries nothing the prover-node acts on.
|
|
548
|
+
case 'chain-proposed':
|
|
549
|
+
case 'chain-finalized':
|
|
550
|
+
case 'blocks-added':
|
|
551
|
+
break;
|
|
552
|
+
default:
|
|
553
|
+
{
|
|
554
|
+
const _ = event;
|
|
555
|
+
break;
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
// Advance the local tips store only after the proving-side handling (registration / prune) has
|
|
559
|
+
// succeeded. Any failure above propagates to the L2BlockStream (which logs and stops this poll
|
|
560
|
+
// pass) and skips this update, so the event is re-emitted on the next poll rather than skipped
|
|
561
|
+
await this.tipsStore.handleBlockStreamEvent(event);
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* Walks every checkpoint between the local cursor and the newly-reported checkpointed tip, registering
|
|
565
|
+
* each one that belongs to an epoch that can still be proven. The block stream now delivers a single thin
|
|
566
|
+
* `chain-checkpointed` tip event per pass rather than one fat event per checkpoint, so this drives the
|
|
567
|
+
* catch-up itself: light metadata first (`getCheckpointsData`) to decide relevance per epoch, then a heavy
|
|
568
|
+
* `getCheckpoints` fetch only for checkpoints in provable epochs.
|
|
569
|
+
*
|
|
570
|
+
* The cursor advances one checkpoint at a time and only after that checkpoint's proving-side handling has
|
|
571
|
+
* fully succeeded, preserving the A-1041 at-least-once semantics: a mid-jump failure leaves the cursor
|
|
572
|
+
* behind so the next pass retries from the first checkpoint that did not complete.
|
|
573
|
+
*/ async processCheckpointJump(targetCheckpoint) {
|
|
574
|
+
if (targetCheckpoint <= this.lastProcessedCheckpoint) {
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
const l1Constants = await this.getL1Constants();
|
|
578
|
+
// Cap the catch-up at the `(proofSubmissionEpochs + 1) * epochDuration` most recent checkpoints.
|
|
579
|
+
// When the cursor is much further behind (e.g. resyncing after a long time offline), fetching the whole gap could
|
|
580
|
+
// load thousands of checkpoints we cannot act on: anything older than the last two epochs is already past
|
|
581
|
+
// its proof-submission window, so we skip it and jump the cursor forward to the start of the capped range.
|
|
582
|
+
const maxCheckpoints = (l1Constants.proofSubmissionEpochs + 1) * l1Constants.epochDuration;
|
|
583
|
+
let from = CheckpointNumber(this.lastProcessedCheckpoint + 1);
|
|
584
|
+
if (Number(targetCheckpoint - from) + 1 > maxCheckpoints) {
|
|
585
|
+
const cappedFrom = CheckpointNumber(targetCheckpoint - maxCheckpoints + 1);
|
|
586
|
+
this.log.warn(`Skipping unprovable checkpoints during catch-up; the prover node is far behind`, {
|
|
587
|
+
from,
|
|
588
|
+
cappedFrom,
|
|
589
|
+
targetCheckpoint,
|
|
590
|
+
maxCheckpoints
|
|
591
|
+
});
|
|
592
|
+
// Advance the cursor past the skipped checkpoints so they are never retried.
|
|
593
|
+
this.lastProcessedCheckpoint = CheckpointNumber(cappedFrom - 1);
|
|
594
|
+
from = cappedFrom;
|
|
595
|
+
}
|
|
596
|
+
const limit = Number(targetCheckpoint - from) + 1;
|
|
597
|
+
const metadatas = await this.l2BlockSource.getCheckpointsData({
|
|
598
|
+
from,
|
|
599
|
+
limit
|
|
600
|
+
});
|
|
601
|
+
// Per-epoch relevance is cached so a multi-checkpoint epoch resolves it once. Skipping is whole-epoch
|
|
602
|
+
// only: the SessionManager requires an epoch's checkpoints fully covered before it opens a session, so we
|
|
603
|
+
// never drop an individual checkpoint inside an epoch we will prove.
|
|
604
|
+
const epochSkippable = new Map();
|
|
605
|
+
for (const metadata of metadatas){
|
|
606
|
+
const epochNumber = getEpochAtSlot(metadata.header.slotNumber, l1Constants);
|
|
607
|
+
let skippable = epochSkippable.get(epochNumber);
|
|
608
|
+
if (skippable === undefined) {
|
|
609
|
+
skippable = await this.isEpochFullyProven(epochNumber, l1Constants) || await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants);
|
|
610
|
+
epochSkippable.set(epochNumber, skippable);
|
|
611
|
+
}
|
|
612
|
+
if (skippable) {
|
|
613
|
+
this.log.debug(`Skipping checkpoint ${metadata.checkpointNumber} for unprovable epoch ${epochNumber}`);
|
|
614
|
+
} else {
|
|
615
|
+
await this.registerCheckpoint(metadata.checkpointNumber, epochNumber);
|
|
616
|
+
}
|
|
617
|
+
// Advance only after the checkpoint's handling succeeded (or it was legitimately skipped). registerCheckpoint
|
|
618
|
+
// throws on failure, which leaves the cursor here for the next pass to retry (A-1041).
|
|
619
|
+
this.lastProcessedCheckpoint = metadata.checkpointNumber;
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
/** Heavy-fetch a single checkpoint, register it with the store, and notify the session manager. */ async registerCheckpoint(checkpointNumber, epochNumber) {
|
|
623
|
+
const published = await this.l2BlockSource.getCheckpoint({
|
|
624
|
+
number: checkpointNumber
|
|
625
|
+
});
|
|
626
|
+
if (!published) {
|
|
627
|
+
throw new Error(`Checkpoint ${checkpointNumber} not found in block source during catch-up`);
|
|
628
|
+
}
|
|
629
|
+
const checkpoint = published.checkpoint;
|
|
630
|
+
this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
|
|
631
|
+
checkpointNumber: checkpoint.number,
|
|
632
|
+
epochNumber,
|
|
633
|
+
slotNumber: checkpoint.header.slotNumber
|
|
634
|
+
});
|
|
635
|
+
const registerData = await this.collectRegisterData(checkpoint, published.attestations);
|
|
636
|
+
await this.checkpointStore.addOrUpdate(checkpoint, registerData);
|
|
637
|
+
await this.sessionManager?.onCheckpointAdded(epochNumber);
|
|
638
|
+
// Tips-only mode delivers no blocks, so record one witness per checkpointed block: a reorg into the checkpoint's
|
|
639
|
+
// range then prunes at the true divergence instead of the nearest sparse tip anchor.
|
|
640
|
+
await this.tipsStore.recordBlockHashes(await Promise.all(checkpoint.blocks.map(async (block)=>({
|
|
641
|
+
number: block.number,
|
|
642
|
+
hash: (await block.header.hash()).toString()
|
|
643
|
+
}))));
|
|
644
|
+
}
|
|
645
|
+
/**
|
|
646
|
+
* Gathers register-time data for a checkpoint: previous block header, L1-to-L2 messages,
|
|
647
|
+
* and the archive sibling path.
|
|
648
|
+
*/ async collectRegisterData(checkpoint, attestations) {
|
|
649
|
+
const previousBlockNumber = BlockNumber(checkpoint.blocks[0].number - 1);
|
|
650
|
+
const previousBlockHeader = await this.gatherPreviousBlockHeader(previousBlockNumber);
|
|
651
|
+
const lastBlock = checkpoint.blocks.at(-1);
|
|
652
|
+
// Streaming Inbox: the checkpoint's consumed messages are those between the parent
|
|
653
|
+
// checkpoint's consumed position and this checkpoint's last block, as a compact leaf-count range. The prover
|
|
654
|
+
// slices them per block.
|
|
655
|
+
const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts(BigInt(previousBlockHeader.state.l1ToL2MessageTree.nextAvailableLeafIndex), BigInt(lastBlock.header.state.l1ToL2MessageTree.nextAvailableLeafIndex));
|
|
656
|
+
const previousInboxRollingHash = await this.gatherPreviousInboxRollingHash(checkpoint.number);
|
|
657
|
+
const lastBlockHash = await lastBlock.header.hash();
|
|
658
|
+
await this.worldState.syncImmediate(lastBlock.number, lastBlockHash);
|
|
659
|
+
const previousArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, this.worldState.getSnapshot(previousBlockNumber));
|
|
660
|
+
return {
|
|
661
|
+
attestations,
|
|
662
|
+
previousBlockHeader,
|
|
663
|
+
l1ToL2Messages,
|
|
664
|
+
previousInboxRollingHash,
|
|
665
|
+
previousArchiveSiblingPath
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* Sources the inbox rolling hash chain-start for a checkpoint: the previous checkpoint's `inboxRollingHash`, or zero
|
|
670
|
+
* for the genesis checkpoint. The prover threads this into the base parity circuits so the rebuilt checkpoint header
|
|
671
|
+
* matches the proposer's.
|
|
672
|
+
*/ async gatherPreviousInboxRollingHash(checkpointNumber) {
|
|
673
|
+
if (checkpointNumber <= 1) {
|
|
674
|
+
return Fr.ZERO;
|
|
675
|
+
}
|
|
676
|
+
const previous = await this.l2BlockSource.getCheckpoint({
|
|
677
|
+
number: CheckpointNumber(checkpointNumber - 1)
|
|
678
|
+
});
|
|
679
|
+
if (!previous) {
|
|
680
|
+
throw new Error(`Previous checkpoint ${checkpointNumber - 1} not found when sourcing inbox rolling hash`);
|
|
681
|
+
}
|
|
682
|
+
return previous.checkpoint.header.inboxRollingHash;
|
|
683
|
+
}
|
|
684
|
+
/**
|
|
685
|
+
* Marks every prover orphaned by the prune as pruned, clamps the catch-up cursor below the prune target's
|
|
686
|
+
* checkpoint, and notifies the session manager. Keyed off the prune target block (the highest surviving block)
|
|
687
|
+
* rather than the source's checkpointed tip, which can sit above the target after a re-checkpoint and would leave
|
|
688
|
+
* orphaned provers canonical. Throws (rather than warning) if the cursor floor cannot be resolved, so the pass
|
|
689
|
+
* fails and the prune is retried next iteration.
|
|
690
|
+
*/ async handlePruneEvent(prunedToBlock) {
|
|
691
|
+
this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, {
|
|
692
|
+
prunedToBlock
|
|
693
|
+
});
|
|
694
|
+
// Resolve the cursor floor BEFORE removing provers: cancelAndRemoveAboveBlock returns only the provers it removed,
|
|
695
|
+
// so a throw after removing would leave a retry pass with nothing to act on. Resolving first means a throw leaves
|
|
696
|
+
// everything untouched and the next pass retries the whole handler (the tips cursor only advances on success).
|
|
697
|
+
let cursorFloor;
|
|
698
|
+
if (prunedToBlock.number === 0) {
|
|
699
|
+
cursorFloor = CheckpointNumber.ZERO;
|
|
700
|
+
} else {
|
|
701
|
+
const targetData = await this.l2BlockSource.getBlockData({
|
|
702
|
+
number: prunedToBlock.number
|
|
703
|
+
});
|
|
704
|
+
if (targetData === undefined) {
|
|
705
|
+
throw new Error(`No block data found for prune target block ${prunedToBlock.number}; cannot clamp checkpoint cursor`);
|
|
706
|
+
}
|
|
707
|
+
// Clamp to `cpAtTarget - 1`: a mid-checkpoint target leaves that checkpoint partially orphaned and it must be
|
|
708
|
+
// reprocessed. Over-clamping merely re-registers a checkpoint (at-least-once by design — A-1041); under-clamping
|
|
709
|
+
// would permanently skip a rebuilt same-number checkpoint.
|
|
710
|
+
cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1));
|
|
711
|
+
}
|
|
712
|
+
const affected = this.checkpointStore.cancelAndRemoveAboveBlock(prunedToBlock.number);
|
|
713
|
+
if (this.lastProcessedCheckpoint > cursorFloor) {
|
|
714
|
+
this.lastProcessedCheckpoint = cursorFloor;
|
|
715
|
+
}
|
|
716
|
+
if (affected.length === 0) {
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
const l1Constants = await this.getL1Constants();
|
|
720
|
+
const affectedEpochs = Array.from(new Set(affected.map((p)=>Number(getEpochAtSlot(p.slotNumber, l1Constants))))).map((n)=>EpochNumber(n));
|
|
721
|
+
// The session manager cancels every affected session, which in turn calls
|
|
722
|
+
// publishingService.withdraw(uuid) for each candidate; no separate notification to the
|
|
723
|
+
// publishing service is needed.
|
|
724
|
+
await this.sessionManager?.onPrune(affectedEpochs);
|
|
725
|
+
}
|
|
726
|
+
/**
|
|
727
|
+
* Returns true once the chain has advanced past the given epoch's proof-submission window.
|
|
728
|
+
* Used to ignore checkpoints whose epoch can no longer be proven in time — chiefly while the
|
|
729
|
+
* archiver replays old blocks after a restart. Compares the archiver's latest synced L2 slot
|
|
730
|
+
* against the epoch's submission-deadline epoch; conservatively returns false if the slot can't
|
|
731
|
+
* be read yet.
|
|
732
|
+
*/ async isEpochPastProofSubmissionWindow(epochNumber, l1Constants) {
|
|
733
|
+
const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
734
|
+
if (latestSlot === undefined) {
|
|
735
|
+
return false;
|
|
736
|
+
}
|
|
737
|
+
const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
|
|
738
|
+
return latestEpoch >= getProofSubmissionDeadlineEpoch(epochNumber, l1Constants);
|
|
739
|
+
}
|
|
740
|
+
/**
|
|
741
|
+
* Compares the archiver's latest synced L2 slot against `lastExpiredEpoch` and, for each
|
|
742
|
+
* newly-expired epoch, releases the chonk-cache entries for its blocks and reaps any
|
|
743
|
+
* CheckpointProvers in the store. An epoch E is expired once the chain reaches the start
|
|
744
|
+
* of epoch `E + proofSubmissionEpochs + 1`. Silently no-ops if nothing has expired since
|
|
745
|
+
* the last check or the archiver's slot can't be read.
|
|
746
|
+
*/ async checkEpochExpiry() {
|
|
747
|
+
const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
748
|
+
if (latestSlot === undefined) {
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
const l1Constants = await this.getL1Constants();
|
|
752
|
+
const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
|
|
753
|
+
const offset = l1Constants.proofSubmissionEpochs + 1;
|
|
754
|
+
if (latestEpoch < offset) {
|
|
755
|
+
return;
|
|
756
|
+
}
|
|
757
|
+
const newlyExpiredUpTo = EpochNumber(latestEpoch - offset);
|
|
758
|
+
const from = this.lastExpiredEpoch === undefined ? EpochNumber(0) : EpochNumber(this.lastExpiredEpoch + 1);
|
|
759
|
+
if (newlyExpiredUpTo < from) {
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
for(let e = from; e <= newlyExpiredUpTo; e = EpochNumber(e + 1)){
|
|
763
|
+
await this.expireEpoch(e);
|
|
764
|
+
}
|
|
765
|
+
this.lastExpiredEpoch = newlyExpiredUpTo;
|
|
766
|
+
}
|
|
767
|
+
/**
|
|
768
|
+
* Releases chonk-cache entries for every block in the supplied epoch (best-effort) and reaps every
|
|
769
|
+
* CheckpointProver in the store whose epoch is at or below it.
|
|
770
|
+
*/ async expireEpoch(epoch) {
|
|
771
|
+
try {
|
|
772
|
+
const blocks = await this.l2BlockSource.getBlocks({
|
|
773
|
+
epoch,
|
|
774
|
+
onlyCheckpointed: true
|
|
775
|
+
});
|
|
776
|
+
if (blocks.length > 0) {
|
|
777
|
+
this.chonkCache.releaseForBlocks(blocks);
|
|
778
|
+
}
|
|
779
|
+
} catch (err) {
|
|
780
|
+
this.log.warn(`Could not release chonk-cache entries for expired epoch ${epoch}`, err);
|
|
781
|
+
}
|
|
782
|
+
this.checkpointStore.reapExpired(epoch);
|
|
783
|
+
}
|
|
784
|
+
// ---------------- public API ----------------
|
|
785
|
+
/**
|
|
786
|
+
* Schedules proving for the given epoch and returns the job id without waiting for completion.
|
|
787
|
+
*/ async startProof(epochNumber) {
|
|
788
|
+
if (!this.sessionManager) {
|
|
789
|
+
throw new Error('ProverNode not started');
|
|
790
|
+
}
|
|
791
|
+
return await this.sessionManager.startProof(epochNumber);
|
|
792
|
+
}
|
|
793
|
+
// ---------------- Service lifecycle ----------------
|
|
794
|
+
async start() {
|
|
795
|
+
await this.checkpointStore.start();
|
|
796
|
+
await this.publisherFactory.start();
|
|
797
|
+
this.publishingService = new ProofPublishingService({
|
|
798
|
+
publisherFactory: this.publisherFactory,
|
|
799
|
+
l2BlockSource: this.l2BlockSource,
|
|
800
|
+
dateProvider: this.dateProvider,
|
|
801
|
+
config: {
|
|
802
|
+
skipSubmitProof: !!this.config.proverNodeDisableProofPublish
|
|
803
|
+
},
|
|
804
|
+
bindings: this.log.getBindings()
|
|
805
|
+
});
|
|
806
|
+
this.sessionManager = this.createSessionManager(this.publishingService);
|
|
807
|
+
// SessionManager owns its own periodic tick; start it here so it begins picking up
|
|
808
|
+
// epochs that become complete by time (no fresh checkpoint event) and advances once
|
|
809
|
+
// the previous epoch is proven on L1.
|
|
810
|
+
this.sessionManager.start();
|
|
811
|
+
// Now that the store + manager exist, arm the live-state observable gauges.
|
|
812
|
+
this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
|
|
813
|
+
const { lastFullyProvenEpoch } = await this.resolveLastFullyProvenEpoch();
|
|
814
|
+
this.lastExpiredEpoch = lastFullyProvenEpoch;
|
|
815
|
+
this.lastProcessedCheckpoint = await this.computeStartingCheckpoint(lastFullyProvenEpoch);
|
|
816
|
+
this.blockStream = new EventDrivenL2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
|
|
817
|
+
pollIntervalMS: this.config.proverNodePollingIntervalMs,
|
|
818
|
+
tipsOnly: true
|
|
819
|
+
});
|
|
820
|
+
this.blockStream.start();
|
|
821
|
+
// The periodic ticker is the sole driver of the expiry sweep: it fires every poll interval whether
|
|
822
|
+
// or not block-stream events arrive, and RunningPromise never overlaps its own runs, so the sweep's
|
|
823
|
+
// `lastExpiredEpoch` high-water mark advances — and each epoch's post-mortem uploads — exactly once.
|
|
824
|
+
this.expiryTicker = new RunningPromise(()=>this.checkEpochExpiry(), this.log, this.config.proverNodePollingIntervalMs);
|
|
825
|
+
this.expiryTicker.start();
|
|
826
|
+
await this.rewardsMetrics.start();
|
|
827
|
+
this.l1Metrics.start();
|
|
828
|
+
this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
|
|
829
|
+
}
|
|
830
|
+
async stop() {
|
|
831
|
+
this.log.info('Stopping ProverNode');
|
|
832
|
+
this.jobMetrics.stopObservingState();
|
|
833
|
+
await this.blockStream?.stop();
|
|
834
|
+
await this.expiryTicker?.stop();
|
|
835
|
+
if (this.sessionManager) {
|
|
836
|
+
await this.sessionManager.stop();
|
|
837
|
+
}
|
|
838
|
+
if (this.publishingService) {
|
|
839
|
+
// Bound the wait: the publishing service blocks until any in-flight L1 proof-submission tx
|
|
840
|
+
// settles, which can outlast a reasonable shutdown window. On timeout we log and move on —
|
|
841
|
+
// the tx may still mine, but shutdown must not hang on it.
|
|
842
|
+
const publishingService = this.publishingService;
|
|
843
|
+
await executeTimeout(()=>publishingService.stop(), PUBLISHING_SERVICE_STOP_TIMEOUT_MS, 'prover-node publishing-service stop').catch((err)=>this.log.warn(`Timed out stopping proof publishing service`, err));
|
|
844
|
+
}
|
|
845
|
+
await this.checkpointStore.stop();
|
|
846
|
+
this.chonkCache.stop();
|
|
847
|
+
await this.prover.stop();
|
|
848
|
+
await tryStop(this.publisherFactory);
|
|
849
|
+
this.rewardsMetrics.stop();
|
|
850
|
+
this.l1Metrics.stop();
|
|
851
|
+
await this.telemetryClient.stop();
|
|
852
|
+
this.log.info('Stopped ProverNode');
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* Constructs the session manager. Extracted so subclasses (test harness) can swap the
|
|
856
|
+
* implementation. Wired to upload a post-mortem when a full session ends in its own genuine failure
|
|
857
|
+
* (`EpochSession.hasFailed()` — top-tree/submit failed with every prover healthy, so definitively not
|
|
858
|
+
* a prune). A `stopped` session (a prover under it failed) is not uploaded; it recovers on re-add.
|
|
859
|
+
*/ createSessionManager(publishingService) {
|
|
860
|
+
return new SessionManager({
|
|
861
|
+
checkpointStore: this.checkpointStore,
|
|
862
|
+
l2BlockSource: this.l2BlockSource,
|
|
863
|
+
proverFactory: this.prover,
|
|
864
|
+
proverId: this.prover.getProverId(),
|
|
865
|
+
publishingService,
|
|
866
|
+
metrics: this.jobMetrics,
|
|
867
|
+
dateProvider: this.dateProvider,
|
|
868
|
+
config: {
|
|
869
|
+
maxPendingJobs: this.config.proverNodeMaxPendingJobs,
|
|
870
|
+
tickIntervalMs: this.config.proverNodePollingIntervalMs,
|
|
871
|
+
finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs
|
|
872
|
+
},
|
|
873
|
+
onSessionFailed: async (session)=>{
|
|
874
|
+
await this.tryUploadEpochFailure(session.getId(), session.getCheckpoints());
|
|
875
|
+
},
|
|
876
|
+
bindings: this.log.getBindings()
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
/**
|
|
880
|
+
* Installs session hooks for the e2e harness to interpose around top-tree proving
|
|
881
|
+
* (gate, override, or observe it) without monkey-patching the orchestrator factory.
|
|
882
|
+
* Applies to every session constructed after this call.
|
|
883
|
+
*/ setSessionHooks(hooks) {
|
|
884
|
+
if (!this.sessionManager) {
|
|
885
|
+
throw new Error('ProverNode not started; call start() before setting session hooks.');
|
|
886
|
+
}
|
|
887
|
+
this.sessionManager.setSessionHooks(hooks);
|
|
888
|
+
}
|
|
889
|
+
/**
|
|
890
|
+
* Installs checkpoint-prover test hooks (e.g. forcing a sub-tree failure) applied to every
|
|
891
|
+
* CheckpointProver constructed after this call. For the e2e harness only.
|
|
892
|
+
*/ setCheckpointHooks(hooks) {
|
|
893
|
+
this.checkpointStore.setTestHooks(hooks);
|
|
894
|
+
}
|
|
895
|
+
/**
|
|
896
|
+
* Uploads a post-mortem snapshot for an epoch whose full session failed to prove, built from that
|
|
897
|
+
* session's checkpoint provers. Fired from the session manager's `onSessionFailed` callback (a
|
|
898
|
+
* genuine, race-free failure). Exposed as a method so tests can spy on it. No-ops if no failed-epoch
|
|
899
|
+
* store is configured or the checkpoint set is empty.
|
|
900
|
+
*/ async tryUploadEpochFailure(id, checkpoints) {
|
|
901
|
+
if (!this.config.proverNodeFailedEpochStore || checkpoints.length === 0) {
|
|
902
|
+
return undefined;
|
|
903
|
+
}
|
|
904
|
+
const data = await SessionManager.buildProvingData(checkpoints);
|
|
905
|
+
return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, // The session's own id; `uploadEpochProofFailure` already prefixes the path with the epoch number.
|
|
906
|
+
id, data, this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
|
|
907
|
+
}
|
|
908
|
+
/**
|
|
909
|
+
* Uploads a post-mortem for a single failed checkpoint prover, built from just that checkpoint's
|
|
910
|
+
* proving data. Fired (fire-and-forget) from the store's `onFailed` callback for any non-cancel
|
|
911
|
+
* block-proof failure — a genuine sub-tree fault or a prune-induced fork fault alike. No-ops if no
|
|
912
|
+
* failed-epoch store is configured, or if the checkpoint is no longer canonical (a prune left nothing
|
|
913
|
+
* to diagnose). Swallows its own errors so a fire-and-forget caller can't leak.
|
|
914
|
+
*/ async tryUploadCheckpointFailure(prover) {
|
|
915
|
+
if (!this.config.proverNodeFailedEpochStore) {
|
|
916
|
+
return undefined;
|
|
917
|
+
}
|
|
918
|
+
try {
|
|
919
|
+
// A prune-induced fork fault and a genuine sub-tree failure are indistinguishable at the moment the
|
|
920
|
+
// prover rejects (no control-plane cancel has landed yet). But the archiver is the authoritative
|
|
921
|
+
// committed chain: if this checkpoint was pruned out, its last block is no longer canonical there.
|
|
922
|
+
// Only upload for a checkpoint that still exists on-chain — a prune leaves nothing to diagnose, and
|
|
923
|
+
// the snapshot (full world-state + archiver) is expensive to produce and store.
|
|
924
|
+
if (!await this.isCheckpointCanonical(prover.checkpoint)) {
|
|
925
|
+
this.log.debug(`Skipping checkpoint-failure upload for ${prover.id}: no longer canonical (pruned)`, {
|
|
926
|
+
checkpointNumber: prover.checkpoint.number
|
|
927
|
+
});
|
|
928
|
+
return undefined;
|
|
929
|
+
}
|
|
930
|
+
const data = await SessionManager.buildProvingData([
|
|
931
|
+
prover
|
|
932
|
+
]);
|
|
933
|
+
return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, // The prover's content-addressed id; the epoch number is already in the upload path.
|
|
934
|
+
prover.id, data, this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
|
|
935
|
+
} catch (err) {
|
|
936
|
+
this.log.error(`Error uploading checkpoint failure for ${prover.id}`, err);
|
|
937
|
+
return undefined;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
// ---------------- helpers ----------------
|
|
941
|
+
/**
|
|
942
|
+
* True if the checkpoint still exists on the canonical chain: the archiver holds a block at its last
|
|
943
|
+
* block's height whose archive root matches. A prune (fork fault) leaves the block missing or replaced,
|
|
944
|
+
* so this returns false. Protected for direct unit-test access.
|
|
945
|
+
*/ async isCheckpointCanonical(checkpoint) {
|
|
946
|
+
const lastBlock = checkpoint.blocks.at(-1);
|
|
947
|
+
if (!lastBlock) {
|
|
948
|
+
return false;
|
|
949
|
+
}
|
|
950
|
+
const onChain = await this.l2BlockSource.getBlock({
|
|
951
|
+
number: lastBlock.number
|
|
952
|
+
});
|
|
953
|
+
return !!onChain && onChain.archive.root.equals(checkpoint.archive.root);
|
|
954
|
+
}
|
|
955
|
+
getL1Constants() {
|
|
956
|
+
return this.l2BlockSource.getL1Constants();
|
|
957
|
+
}
|
|
958
|
+
/**
|
|
959
|
+
* Returns true if every block in the given epoch is proven on L1. An epoch is only
|
|
960
|
+
* fully proven when its *last* block is proven. Protected for direct unit-test access.
|
|
961
|
+
*/ async isEpochFullyProven(epochNumber, l1Constants) {
|
|
962
|
+
const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
|
|
963
|
+
tag: 'proven'
|
|
964
|
+
});
|
|
965
|
+
if (!provenBlockNumber || provenBlockNumber <= 0) {
|
|
966
|
+
return false;
|
|
967
|
+
}
|
|
968
|
+
const provenHeader = (await this.l2BlockSource.getBlockData({
|
|
969
|
+
number: BlockNumber(provenBlockNumber)
|
|
970
|
+
}))?.header;
|
|
971
|
+
if (!provenHeader) {
|
|
972
|
+
return false;
|
|
973
|
+
}
|
|
974
|
+
const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
|
|
975
|
+
if (epochNumber < provenEpoch) {
|
|
976
|
+
return true;
|
|
977
|
+
}
|
|
978
|
+
if (epochNumber > provenEpoch) {
|
|
979
|
+
return false;
|
|
980
|
+
}
|
|
981
|
+
return this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants);
|
|
982
|
+
}
|
|
983
|
+
/** Protected for direct unit-test access. */ async isProvenBlockLastOfItsEpoch(provenBlockNumber, provenEpoch, l1Constants) {
|
|
984
|
+
const nextHeader = (await this.l2BlockSource.getBlockData({
|
|
985
|
+
number: BlockNumber(provenBlockNumber + 1)
|
|
986
|
+
}))?.header;
|
|
987
|
+
if (nextHeader) {
|
|
988
|
+
return getEpochAtSlot(nextHeader.getSlot(), l1Constants) > provenEpoch;
|
|
989
|
+
}
|
|
990
|
+
return this.l2BlockSource.isEpochComplete(provenEpoch);
|
|
991
|
+
}
|
|
992
|
+
/**
|
|
993
|
+
* Resolves the last fully-proven epoch from L1 proven state, used to seed the catch-up cursor (via
|
|
994
|
+
* `computeStartingCheckpoint`) and `lastExpiredEpoch`. The fully-proven epoch is `provenEpoch` when the
|
|
995
|
+
* proven tip is the last block of its epoch, otherwise `provenEpoch - 1`, or `undefined` if no block is
|
|
996
|
+
* proven yet (so a restart reprocesses the partially-proven epoch rather than trusting a stale tip).
|
|
997
|
+
*/ async resolveLastFullyProvenEpoch() {
|
|
998
|
+
const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
|
|
999
|
+
tag: 'proven'
|
|
1000
|
+
});
|
|
1001
|
+
if (!provenBlockNumber || provenBlockNumber <= 0) {
|
|
1002
|
+
return {
|
|
1003
|
+
lastFullyProvenEpoch: undefined
|
|
1004
|
+
};
|
|
1005
|
+
}
|
|
1006
|
+
const l1Constants = await this.getL1Constants();
|
|
1007
|
+
const provenHeader = (await this.l2BlockSource.getBlockData({
|
|
1008
|
+
number: BlockNumber(provenBlockNumber)
|
|
1009
|
+
}))?.header;
|
|
1010
|
+
if (!provenHeader) {
|
|
1011
|
+
return {
|
|
1012
|
+
lastFullyProvenEpoch: undefined
|
|
1013
|
+
};
|
|
1014
|
+
}
|
|
1015
|
+
const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
|
|
1016
|
+
if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
|
|
1017
|
+
return {
|
|
1018
|
+
lastFullyProvenEpoch: provenEpoch
|
|
1019
|
+
};
|
|
1020
|
+
}
|
|
1021
|
+
const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
|
|
1022
|
+
return {
|
|
1023
|
+
lastFullyProvenEpoch
|
|
1024
|
+
};
|
|
1025
|
+
}
|
|
1026
|
+
/**
|
|
1027
|
+
* Resolves the catch-up cursor seed: the last checkpoint of the last fully-proven epoch, or 0 if none. Seeding
|
|
1028
|
+
* from a checkpoint (rather than a checkpointed tip) guarantees a restart reprocesses every checkpoint of the
|
|
1029
|
+
* partially-proven epoch, since the checkpointed tip can sit ahead of the last fully-proven checkpoint.
|
|
1030
|
+
*/ async computeStartingCheckpoint(lastFullyProvenEpoch) {
|
|
1031
|
+
if (lastFullyProvenEpoch === undefined) {
|
|
1032
|
+
return CheckpointNumber.ZERO;
|
|
1033
|
+
}
|
|
1034
|
+
const checkpoints = await this.l2BlockSource.getCheckpointsData({
|
|
1035
|
+
epoch: lastFullyProvenEpoch
|
|
1036
|
+
});
|
|
1037
|
+
return checkpoints.at(-1)?.checkpointNumber ?? CheckpointNumber.ZERO;
|
|
1038
|
+
}
|
|
1039
|
+
async gatherPreviousBlockHeader(previousBlockNumber) {
|
|
1040
|
+
const data = await this.l2BlockSource.getBlockData({
|
|
1041
|
+
number: BlockNumber(previousBlockNumber)
|
|
1042
|
+
});
|
|
1043
|
+
if (!data?.header) {
|
|
1044
|
+
throw new Error(`Previous block header ${previousBlockNumber} not found`);
|
|
1045
|
+
}
|
|
1046
|
+
return data.header;
|
|
1047
|
+
}
|
|
1048
|
+
validateConfig() {
|
|
1049
|
+
if (this.config.proverNodeFailedEpochStore && (!this.config.dataDirectory || !this.config.l1ChainId || this.config.rollupVersion === undefined)) {
|
|
1050
|
+
this.log.warn(`Invalid prover-node config (missing dataDirectory, l1ChainId, or rollupVersion)`, pick(this.config, 'proverNodeFailedEpochStore', 'dataDirectory', 'l1ChainId', 'rollupVersion'));
|
|
1051
|
+
throw new Error('All of dataDirectory, l1ChainId, and rollupVersion are required if proverNodeFailedEpochStore is set.');
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
// Re-export so handlers can compare states externally.
|
|
1056
|
+
export { EpochProvingJobTerminalState };
|