@aztec/prover-node 0.0.1-commit.96bb3f7 → 0.0.1-commit.993d240
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/actions/download-epoch-proving-job.d.ts +1 -1
- package/dest/actions/download-epoch-proving-job.js +1 -1
- package/dest/actions/rerun-epoch-proving-job.d.ts +5 -3
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +9 -7
- package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
- package/dest/bin/run-failed-epoch.js +6 -5
- package/dest/config.d.ts +5 -8
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +16 -19
- package/dest/factory.d.ts +19 -13
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +40 -64
- package/dest/job/epoch-proving-job.d.ts +7 -2
- package/dest/job/epoch-proving-job.d.ts.map +1 -1
- package/dest/job/epoch-proving-job.js +213 -39
- package/dest/metrics.d.ts +21 -1
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +58 -3
- package/dest/monitors/epoch-monitor.d.ts +1 -1
- package/dest/monitors/epoch-monitor.d.ts.map +1 -1
- package/dest/monitors/epoch-monitor.js +11 -9
- package/dest/prover-node-publisher.d.ts +25 -5
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +205 -9
- package/dest/prover-node.d.ts +20 -11
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +32 -27
- package/dest/prover-publisher-factory.d.ts +7 -5
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +7 -5
- package/package.json +24 -23
- package/src/actions/download-epoch-proving-job.ts +1 -1
- package/src/actions/rerun-epoch-proving-job.ts +20 -7
- package/src/actions/upload-epoch-proof-failure.ts +1 -1
- package/src/bin/run-failed-epoch.ts +5 -3
- package/src/config.ts +23 -31
- package/src/factory.ts +68 -104
- package/src/job/epoch-proving-job.ts +151 -47
- package/src/metrics.ts +77 -2
- package/src/monitors/epoch-monitor.ts +5 -6
- package/src/prover-node-publisher.ts +239 -14
- package/src/prover-node.ts +31 -31
- package/src/prover-publisher-factory.ts +16 -10
|
@@ -1,17 +1,16 @@
|
|
|
1
|
-
import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
|
|
2
1
|
import { asyncPool } from '@aztec/foundation/async-pool';
|
|
3
2
|
import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
|
|
4
|
-
import { padArrayEnd } from '@aztec/foundation/collection';
|
|
5
3
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
6
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
4
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
7
5
|
import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
8
6
|
import { Timer } from '@aztec/foundation/timer';
|
|
7
|
+
import { AVM_MAX_CONCURRENT_SIMULATIONS } from '@aztec/native';
|
|
9
8
|
import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
|
|
10
9
|
import { protocolContractsHash } from '@aztec/protocol-contracts';
|
|
11
10
|
import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
|
|
12
11
|
import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
|
|
13
12
|
import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
|
|
14
|
-
import type {
|
|
13
|
+
import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
|
|
15
14
|
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
16
15
|
import {
|
|
17
16
|
type EpochProver,
|
|
@@ -19,8 +18,8 @@ import {
|
|
|
19
18
|
EpochProvingJobTerminalState,
|
|
20
19
|
type ForkMerkleTreeOperations,
|
|
21
20
|
} from '@aztec/stdlib/interfaces/server';
|
|
21
|
+
import { appendL1ToL2MessagesToTree } from '@aztec/stdlib/messaging';
|
|
22
22
|
import { CheckpointConstantData } from '@aztec/stdlib/rollup';
|
|
23
|
-
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
24
23
|
import type { ProcessedTx, Tx } from '@aztec/stdlib/tx';
|
|
25
24
|
import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
|
|
26
25
|
|
|
@@ -43,10 +42,11 @@ export type EpochProvingJobOptions = {
|
|
|
43
42
|
*/
|
|
44
43
|
export class EpochProvingJob implements Traceable {
|
|
45
44
|
private state: EpochProvingJobState = 'initialized';
|
|
46
|
-
private log
|
|
45
|
+
private log: Logger;
|
|
47
46
|
private uuid: string;
|
|
48
47
|
|
|
49
48
|
private runPromise: Promise<void> | undefined;
|
|
49
|
+
private abortController = new AbortController();
|
|
50
50
|
private epochCheckPromise: RunningPromise | undefined;
|
|
51
51
|
private deadlineTimeoutHandler: NodeJS.Timeout | undefined;
|
|
52
52
|
|
|
@@ -57,14 +57,19 @@ export class EpochProvingJob implements Traceable {
|
|
|
57
57
|
private dbProvider: Pick<ForkMerkleTreeOperations, 'fork'>,
|
|
58
58
|
private prover: EpochProver,
|
|
59
59
|
private publicProcessorFactory: PublicProcessorFactory,
|
|
60
|
-
private publisher: Pick<ProverNodePublisher, 'submitEpochProof'>,
|
|
60
|
+
private publisher: Pick<ProverNodePublisher, 'submitEpochProof' | 'analyzeEpochProofSubmission'>,
|
|
61
61
|
private l2BlockSource: L2BlockSource | undefined,
|
|
62
62
|
private metrics: ProverNodeJobMetrics,
|
|
63
63
|
private deadline: Date | undefined,
|
|
64
64
|
private config: EpochProvingJobOptions,
|
|
65
|
+
bindings?: LoggerBindings,
|
|
65
66
|
) {
|
|
66
67
|
validateEpochProvingJobData(data);
|
|
67
68
|
this.uuid = crypto.randomUUID();
|
|
69
|
+
this.log = createLogger('prover-node:epoch-proving-job', {
|
|
70
|
+
...bindings,
|
|
71
|
+
instanceId: `epoch-${data.epochNumber}`,
|
|
72
|
+
});
|
|
68
73
|
this.tracer = metrics.tracer;
|
|
69
74
|
}
|
|
70
75
|
|
|
@@ -143,19 +148,32 @@ export class EpochProvingJob implements Traceable {
|
|
|
143
148
|
this.runPromise = promise;
|
|
144
149
|
|
|
145
150
|
try {
|
|
151
|
+
const blobTimer = new Timer();
|
|
146
152
|
const blobFieldsPerCheckpoint = this.checkpoints.map(checkpoint => checkpoint.toBlobFields());
|
|
147
153
|
const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
|
|
154
|
+
this.metrics.recordBlobProcessing(blobTimer.ms());
|
|
148
155
|
|
|
149
156
|
this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
|
|
157
|
+
const chonkTimer = new Timer();
|
|
150
158
|
await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
|
|
159
|
+
this.metrics.recordChonkVerifier(chonkTimer.ms());
|
|
151
160
|
|
|
152
161
|
// Everything in the epoch should have the same chainId and version.
|
|
153
162
|
const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
|
|
154
163
|
|
|
155
164
|
const previousBlockHeaders = this.gatherPreviousBlockHeaders();
|
|
156
165
|
|
|
157
|
-
|
|
166
|
+
const allCheckpointsTimer = new Timer();
|
|
167
|
+
|
|
168
|
+
const parallelism = this.config.parallelBlockLimit
|
|
169
|
+
? this.config.parallelBlockLimit
|
|
170
|
+
: AVM_MAX_CONCURRENT_SIMULATIONS > 0
|
|
171
|
+
? AVM_MAX_CONCURRENT_SIMULATIONS
|
|
172
|
+
: this.checkpoints.length;
|
|
173
|
+
|
|
174
|
+
await this.processCheckpoints(parallelism, async checkpoint => {
|
|
158
175
|
this.checkState();
|
|
176
|
+
const checkpointTimer = new Timer();
|
|
159
177
|
|
|
160
178
|
const checkpointIndex = checkpoint.number - fromCheckpoint;
|
|
161
179
|
const checkpointConstants = CheckpointConstantData.from({
|
|
@@ -172,11 +190,12 @@ export class EpochProvingJob implements Traceable {
|
|
|
172
190
|
const previousHeader = previousBlockHeaders[checkpointIndex];
|
|
173
191
|
const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
|
|
174
192
|
|
|
175
|
-
this.log.
|
|
193
|
+
this.log.debug(`Starting processing checkpoint ${checkpoint.number}`, {
|
|
176
194
|
number: checkpoint.number,
|
|
177
195
|
checkpointHash: checkpoint.hash().toString(),
|
|
178
|
-
|
|
179
|
-
|
|
196
|
+
headerHash: checkpoint.header.hash().toString(),
|
|
197
|
+
numL1ToL2Messages: l1ToL2Messages.length,
|
|
198
|
+
previousBlockNumber: previousHeader.globalVariables.blockNumber,
|
|
180
199
|
uuid: this.uuid,
|
|
181
200
|
});
|
|
182
201
|
|
|
@@ -188,7 +207,9 @@ export class EpochProvingJob implements Traceable {
|
|
|
188
207
|
previousHeader,
|
|
189
208
|
);
|
|
190
209
|
|
|
191
|
-
for (
|
|
210
|
+
for (let blockIndex = 0; blockIndex < checkpoint.blocks.length; blockIndex++) {
|
|
211
|
+
const blockTimer = new Timer();
|
|
212
|
+
const block = checkpoint.blocks[blockIndex];
|
|
192
213
|
const globalVariables = block.header.globalVariables;
|
|
193
214
|
const txs = this.getTxs(block);
|
|
194
215
|
|
|
@@ -206,20 +227,28 @@ export class EpochProvingJob implements Traceable {
|
|
|
206
227
|
// Start block proving
|
|
207
228
|
await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
|
|
208
229
|
|
|
209
|
-
// Process public fns
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
230
|
+
// Process public fns. L1 to L2 messages are only inserted for the first block of a checkpoint,
|
|
231
|
+
// as the fork for subsequent blocks already includes them from the previous block's synced state.
|
|
232
|
+
{
|
|
233
|
+
await using db = await this.createFork(
|
|
234
|
+
BlockNumber(block.number - 1),
|
|
235
|
+
blockIndex === 0 ? l1ToL2Messages : undefined,
|
|
236
|
+
);
|
|
237
|
+
this.checkState();
|
|
238
|
+
const config = PublicSimulatorConfig.from({
|
|
239
|
+
proverId: this.prover.getProverId().toField(),
|
|
240
|
+
skipFeeEnforcement: false,
|
|
241
|
+
collectDebugLogs: false,
|
|
242
|
+
collectHints: true,
|
|
243
|
+
collectPublicInputs: true,
|
|
244
|
+
collectStatistics: false,
|
|
245
|
+
});
|
|
246
|
+
const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
|
|
247
|
+
const processed = await this.processTxs(publicProcessor, txs);
|
|
248
|
+
this.checkState();
|
|
249
|
+
await this.prover.addTxs(processed);
|
|
250
|
+
}
|
|
251
|
+
this.checkState();
|
|
223
252
|
this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
|
|
224
253
|
blockNumber: block.number,
|
|
225
254
|
blockHash: (await block.hash()).toString(),
|
|
@@ -229,8 +258,11 @@ export class EpochProvingJob implements Traceable {
|
|
|
229
258
|
// Mark block as completed to pad it
|
|
230
259
|
const expectedBlockHeader = block.header;
|
|
231
260
|
await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
|
|
261
|
+
this.metrics.recordBlockProcessing(blockTimer.ms());
|
|
232
262
|
}
|
|
263
|
+
this.metrics.recordCheckpointProcessing(checkpointTimer.ms());
|
|
233
264
|
});
|
|
265
|
+
this.metrics.recordAllCheckpointsProcessing(allCheckpointsTimer.ms());
|
|
234
266
|
|
|
235
267
|
const executionTime = timer.ms();
|
|
236
268
|
|
|
@@ -242,8 +274,21 @@ export class EpochProvingJob implements Traceable {
|
|
|
242
274
|
|
|
243
275
|
if (this.config.skipSubmitProof) {
|
|
244
276
|
this.log.info(
|
|
245
|
-
`Proof publishing is disabled.
|
|
277
|
+
`Proof publishing is disabled. Analyzing estimated L1 fees for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`,
|
|
246
278
|
);
|
|
279
|
+
try {
|
|
280
|
+
await this.publisher.analyzeEpochProofSubmission({
|
|
281
|
+
fromCheckpoint,
|
|
282
|
+
toCheckpoint,
|
|
283
|
+
epochNumber,
|
|
284
|
+
publicInputs,
|
|
285
|
+
proof,
|
|
286
|
+
batchedBlobInputs,
|
|
287
|
+
attestations,
|
|
288
|
+
});
|
|
289
|
+
} catch (err) {
|
|
290
|
+
this.log.warn(`Failed to analyze estimated L1 fees for epoch ${epochNumber}`, err);
|
|
291
|
+
}
|
|
247
292
|
this.state = 'completed';
|
|
248
293
|
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
|
|
249
294
|
return;
|
|
@@ -290,25 +335,63 @@ export class EpochProvingJob implements Traceable {
|
|
|
290
335
|
}
|
|
291
336
|
|
|
292
337
|
/**
|
|
293
|
-
* Create a new db fork for tx processing, inserting
|
|
338
|
+
* Create a new db fork for tx processing, optionally inserting L1 to L2 messages.
|
|
339
|
+
* L1 to L2 messages should only be inserted for the first block in a checkpoint,
|
|
340
|
+
* as subsequent blocks' synced state already includes them.
|
|
294
341
|
* REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
|
|
295
342
|
*/
|
|
296
|
-
private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[]) {
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
343
|
+
private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[] | undefined) {
|
|
344
|
+
this.log.verbose(`Creating fork at ${blockNumber}`, { blockNumber });
|
|
345
|
+
// temporary stack to control fork lifetime
|
|
346
|
+
await using cleanup = new AsyncDisposableStack();
|
|
347
|
+
const db = cleanup.use(await this.dbProvider.fork(blockNumber));
|
|
348
|
+
|
|
349
|
+
if (l1ToL2Messages !== undefined) {
|
|
350
|
+
this.log.verbose(`Inserting ${l1ToL2Messages.length} L1 to L2 messages in fork`, {
|
|
351
|
+
blockNumber,
|
|
352
|
+
l1ToL2Messages: l1ToL2Messages.map(m => m.toString()),
|
|
353
|
+
});
|
|
354
|
+
await appendL1ToL2MessagesToTree(db, l1ToL2Messages);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// everything run succesfully so we can release this stack and give control of the fork's lifetime to the caller
|
|
358
|
+
cleanup.move();
|
|
309
359
|
return db;
|
|
310
360
|
}
|
|
311
361
|
|
|
362
|
+
private async processCheckpoints(
|
|
363
|
+
parallelism: number,
|
|
364
|
+
processCheckpoint: (checkpoint: Checkpoint) => Promise<void>,
|
|
365
|
+
): Promise<void> {
|
|
366
|
+
let hasError = false;
|
|
367
|
+
let firstError: unknown;
|
|
368
|
+
|
|
369
|
+
await asyncPool(Math.max(parallelism, 1), this.checkpoints, async checkpoint => {
|
|
370
|
+
if (hasError || this.abortController.signal.aborted) {
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
try {
|
|
375
|
+
this.checkState();
|
|
376
|
+
await processCheckpoint(checkpoint);
|
|
377
|
+
} catch (err) {
|
|
378
|
+
if (!hasError) {
|
|
379
|
+
hasError = true;
|
|
380
|
+
firstError = err;
|
|
381
|
+
this.failProcessing();
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
if (hasError) {
|
|
387
|
+
throw firstError;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
if (this.abortController.signal.aborted) {
|
|
391
|
+
this.checkState();
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
312
395
|
private progressState(state: EpochProvingJobState) {
|
|
313
396
|
this.checkState();
|
|
314
397
|
this.state = state;
|
|
@@ -322,12 +405,24 @@ export class EpochProvingJob implements Traceable {
|
|
|
322
405
|
|
|
323
406
|
public async stop(state: EpochProvingJobTerminalState = 'stopped') {
|
|
324
407
|
this.state = state;
|
|
325
|
-
this.
|
|
408
|
+
this.interruptProcessing();
|
|
326
409
|
if (this.runPromise) {
|
|
327
410
|
await this.runPromise;
|
|
328
411
|
}
|
|
329
412
|
}
|
|
330
413
|
|
|
414
|
+
private failProcessing() {
|
|
415
|
+
if (!EpochProvingJobTerminalState.includes(this.state)) {
|
|
416
|
+
this.state = 'failed';
|
|
417
|
+
}
|
|
418
|
+
this.interruptProcessing();
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
private interruptProcessing() {
|
|
422
|
+
this.abortController.abort();
|
|
423
|
+
this.prover.cancel();
|
|
424
|
+
}
|
|
425
|
+
|
|
331
426
|
private scheduleDeadlineStop() {
|
|
332
427
|
const deadline = this.deadline;
|
|
333
428
|
if (deadline) {
|
|
@@ -362,11 +457,16 @@ export class EpochProvingJob implements Traceable {
|
|
|
362
457
|
const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
|
|
363
458
|
this.epochCheckPromise = new RunningPromise(
|
|
364
459
|
async () => {
|
|
365
|
-
const
|
|
366
|
-
|
|
460
|
+
const blockHeaders = (
|
|
461
|
+
await l2BlockSource.getBlocksData({ epoch: this.epochNumber, onlyCheckpointed: true })
|
|
462
|
+
).map(d => d.header);
|
|
463
|
+
const blockHashes = await Promise.all(blockHeaders.map(header => header.hash()));
|
|
367
464
|
const thisBlocks = this.checkpoints.flatMap(checkpoint => checkpoint.blocks);
|
|
368
465
|
const thisBlockHashes = await Promise.all(thisBlocks.map(block => block.hash()));
|
|
369
|
-
if (
|
|
466
|
+
if (
|
|
467
|
+
blockHeaders.length !== thisBlocks.length ||
|
|
468
|
+
!blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))
|
|
469
|
+
) {
|
|
370
470
|
this.log.warn('Epoch blocks changed underfoot', {
|
|
371
471
|
uuid: this.uuid,
|
|
372
472
|
epochNumber: this.epochNumber,
|
|
@@ -388,7 +488,7 @@ export class EpochProvingJob implements Traceable {
|
|
|
388
488
|
return [this.data.previousBlockHeader, ...lastBlocks.map(block => block.header).slice(0, -1)];
|
|
389
489
|
}
|
|
390
490
|
|
|
391
|
-
private getTxs(block:
|
|
491
|
+
private getTxs(block: L2Block): Tx[] {
|
|
392
492
|
return block.body.txEffects.map(txEffect => this.txs.get(txEffect.txHash.toString())!);
|
|
393
493
|
}
|
|
394
494
|
|
|
@@ -398,7 +498,11 @@ export class EpochProvingJob implements Traceable {
|
|
|
398
498
|
|
|
399
499
|
private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {
|
|
400
500
|
const { deadline } = this;
|
|
401
|
-
const [processedTxs, failedTxs] = await publicProcessor.process(txs, {
|
|
501
|
+
const [processedTxs, failedTxs] = await publicProcessor.process(txs, {
|
|
502
|
+
deadline,
|
|
503
|
+
signal: this.abortController.signal,
|
|
504
|
+
});
|
|
505
|
+
this.checkState();
|
|
402
506
|
|
|
403
507
|
if (failedTxs.length) {
|
|
404
508
|
const failedTxHashes = await Promise.all(failedTxs.map(({ tx }) => tx.getTxHash()));
|
package/src/metrics.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
type TelemetryClient,
|
|
14
14
|
type Tracer,
|
|
15
15
|
type UpDownCounter,
|
|
16
|
+
createUpDownCounterWithDefault,
|
|
16
17
|
} from '@aztec/telemetry-client';
|
|
17
18
|
|
|
18
19
|
import { formatEther, formatUnits } from 'viem';
|
|
@@ -24,6 +25,12 @@ export class ProverNodeJobMetrics {
|
|
|
24
25
|
provingJobBlocks: Gauge;
|
|
25
26
|
provingJobTransactions: Gauge;
|
|
26
27
|
|
|
28
|
+
private blobProcessingDuration: Gauge;
|
|
29
|
+
private chonkVerifierDuration: Gauge;
|
|
30
|
+
private blockProcessingDuration: Histogram;
|
|
31
|
+
private checkpointProcessingDuration: Histogram;
|
|
32
|
+
private allCheckpointsProcessingDuration: Gauge;
|
|
33
|
+
|
|
27
34
|
constructor(
|
|
28
35
|
private meter: Meter,
|
|
29
36
|
public readonly tracer: Tracer,
|
|
@@ -34,6 +41,14 @@ export class ProverNodeJobMetrics {
|
|
|
34
41
|
this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
|
|
35
42
|
this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
|
|
36
43
|
this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS);
|
|
44
|
+
|
|
45
|
+
this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
|
|
46
|
+
this.chonkVerifierDuration = this.meter.createGauge(Metrics.PROVER_NODE_CHONK_VERIFIER_LAST_DURATION);
|
|
47
|
+
this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
|
|
48
|
+
this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
|
|
49
|
+
this.allCheckpointsProcessingDuration = this.meter.createGauge(
|
|
50
|
+
Metrics.PROVER_NODE_ALL_CHECKPOINTS_PROCESSING_LAST_DURATION,
|
|
51
|
+
);
|
|
37
52
|
}
|
|
38
53
|
|
|
39
54
|
public recordProvingJob(
|
|
@@ -49,6 +64,26 @@ export class ProverNodeJobMetrics {
|
|
|
49
64
|
this.provingJobBlocks.record(Math.floor(numBlocks));
|
|
50
65
|
this.provingJobTransactions.record(Math.floor(numTxs));
|
|
51
66
|
}
|
|
67
|
+
|
|
68
|
+
public recordBlobProcessing(durationMs: number) {
|
|
69
|
+
this.blobProcessingDuration.record(Math.ceil(durationMs));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
public recordChonkVerifier(durationMs: number) {
|
|
73
|
+
this.chonkVerifierDuration.record(Math.ceil(durationMs));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
public recordBlockProcessing(durationMs: number) {
|
|
77
|
+
this.blockProcessingDuration.record(Math.ceil(durationMs));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
public recordCheckpointProcessing(durationMs: number) {
|
|
81
|
+
this.checkpointProcessingDuration.record(Math.ceil(durationMs));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
public recordAllCheckpointsProcessing(durationMs: number) {
|
|
85
|
+
this.allCheckpointsProcessingDuration.record(Math.ceil(durationMs));
|
|
86
|
+
}
|
|
52
87
|
}
|
|
53
88
|
|
|
54
89
|
export class ProverNodeRewardsMetrics {
|
|
@@ -65,7 +100,7 @@ export class ProverNodeRewardsMetrics {
|
|
|
65
100
|
) {
|
|
66
101
|
this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH);
|
|
67
102
|
|
|
68
|
-
this.accumulatedRewards = this.meter
|
|
103
|
+
this.accumulatedRewards = createUpDownCounterWithDefault(this.meter, Metrics.PROVER_NODE_REWARDS_TOTAL);
|
|
69
104
|
}
|
|
70
105
|
|
|
71
106
|
public async start() {
|
|
@@ -105,6 +140,13 @@ export class ProverNodeRewardsMetrics {
|
|
|
105
140
|
};
|
|
106
141
|
}
|
|
107
142
|
|
|
143
|
+
export type EstimatedSubmitProofStats = {
|
|
144
|
+
gasLimit: bigint;
|
|
145
|
+
baseFeePerGas: bigint;
|
|
146
|
+
maxPriorityFeePerGas: bigint;
|
|
147
|
+
estimatedTotalFee: bigint;
|
|
148
|
+
};
|
|
149
|
+
|
|
108
150
|
export class ProverNodePublisherMetrics {
|
|
109
151
|
gasPrice: Histogram;
|
|
110
152
|
txCount: UpDownCounter;
|
|
@@ -116,6 +158,10 @@ export class ProverNodePublisherMetrics {
|
|
|
116
158
|
txBlobDataGasCost: Histogram;
|
|
117
159
|
txTotalFee: Histogram;
|
|
118
160
|
|
|
161
|
+
private txGasEstimated: Histogram;
|
|
162
|
+
private gasPriceEstimated: Histogram;
|
|
163
|
+
private txTotalFeeEstimated: Histogram;
|
|
164
|
+
|
|
119
165
|
private senderBalance: Gauge;
|
|
120
166
|
private meter: Meter;
|
|
121
167
|
|
|
@@ -128,7 +174,10 @@ export class ProverNodePublisherMetrics {
|
|
|
128
174
|
|
|
129
175
|
this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE);
|
|
130
176
|
|
|
131
|
-
this.txCount = this.meter
|
|
177
|
+
this.txCount = createUpDownCounterWithDefault(this.meter, Metrics.L1_PUBLISHER_TX_COUNT, {
|
|
178
|
+
[Attributes.L1_TX_TYPE]: ['submitProof'],
|
|
179
|
+
[Attributes.OK]: [true, false],
|
|
180
|
+
});
|
|
132
181
|
|
|
133
182
|
this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION);
|
|
134
183
|
|
|
@@ -144,6 +193,12 @@ export class ProverNodePublisherMetrics {
|
|
|
144
193
|
|
|
145
194
|
this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE);
|
|
146
195
|
|
|
196
|
+
this.txGasEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS);
|
|
197
|
+
|
|
198
|
+
this.gasPriceEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE);
|
|
199
|
+
|
|
200
|
+
this.txTotalFeeEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE);
|
|
201
|
+
|
|
147
202
|
this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE);
|
|
148
203
|
}
|
|
149
204
|
|
|
@@ -158,6 +213,26 @@ export class ProverNodePublisherMetrics {
|
|
|
158
213
|
this.recordTx(durationMs, stats);
|
|
159
214
|
}
|
|
160
215
|
|
|
216
|
+
public recordEstimatedSubmitProof(stats: EstimatedSubmitProofStats) {
|
|
217
|
+
const attributes = { [Attributes.L1_TX_TYPE]: 'submitProof' } as const;
|
|
218
|
+
|
|
219
|
+
this.txGasEstimated.record(Number(stats.gasLimit), attributes);
|
|
220
|
+
|
|
221
|
+
try {
|
|
222
|
+
this.gasPriceEstimated.record(
|
|
223
|
+
parseInt(formatEther(stats.baseFeePerGas + stats.maxPriorityFeePerGas, 'gwei'), 10),
|
|
224
|
+
);
|
|
225
|
+
} catch {
|
|
226
|
+
// ignore
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
try {
|
|
230
|
+
this.txTotalFeeEstimated.record(parseFloat(formatEther(stats.estimatedTotalFee)));
|
|
231
|
+
} catch {
|
|
232
|
+
// ignore
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
161
236
|
public recordSenderBalance(wei: bigint, senderAddress: string) {
|
|
162
237
|
const eth = parseFloat(formatEther(wei, 'wei'));
|
|
163
238
|
this.senderBalance.record(eth, {
|
|
@@ -69,9 +69,8 @@ export class EpochMonitor implements Traceable {
|
|
|
69
69
|
|
|
70
70
|
public async work() {
|
|
71
71
|
const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
|
|
72
|
-
this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
|
|
73
72
|
if (epochToProve === undefined) {
|
|
74
|
-
this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
|
|
73
|
+
this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { epochToProve, blockNumber, slotNumber });
|
|
75
74
|
return;
|
|
76
75
|
}
|
|
77
76
|
if (this.latestEpochNumber !== undefined && epochToProve <= this.latestEpochNumber) {
|
|
@@ -86,20 +85,20 @@ export class EpochMonitor implements Traceable {
|
|
|
86
85
|
}
|
|
87
86
|
|
|
88
87
|
if (this.options.provingDelayMs) {
|
|
89
|
-
this.log.
|
|
88
|
+
this.log.warn(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
|
|
90
89
|
await sleep(this.options.provingDelayMs);
|
|
91
90
|
}
|
|
92
91
|
|
|
93
|
-
this.log.
|
|
92
|
+
this.log.verbose(`Epoch ${epochToProve} is ready to be proven`);
|
|
94
93
|
if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
|
|
95
94
|
this.latestEpochNumber = epochToProve;
|
|
96
95
|
}
|
|
97
96
|
}
|
|
98
97
|
|
|
99
98
|
private async getEpochNumberToProve() {
|
|
100
|
-
const lastBlockProven = await this.l2BlockSource.
|
|
99
|
+
const lastBlockProven = (await this.l2BlockSource.getBlockNumber({ tag: 'proven' })) ?? BlockNumber.ZERO;
|
|
101
100
|
const firstBlockToProve = BlockNumber(lastBlockProven + 1);
|
|
102
|
-
const firstBlockHeaderToProve = await this.l2BlockSource.
|
|
101
|
+
const firstBlockHeaderToProve = (await this.l2BlockSource.getBlockData({ number: firstBlockToProve }))?.header;
|
|
103
102
|
if (!firstBlockHeaderToProve) {
|
|
104
103
|
return { epochToProve: undefined, blockNumber: firstBlockToProve };
|
|
105
104
|
}
|