@aztec/prover-node 0.0.0-test.0 → 0.0.1-commit.03f7ef2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/actions/download-epoch-proving-job.d.ts +18 -0
- package/dest/actions/download-epoch-proving-job.d.ts.map +1 -0
- package/dest/actions/download-epoch-proving-job.js +37 -0
- package/dest/actions/index.d.ts +3 -0
- package/dest/actions/index.d.ts.map +1 -0
- package/dest/actions/index.js +2 -0
- package/dest/actions/rerun-epoch-proving-job.d.ts +11 -0
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -0
- package/dest/actions/rerun-epoch-proving-job.js +40 -0
- package/dest/actions/upload-epoch-proof-failure.d.ts +15 -0
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -0
- package/dest/actions/upload-epoch-proof-failure.js +78 -0
- package/dest/bin/run-failed-epoch.d.ts +2 -0
- package/dest/bin/run-failed-epoch.d.ts.map +1 -0
- package/dest/bin/run-failed-epoch.js +67 -0
- package/dest/config.d.ts +15 -11
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +83 -15
- package/dest/factory.d.ts +16 -12
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +100 -32
- package/dest/index.d.ts +2 -2
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -1
- package/dest/job/epoch-proving-job-data.d.ts +18 -0
- package/dest/job/epoch-proving-job-data.d.ts.map +1 -0
- package/dest/job/epoch-proving-job-data.js +59 -0
- package/dest/job/epoch-proving-job.d.ts +26 -18
- package/dest/job/epoch-proving-job.d.ts.map +1 -1
- package/dest/job/epoch-proving-job.js +193 -84
- package/dest/metrics.d.ts +30 -5
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +147 -35
- package/dest/monitors/epoch-monitor.d.ts +5 -2
- package/dest/monitors/epoch-monitor.d.ts.map +1 -1
- package/dest/monitors/epoch-monitor.js +17 -3
- package/dest/monitors/index.d.ts +1 -1
- package/dest/prover-node-publisher.d.ts +14 -15
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +86 -81
- package/dest/prover-node.d.ts +49 -44
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +186 -112
- package/dest/prover-publisher-factory.d.ts +23 -0
- package/dest/prover-publisher-factory.d.ts.map +1 -0
- package/dest/prover-publisher-factory.js +26 -0
- package/dest/test/index.d.ts +5 -3
- package/dest/test/index.d.ts.map +1 -1
- package/dest/test/index.js +1 -3
- package/package.json +39 -33
- package/src/actions/download-epoch-proving-job.ts +44 -0
- package/src/actions/index.ts +2 -0
- package/src/actions/rerun-epoch-proving-job.ts +61 -0
- package/src/actions/upload-epoch-proof-failure.ts +88 -0
- package/src/bin/run-failed-epoch.ts +77 -0
- package/src/config.ts +111 -25
- package/src/factory.ts +167 -44
- package/src/index.ts +1 -1
- package/src/job/epoch-proving-job-data.ts +82 -0
- package/src/job/epoch-proving-job.ts +255 -83
- package/src/metrics.ts +147 -37
- package/src/monitors/epoch-monitor.ts +19 -7
- package/src/prover-node-publisher.ts +132 -115
- package/src/prover-node.ts +223 -142
- package/src/prover-publisher-factory.ts +39 -0
- package/src/test/index.ts +7 -4
- package/dest/http.d.ts +0 -8
- package/dest/http.d.ts.map +0 -1
- package/dest/http.js +0 -9
- package/dest/prover-coordination/config.d.ts +0 -7
- package/dest/prover-coordination/config.d.ts.map +0 -1
- package/dest/prover-coordination/config.js +0 -11
- package/dest/prover-coordination/factory.d.ts +0 -22
- package/dest/prover-coordination/factory.d.ts.map +0 -1
- package/dest/prover-coordination/factory.js +0 -42
- package/dest/prover-coordination/index.d.ts +0 -3
- package/dest/prover-coordination/index.d.ts.map +0 -1
- package/dest/prover-coordination/index.js +0 -2
- package/src/http.ts +0 -13
- package/src/prover-coordination/config.ts +0 -17
- package/src/prover-coordination/factory.ts +0 -72
- package/src/prover-coordination/index.ts +0 -2
|
@@ -4,57 +4,60 @@ function _ts_decorate(decorators, target, key, desc) {
|
|
|
4
4
|
else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
5
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
6
|
}
|
|
7
|
+
import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
|
|
7
8
|
import { asyncPool } from '@aztec/foundation/async-pool';
|
|
9
|
+
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
10
|
+
import { padArrayEnd } from '@aztec/foundation/collection';
|
|
11
|
+
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
8
12
|
import { createLogger } from '@aztec/foundation/log';
|
|
9
|
-
import { promiseWithResolvers } from '@aztec/foundation/promise';
|
|
13
|
+
import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
10
14
|
import { Timer } from '@aztec/foundation/timer';
|
|
15
|
+
import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
|
|
16
|
+
import { protocolContractsHash } from '@aztec/protocol-contracts';
|
|
17
|
+
import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
|
|
18
|
+
import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
|
|
11
19
|
import { EpochProvingJobTerminalState } from '@aztec/stdlib/interfaces/server';
|
|
20
|
+
import { CheckpointConstantData } from '@aztec/stdlib/rollup';
|
|
21
|
+
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
12
22
|
import { Attributes, trackSpan } from '@aztec/telemetry-client';
|
|
13
23
|
import * as crypto from 'node:crypto';
|
|
24
|
+
import { validateEpochProvingJobData } from './epoch-proving-job-data.js';
|
|
14
25
|
/**
|
|
15
|
-
* Job that grabs a range of blocks from the
|
|
26
|
+
* Job that grabs a range of blocks from the unfinalized chain from L1, gets their txs given their hashes,
|
|
16
27
|
* re-executes their public calls, generates a rollup proof, and submits it to L1. This job will update the
|
|
17
28
|
* world state as part of public call execution via the public processor.
|
|
18
29
|
*/ export class EpochProvingJob {
|
|
30
|
+
data;
|
|
19
31
|
dbProvider;
|
|
20
|
-
epochNumber;
|
|
21
|
-
blocks;
|
|
22
|
-
txs;
|
|
23
32
|
prover;
|
|
24
33
|
publicProcessorFactory;
|
|
25
34
|
publisher;
|
|
26
35
|
l2BlockSource;
|
|
27
|
-
l1ToL2MessageSource;
|
|
28
36
|
metrics;
|
|
29
37
|
deadline;
|
|
30
38
|
config;
|
|
31
|
-
cleanUp;
|
|
32
39
|
state;
|
|
33
40
|
log;
|
|
34
41
|
uuid;
|
|
35
42
|
runPromise;
|
|
43
|
+
epochCheckPromise;
|
|
36
44
|
deadlineTimeoutHandler;
|
|
37
45
|
tracer;
|
|
38
|
-
constructor(
|
|
39
|
-
|
|
40
|
-
}, cleanUp = ()=>Promise.resolve()){
|
|
46
|
+
constructor(data, dbProvider, prover, publicProcessorFactory, publisher, l2BlockSource, metrics, deadline, config){
|
|
47
|
+
this.data = data;
|
|
41
48
|
this.dbProvider = dbProvider;
|
|
42
|
-
this.epochNumber = epochNumber;
|
|
43
|
-
this.blocks = blocks;
|
|
44
|
-
this.txs = txs;
|
|
45
49
|
this.prover = prover;
|
|
46
50
|
this.publicProcessorFactory = publicProcessorFactory;
|
|
47
51
|
this.publisher = publisher;
|
|
48
52
|
this.l2BlockSource = l2BlockSource;
|
|
49
|
-
this.l1ToL2MessageSource = l1ToL2MessageSource;
|
|
50
53
|
this.metrics = metrics;
|
|
51
54
|
this.deadline = deadline;
|
|
52
55
|
this.config = config;
|
|
53
|
-
this.cleanUp = cleanUp;
|
|
54
56
|
this.state = 'initialized';
|
|
55
57
|
this.log = createLogger('prover-node:epoch-proving-job');
|
|
58
|
+
validateEpochProvingJobData(data);
|
|
56
59
|
this.uuid = crypto.randomUUID();
|
|
57
|
-
this.tracer = metrics.
|
|
60
|
+
this.tracer = metrics.tracer;
|
|
58
61
|
}
|
|
59
62
|
getId() {
|
|
60
63
|
return this.uuid;
|
|
@@ -63,23 +66,46 @@ import * as crypto from 'node:crypto';
|
|
|
63
66
|
return this.state;
|
|
64
67
|
}
|
|
65
68
|
getEpochNumber() {
|
|
66
|
-
return this.epochNumber;
|
|
69
|
+
return this.data.epochNumber;
|
|
70
|
+
}
|
|
71
|
+
getDeadline() {
|
|
72
|
+
return this.deadline;
|
|
73
|
+
}
|
|
74
|
+
getProvingData() {
|
|
75
|
+
return this.data;
|
|
76
|
+
}
|
|
77
|
+
get epochNumber() {
|
|
78
|
+
return this.data.epochNumber;
|
|
79
|
+
}
|
|
80
|
+
get checkpoints() {
|
|
81
|
+
return this.data.checkpoints;
|
|
82
|
+
}
|
|
83
|
+
get txs() {
|
|
84
|
+
return this.data.txs;
|
|
85
|
+
}
|
|
86
|
+
get attestations() {
|
|
87
|
+
return this.data.attestations;
|
|
67
88
|
}
|
|
68
89
|
/**
|
|
69
90
|
* Proves the given epoch and submits the proof to L1.
|
|
70
91
|
*/ async run() {
|
|
71
92
|
this.scheduleDeadlineStop();
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
this.
|
|
93
|
+
if (!this.config.skipEpochCheck) {
|
|
94
|
+
await this.scheduleEpochCheck();
|
|
95
|
+
}
|
|
96
|
+
const attestations = this.attestations.map((attestation)=>attestation.toViem());
|
|
97
|
+
const epochNumber = this.epochNumber;
|
|
98
|
+
const epochSizeCheckpoints = this.checkpoints.length;
|
|
99
|
+
const epochSizeBlocks = this.checkpoints.reduce((accum, checkpoint)=>accum + checkpoint.blocks.length, 0);
|
|
100
|
+
const epochSizeTxs = this.checkpoints.reduce((accum, checkpoint)=>accum + checkpoint.blocks.reduce((accumC, block)=>accumC + block.body.txEffects.length, 0), 0);
|
|
101
|
+
const fromCheckpoint = this.checkpoints[0].number;
|
|
102
|
+
const toCheckpoint = this.checkpoints.at(-1).number;
|
|
103
|
+
const fromBlock = this.checkpoints[0].blocks[0].number;
|
|
104
|
+
const toBlock = this.checkpoints.at(-1).blocks.at(-1).number;
|
|
105
|
+
this.log.info(`Starting epoch ${epochNumber} proving job with checkpoints ${fromCheckpoint} to ${toCheckpoint}`, {
|
|
80
106
|
fromBlock,
|
|
81
107
|
toBlock,
|
|
82
|
-
|
|
108
|
+
epochSizeTxs,
|
|
83
109
|
epochNumber,
|
|
84
110
|
uuid: this.uuid
|
|
85
111
|
});
|
|
@@ -88,71 +114,115 @@ import * as crypto from 'node:crypto';
|
|
|
88
114
|
const { promise, resolve } = promiseWithResolvers();
|
|
89
115
|
this.runPromise = promise;
|
|
90
116
|
try {
|
|
91
|
-
this.
|
|
92
|
-
await
|
|
93
|
-
|
|
117
|
+
const blobFieldsPerCheckpoint = this.checkpoints.map((checkpoint)=>checkpoint.toBlobFields());
|
|
118
|
+
const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
|
|
119
|
+
this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
|
|
120
|
+
await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
|
|
121
|
+
// Everything in the epoch should have the same chainId and version.
|
|
122
|
+
const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
|
|
123
|
+
const previousBlockHeaders = this.gatherPreviousBlockHeaders();
|
|
124
|
+
await asyncPool(this.config.parallelBlockLimit ?? 32, this.checkpoints, async (checkpoint)=>{
|
|
94
125
|
this.checkState();
|
|
95
|
-
const
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
previousHeader: previousHeader.hash(),
|
|
107
|
-
uuid: this.uuid,
|
|
108
|
-
...globalVariables
|
|
126
|
+
const checkpointIndex = checkpoint.number - fromCheckpoint;
|
|
127
|
+
const checkpointConstants = CheckpointConstantData.from({
|
|
128
|
+
chainId,
|
|
129
|
+
version,
|
|
130
|
+
vkTreeRoot: getVKTreeRoot(),
|
|
131
|
+
protocolContractsHash: protocolContractsHash,
|
|
132
|
+
proverId: this.prover.getProverId().toField(),
|
|
133
|
+
slotNumber: checkpoint.header.slotNumber,
|
|
134
|
+
coinbase: checkpoint.header.coinbase,
|
|
135
|
+
feeRecipient: checkpoint.header.feeRecipient,
|
|
136
|
+
gasFees: checkpoint.header.gasFees
|
|
109
137
|
});
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
await db.close();
|
|
118
|
-
this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
|
|
119
|
-
blockNumber: block.number,
|
|
120
|
-
blockHash: (await block.hash()).toString(),
|
|
138
|
+
const previousHeader = previousBlockHeaders[checkpointIndex];
|
|
139
|
+
const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
|
|
140
|
+
this.log.verbose(`Starting processing checkpoint ${checkpoint.number}`, {
|
|
141
|
+
number: checkpoint.number,
|
|
142
|
+
checkpointHash: checkpoint.hash().toString(),
|
|
143
|
+
lastArchive: checkpoint.header.lastArchiveRoot,
|
|
144
|
+
previousHeader: previousHeader.hash(),
|
|
121
145
|
uuid: this.uuid
|
|
122
146
|
});
|
|
123
|
-
|
|
124
|
-
|
|
147
|
+
await this.prover.startNewCheckpoint(checkpointIndex, checkpointConstants, l1ToL2Messages, checkpoint.blocks.length, previousHeader);
|
|
148
|
+
for (const block of checkpoint.blocks){
|
|
149
|
+
const globalVariables = block.header.globalVariables;
|
|
150
|
+
const txs = this.getTxs(block);
|
|
151
|
+
this.log.verbose(`Starting processing block ${block.number}`, {
|
|
152
|
+
number: block.number,
|
|
153
|
+
blockHash: (await block.hash()).toString(),
|
|
154
|
+
lastArchive: block.header.lastArchive.root,
|
|
155
|
+
noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
|
|
156
|
+
nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
|
|
157
|
+
publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
|
|
158
|
+
...globalVariables,
|
|
159
|
+
numTxs: txs.length
|
|
160
|
+
});
|
|
161
|
+
// Start block proving
|
|
162
|
+
await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
|
|
163
|
+
// Process public fns
|
|
164
|
+
const db = await this.createFork(BlockNumber(block.number - 1), l1ToL2Messages);
|
|
165
|
+
const config = PublicSimulatorConfig.from({
|
|
166
|
+
proverId: this.prover.getProverId().toField(),
|
|
167
|
+
skipFeeEnforcement: false,
|
|
168
|
+
collectDebugLogs: false,
|
|
169
|
+
collectHints: true,
|
|
170
|
+
collectPublicInputs: true,
|
|
171
|
+
collectStatistics: false
|
|
172
|
+
});
|
|
173
|
+
const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
|
|
174
|
+
const processed = await this.processTxs(publicProcessor, txs);
|
|
175
|
+
await this.prover.addTxs(processed);
|
|
176
|
+
await db.close();
|
|
177
|
+
this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
|
|
178
|
+
blockNumber: block.number,
|
|
179
|
+
blockHash: (await block.hash()).toString(),
|
|
180
|
+
uuid: this.uuid
|
|
181
|
+
});
|
|
182
|
+
// Mark block as completed to pad it
|
|
183
|
+
const expectedBlockHeader = block.header;
|
|
184
|
+
await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
|
|
185
|
+
}
|
|
125
186
|
});
|
|
126
187
|
const executionTime = timer.ms();
|
|
127
188
|
this.progressState('awaiting-prover');
|
|
128
|
-
const { publicInputs, proof } = await this.prover.
|
|
129
|
-
this.log.info(`
|
|
189
|
+
const { publicInputs, proof, batchedBlobInputs } = await this.prover.finalizeEpoch();
|
|
190
|
+
this.log.info(`Finalized proof for epoch ${epochNumber}`, {
|
|
130
191
|
epochNumber,
|
|
131
192
|
uuid: this.uuid,
|
|
132
193
|
duration: timer.ms()
|
|
133
194
|
});
|
|
134
195
|
this.progressState('publishing-proof');
|
|
196
|
+
if (this.config.skipSubmitProof) {
|
|
197
|
+
this.log.info(`Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`);
|
|
198
|
+
this.state = 'completed';
|
|
199
|
+
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
135
202
|
const success = await this.publisher.submitEpochProof({
|
|
136
|
-
|
|
137
|
-
|
|
203
|
+
fromCheckpoint,
|
|
204
|
+
toCheckpoint,
|
|
138
205
|
epochNumber,
|
|
139
206
|
publicInputs,
|
|
140
|
-
proof
|
|
207
|
+
proof,
|
|
208
|
+
batchedBlobInputs,
|
|
209
|
+
attestations
|
|
141
210
|
});
|
|
142
211
|
if (!success) {
|
|
143
212
|
throw new Error('Failed to submit epoch proof to L1');
|
|
144
213
|
}
|
|
145
|
-
this.log.info(`Submitted proof for epoch ${epochNumber} (
|
|
214
|
+
this.log.info(`Submitted proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`, {
|
|
146
215
|
epochNumber,
|
|
147
216
|
uuid: this.uuid
|
|
148
217
|
});
|
|
149
218
|
this.state = 'completed';
|
|
150
|
-
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
|
|
219
|
+
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
|
|
151
220
|
} catch (err) {
|
|
152
221
|
if (err && err.name === 'HaltExecutionError') {
|
|
153
222
|
this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
|
|
154
223
|
uuid: this.uuid,
|
|
155
|
-
epochNumber
|
|
224
|
+
epochNumber,
|
|
225
|
+
details: err.message
|
|
156
226
|
});
|
|
157
227
|
return;
|
|
158
228
|
}
|
|
@@ -160,27 +230,41 @@ import * as crypto from 'node:crypto';
|
|
|
160
230
|
uuid: this.uuid,
|
|
161
231
|
epochNumber
|
|
162
232
|
});
|
|
163
|
-
this.state
|
|
233
|
+
if (this.state === 'processing' || this.state === 'awaiting-prover' || this.state === 'publishing-proof') {
|
|
234
|
+
this.state = 'failed';
|
|
235
|
+
}
|
|
164
236
|
} finally{
|
|
165
237
|
clearTimeout(this.deadlineTimeoutHandler);
|
|
166
|
-
await this.
|
|
238
|
+
await this.epochCheckPromise?.stop();
|
|
167
239
|
await this.prover.stop();
|
|
168
240
|
resolve();
|
|
169
241
|
}
|
|
170
242
|
}
|
|
243
|
+
/**
|
|
244
|
+
* Create a new db fork for tx processing, inserting all L1 to L2.
|
|
245
|
+
* REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
|
|
246
|
+
*/ async createFork(blockNumber, l1ToL2Messages) {
|
|
247
|
+
const db = await this.dbProvider.fork(blockNumber);
|
|
248
|
+
const l1ToL2MessagesPadded = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 'Too many L1 to L2 messages');
|
|
249
|
+
this.log.verbose(`Creating fork at ${blockNumber} with ${l1ToL2Messages.length} L1 to L2 messages`, {
|
|
250
|
+
blockNumber,
|
|
251
|
+
l1ToL2Messages: l1ToL2Messages.map((m)=>m.toString())
|
|
252
|
+
});
|
|
253
|
+
await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
|
|
254
|
+
return db;
|
|
255
|
+
}
|
|
171
256
|
progressState(state) {
|
|
172
257
|
this.checkState();
|
|
173
258
|
this.state = state;
|
|
174
259
|
}
|
|
175
260
|
checkState() {
|
|
176
|
-
if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed') {
|
|
261
|
+
if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed' || this.state === 'reorg') {
|
|
177
262
|
throw new HaltExecutionError(this.state);
|
|
178
263
|
}
|
|
179
264
|
}
|
|
180
265
|
async stop(state = 'stopped') {
|
|
181
266
|
this.state = state;
|
|
182
267
|
this.prover.cancel();
|
|
183
|
-
// TODO(palla/prover): Stop the publisher as well
|
|
184
268
|
if (this.runPromise) {
|
|
185
269
|
await this.runPromise;
|
|
186
270
|
}
|
|
@@ -209,22 +293,45 @@ import * as crypto from 'node:crypto';
|
|
|
209
293
|
}, timeout);
|
|
210
294
|
}
|
|
211
295
|
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
296
|
+
/**
|
|
297
|
+
* Kicks off a running promise that queries the archiver for the set of L2 blocks of the current epoch.
|
|
298
|
+
* If those change, stops the proving job with a `rerun` state, so the node re-enqueues it.
|
|
299
|
+
*/ async scheduleEpochCheck() {
|
|
300
|
+
const l2BlockSource = this.l2BlockSource;
|
|
301
|
+
if (!l2BlockSource) {
|
|
302
|
+
this.log.warn(`No L2 block source available, skipping epoch check`);
|
|
303
|
+
return;
|
|
215
304
|
}
|
|
216
|
-
|
|
305
|
+
const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
|
|
306
|
+
this.epochCheckPromise = new RunningPromise(async ()=>{
|
|
307
|
+
const blocks = await l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
|
|
308
|
+
const blockHashes = await Promise.all(blocks.map((block)=>block.hash()));
|
|
309
|
+
const thisBlocks = this.checkpoints.flatMap((checkpoint)=>checkpoint.blocks);
|
|
310
|
+
const thisBlockHashes = await Promise.all(thisBlocks.map((block)=>block.hash()));
|
|
311
|
+
if (blocks.length !== thisBlocks.length || !blockHashes.every((block, i)=>block.equals(thisBlockHashes[i]))) {
|
|
312
|
+
this.log.warn('Epoch blocks changed underfoot', {
|
|
313
|
+
uuid: this.uuid,
|
|
314
|
+
epochNumber: this.epochNumber,
|
|
315
|
+
oldBlockHashes: thisBlockHashes,
|
|
316
|
+
newBlockHashes: blockHashes
|
|
317
|
+
});
|
|
318
|
+
void this.stop('reorg');
|
|
319
|
+
}
|
|
320
|
+
}, this.log, intervalMs).start();
|
|
321
|
+
this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
|
|
217
322
|
}
|
|
218
|
-
|
|
219
|
-
const
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
323
|
+
/* Returns the last block header in the previous checkpoint for all checkpoints in the epoch */ gatherPreviousBlockHeaders() {
|
|
324
|
+
const lastBlocks = this.checkpoints.map((checkpoint)=>checkpoint.blocks.at(-1));
|
|
325
|
+
return [
|
|
326
|
+
this.data.previousBlockHeader,
|
|
327
|
+
...lastBlocks.map((block)=>block.header).slice(0, -1)
|
|
328
|
+
];
|
|
329
|
+
}
|
|
330
|
+
getTxs(block) {
|
|
331
|
+
return block.body.txEffects.map((txEffect)=>this.txs.get(txEffect.txHash.toString()));
|
|
225
332
|
}
|
|
226
|
-
getL1ToL2Messages(
|
|
227
|
-
return this.
|
|
333
|
+
getL1ToL2Messages(checkpoint) {
|
|
334
|
+
return this.data.l1ToL2Messages[checkpoint.number];
|
|
228
335
|
}
|
|
229
336
|
async processTxs(publicProcessor, txs) {
|
|
230
337
|
const { deadline } = this;
|
|
@@ -232,7 +339,8 @@ import * as crypto from 'node:crypto';
|
|
|
232
339
|
deadline
|
|
233
340
|
});
|
|
234
341
|
if (failedTxs.length) {
|
|
235
|
-
|
|
342
|
+
const failedTxHashes = await Promise.all(failedTxs.map(({ tx })=>tx.getTxHash()));
|
|
343
|
+
throw new Error(`Txs failed processing: ${failedTxs.map(({ error }, index)=>`${failedTxHashes[index]} (${error})`).join(', ')}`);
|
|
236
344
|
}
|
|
237
345
|
if (processedTxs.length !== txs.length) {
|
|
238
346
|
throw new Error(`Failed to process all txs: processed ${processedTxs.length} out of ${txs.length}`);
|
|
@@ -243,13 +351,14 @@ import * as crypto from 'node:crypto';
|
|
|
243
351
|
_ts_decorate([
|
|
244
352
|
trackSpan('EpochProvingJob.run', function() {
|
|
245
353
|
return {
|
|
246
|
-
[Attributes.EPOCH_NUMBER]:
|
|
354
|
+
[Attributes.EPOCH_NUMBER]: this.data.epochNumber
|
|
247
355
|
};
|
|
248
356
|
})
|
|
249
357
|
], EpochProvingJob.prototype, "run", null);
|
|
250
358
|
class HaltExecutionError extends Error {
|
|
359
|
+
state;
|
|
251
360
|
constructor(state){
|
|
252
|
-
super(`Halted execution due to state ${state}`);
|
|
361
|
+
super(`Halted execution due to state ${state}`), this.state = state;
|
|
253
362
|
this.name = 'HaltExecutionError';
|
|
254
363
|
}
|
|
255
364
|
}
|
package/dest/metrics.d.ts
CHANGED
|
@@ -1,12 +1,36 @@
|
|
|
1
|
+
import type { RollupContract } from '@aztec/ethereum/contracts';
|
|
2
|
+
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
1
3
|
import type { L1PublishProofStats } from '@aztec/stdlib/stats';
|
|
2
|
-
import { type Gauge, type Histogram, type TelemetryClient, type UpDownCounter } from '@aztec/telemetry-client';
|
|
3
|
-
export declare class
|
|
4
|
-
|
|
4
|
+
import { type Gauge, type Histogram, type Meter, type TelemetryClient, type Tracer, type UpDownCounter } from '@aztec/telemetry-client';
|
|
5
|
+
export declare class ProverNodeJobMetrics {
|
|
6
|
+
private meter;
|
|
7
|
+
readonly tracer: Tracer;
|
|
5
8
|
private logger;
|
|
6
9
|
proverEpochExecutionDuration: Histogram;
|
|
7
10
|
provingJobDuration: Histogram;
|
|
11
|
+
provingJobCheckpoints: Gauge;
|
|
8
12
|
provingJobBlocks: Gauge;
|
|
9
13
|
provingJobTransactions: Gauge;
|
|
14
|
+
constructor(meter: Meter, tracer: Tracer, logger?: import("@aztec/foundation/log").Logger);
|
|
15
|
+
recordProvingJob(executionTimeMs: number, totalTimeMs: number, numCheckpoints: number, numBlocks: number, numTxs: number): void;
|
|
16
|
+
}
|
|
17
|
+
export declare class ProverNodeRewardsMetrics {
|
|
18
|
+
private meter;
|
|
19
|
+
private coinbase;
|
|
20
|
+
private rollup;
|
|
21
|
+
private logger;
|
|
22
|
+
private rewards;
|
|
23
|
+
private accumulatedRewards;
|
|
24
|
+
private prevEpoch;
|
|
25
|
+
private proofSubmissionEpochs;
|
|
26
|
+
constructor(meter: Meter, coinbase: EthAddress, rollup: RollupContract, logger?: import("@aztec/foundation/log").Logger);
|
|
27
|
+
start(): Promise<void>;
|
|
28
|
+
stop(): void;
|
|
29
|
+
private observe;
|
|
30
|
+
}
|
|
31
|
+
export declare class ProverNodePublisherMetrics {
|
|
32
|
+
readonly client: TelemetryClient;
|
|
33
|
+
private logger;
|
|
10
34
|
gasPrice: Histogram;
|
|
11
35
|
txCount: UpDownCounter;
|
|
12
36
|
txDuration: Histogram;
|
|
@@ -15,12 +39,13 @@ export declare class ProverNodeMetrics {
|
|
|
15
39
|
txCalldataGas: Histogram;
|
|
16
40
|
txBlobDataGasUsed: Histogram;
|
|
17
41
|
txBlobDataGasCost: Histogram;
|
|
42
|
+
txTotalFee: Histogram;
|
|
18
43
|
private senderBalance;
|
|
44
|
+
private meter;
|
|
19
45
|
constructor(client: TelemetryClient, name?: string, logger?: import("@aztec/foundation/log").Logger);
|
|
20
46
|
recordFailedTx(): void;
|
|
21
47
|
recordSubmitProof(durationMs: number, stats: L1PublishProofStats): void;
|
|
22
|
-
recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number): void;
|
|
23
48
|
recordSenderBalance(wei: bigint, senderAddress: string): void;
|
|
24
49
|
private recordTx;
|
|
25
50
|
}
|
|
26
|
-
//# sourceMappingURL=
|
|
51
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWV0cmljcy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL21ldHJpY3MudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsY0FBYyxFQUFFLE1BQU0sMkJBQTJCLENBQUM7QUFDaEUsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFFaEUsT0FBTyxLQUFLLEVBQUUsbUJBQW1CLEVBQWtCLE1BQU0scUJBQXFCLENBQUM7QUFDL0UsT0FBTyxFQUdMLEtBQUssS0FBSyxFQUNWLEtBQUssU0FBUyxFQUNkLEtBQUssS0FBSyxFQUdWLEtBQUssZUFBZSxFQUNwQixLQUFLLE1BQU0sRUFDWCxLQUFLLGFBQWEsRUFFbkIsTUFBTSx5QkFBeUIsQ0FBQztBQUlqQyxxQkFBYSxvQkFBb0I7SUFRN0IsT0FBTyxDQUFDLEtBQUs7YUFDRyxNQUFNLEVBQUUsTUFBTTtJQUM5QixPQUFPLENBQUMsTUFBTTtJQVRoQiw0QkFBNEIsRUFBRSxTQUFTLENBQUM7SUFDeEMsa0JBQWtCLEVBQUUsU0FBUyxDQUFDO0lBQzlCLHFCQUFxQixFQUFFLEtBQUssQ0FBQztJQUM3QixnQkFBZ0IsRUFBRSxLQUFLLENBQUM7SUFDeEIsc0JBQXNCLEVBQUUsS0FBSyxDQUFDO0lBRTlCLFlBQ1UsS0FBSyxFQUFFLEtBQUssRUFDSixNQUFNLEVBQUUsTUFBTSxFQUN0QixNQUFNLHlDQUFnRCxFQXdCL0Q7SUFFTSxnQkFBZ0IsQ0FDckIsZUFBZSxFQUFFLE1BQU0sRUFDdkIsV0FBVyxFQUFFLE1BQU0sRUFDbkIsY0FBYyxFQUFFLE1BQU0sRUFDdEIsU0FBUyxFQUFFLE1BQU0sRUFDakIsTUFBTSxFQUFFLE1BQU0sUUFPZjtDQUNGO0FBRUQscUJBQWEsd0JBQXdCO0lBT2pDLE9BQU8sQ0FBQyxLQUFLO0lBQ2IsT0FBTyxDQUFDLFFBQVE7SUFDaEIsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsTUFBTTtJQVRoQixPQUFPLENBQUMsT0FBTyxDQUFrQjtJQUNqQyxPQUFPLENBQUMsa0JBQWtCLENBQWdCO0lBQzFDLE9BQU8sQ0FBQyxTQUFTLENBQU87SUFDeEIsT0FBTyxDQUFDLHFCQUFxQixDQUFLO0lBRWxDLFlBQ1UsS0FBSyxFQUFFLEtBQUssRUFDWixRQUFRLEVBQUUsVUFBVSxFQUNwQixNQUFNLEVBQUUsY0FBYyxFQUN0QixNQUFNLHlDQUFnRCxFQVcvRDtJQUVZLEtBQUssa0JBSWpCO0lBRU0sSUFBSSxTQUVWO0lBRUQsT0FBTyxDQUFDLE9BQU8sQ0F3QmI7Q0FDSDtBQUVELHFCQUFhLDBCQUEwQjthQWVuQixNQUFNLEVBQUUsZUFBZTtJQUV2QyxPQUFPLENBQUMsTUFBTTtJQWhCaEIsUUFBUSxFQUFFLFNBQVMsQ0FBQztJQUNwQixPQUFPLEVBQUUsYUFBYSxDQUFDO0lBQ3ZCLFVBQVUsRUFBRSxTQUFTLENBQUM7SUFDdEIsS0FBSyxFQUFFLFNBQVMsQ0FBQztJQUNqQixjQUFjLEVBQUUsU0FBUyxDQUFDO0lBQzFCLGFBQWEsRUFBRSxTQUFTLENBQUM7SUFDekIsaUJBQWlCLEVBQUUsU0FBUyxDQUFDO0lBQzdCLGlCQUFpQixFQUFFLFNBQVMsQ0FBQztJQUM3QixVQUFVLEVBQUUsU0FBUyxDQUFDO0lBRXRCLE9BQU8sQ0FBQyxhQUFhLENBQVE7SUFDN0IsT0FBTyxDQUFDLEtBQUssQ0FBUTtJQUVyQixZQUNrQixNQUFNLEVBQUUsZUFBZSxFQUN2QyxJQUFJLFNBQWUsRUFDWCxNQUFNLHlDQUFnRCxFQWtFL0Q7SUFFRCxjQUFjLFNBS2I7SUFFRCxpQkFBaUIsQ0FBQyxVQUFVLEVBQUUsTUFBTSxFQUFFLEtBQUssRUFBRSxtQkFBbUIsUUFFL0Q7SUFFTSxtQkFBbUIsQ0FBQyxHQUFHLEVBQUUsTUFBTSxFQUFFLGFBQWEsRUFBRSxNQUFNLFFBSzVEO0lBRUQsT0FBTyxDQUFDLFFBQVE7Q0F1Q2pCIn0=
|
package/dest/metrics.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAEhE,OAAO,KAAK,EAAE,mBAAmB,EAAkB,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAGL,KAAK,KAAK,EACV,KAAK,SAAS,EACd,KAAK,KAAK,EAGV,KAAK,eAAe,EACpB,KAAK,MAAM,EACX,KAAK,aAAa,EAEnB,MAAM,yBAAyB,CAAC;AAIjC,qBAAa,oBAAoB;IAQ7B,OAAO,CAAC,KAAK;aACG,MAAM,EAAE,MAAM;IAC9B,OAAO,CAAC,MAAM;IAThB,4BAA4B,EAAE,SAAS,CAAC;IACxC,kBAAkB,EAAE,SAAS,CAAC;IAC9B,qBAAqB,EAAE,KAAK,CAAC;IAC7B,gBAAgB,EAAE,KAAK,CAAC;IACxB,sBAAsB,EAAE,KAAK,CAAC;IAE9B,YACU,KAAK,EAAE,KAAK,EACJ,MAAM,EAAE,MAAM,EACtB,MAAM,yCAAgD,EAwB/D;IAEM,gBAAgB,CACrB,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,QAOf;CACF;AAED,qBAAa,wBAAwB;IAOjC,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAThB,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAgB;IAC1C,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,qBAAqB,CAAK;IAElC,YACU,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,cAAc,EACtB,MAAM,yCAAgD,EAW/D;IAEY,KAAK,kBAIjB;IAEM,IAAI,SAEV;IAED,OAAO,CAAC,OAAO,CAwBb;CACH;AAED,qBAAa,0BAA0B;aAenB,MAAM,EAAE,eAAe;IAEvC,OAAO,CAAC,MAAM;IAhBhB,QAAQ,EAAE,SAAS,CAAC;IACpB,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,EAAE,SAAS,CAAC;IACtB,KAAK,EAAE,SAAS,CAAC;IACjB,cAAc,EAAE,SAAS,CAAC;IAC1B,aAAa,EAAE,SAAS,CAAC;IACzB,iBAAiB,EAAE,SAAS,CAAC;IAC7B,iBAAiB,EAAE,SAAS,CAAC;IAC7B,UAAU,EAAE,SAAS,CAAC;IAEtB,OAAO,CAAC,aAAa,CAAQ;IAC7B,OAAO,CAAC,KAAK,CAAQ;IAErB,YACkB,MAAM,EAAE,eAAe,EACvC,IAAI,SAAe,EACX,MAAM,yCAAgD,EAkE/D;IAED,cAAc,SAKb;IAED,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,QAE/D;IAEM,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,QAK5D;IAED,OAAO,CAAC,QAAQ;CAuCjB"}
|