@aztec/prover-node 0.0.1-commit.181e2d196 → 0.0.1-commit.189eedb3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/actions/download-epoch-proving-job.js +1 -1
- package/dest/actions/rerun-epoch-proving-job.d.ts +4 -3
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +105 -27
- package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
- package/dest/bin/run-failed-epoch.js +1 -3
- package/dest/config.d.ts +2 -2
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +2 -2
- package/dest/factory.d.ts +1 -1
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +26 -6
- package/dest/job/epoch-proving-job.d.ts +6 -2
- package/dest/job/epoch-proving-job.d.ts.map +1 -1
- package/dest/job/epoch-proving-job.js +198 -38
- package/dest/metrics.d.ts +21 -1
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +47 -0
- package/dest/monitors/epoch-monitor.d.ts +1 -1
- package/dest/monitors/epoch-monitor.d.ts.map +1 -1
- package/dest/monitors/epoch-monitor.js +11 -9
- package/dest/prover-node-publisher.d.ts +20 -1
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +200 -5
- package/dest/prover-node.d.ts +7 -7
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +82 -35
- package/dest/prover-publisher-factory.d.ts +2 -2
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +3 -3
- package/package.json +23 -22
- package/src/actions/download-epoch-proving-job.ts +1 -1
- package/src/actions/rerun-epoch-proving-job.ts +17 -6
- package/src/actions/upload-epoch-proof-failure.ts +1 -1
- package/src/bin/run-failed-epoch.ts +1 -2
- package/src/config.ts +2 -2
- package/src/factory.ts +21 -4
- package/src/job/epoch-proving-job.ts +120 -38
- package/src/metrics.ts +71 -0
- package/src/monitors/epoch-monitor.ts +5 -6
- package/src/prover-node-publisher.ts +232 -9
- package/src/prover-node.ts +87 -48
- package/src/prover-publisher-factory.ts +3 -3
|
@@ -3,14 +3,17 @@ import { MAX_CHECKPOINTS_PER_EPOCH } from '@aztec/constants';
|
|
|
3
3
|
import { makeTuple } from '@aztec/foundation/array';
|
|
4
4
|
import { CheckpointNumber } from '@aztec/foundation/branded-types';
|
|
5
5
|
import { areArraysEqual } from '@aztec/foundation/collection';
|
|
6
|
+
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
6
7
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
7
8
|
import { createLogger } from '@aztec/foundation/log';
|
|
9
|
+
import { retryUntil } from '@aztec/foundation/retry';
|
|
8
10
|
import { Timer } from '@aztec/foundation/timer';
|
|
9
11
|
import { RollupAbi } from '@aztec/l1-artifacts';
|
|
10
12
|
import { CommitteeAttestation, CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
|
|
13
|
+
import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
11
14
|
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
12
15
|
import { inspect } from 'util';
|
|
13
|
-
import { encodeFunctionData } from 'viem';
|
|
16
|
+
import { encodeFunctionData, formatEther, formatGwei } from 'viem';
|
|
14
17
|
import { ProverNodePublisherMetrics } from './metrics.js';
|
|
15
18
|
export class ProverNodePublisher {
|
|
16
19
|
interrupted = false;
|
|
@@ -52,6 +55,10 @@ export class ProverNodePublisher {
|
|
|
52
55
|
toCheckpoint
|
|
53
56
|
};
|
|
54
57
|
if (!this.interrupted) {
|
|
58
|
+
if (!await this.waitUntilStartBuildsOnProven(args)) {
|
|
59
|
+
this.log.verbose('Checkpoint data syncing interrupted', ctx);
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
55
62
|
const timer = new Timer();
|
|
56
63
|
// Validate epoch proof range and hashes are correct before submitting
|
|
57
64
|
await this.validateEpochProofSubmission(args);
|
|
@@ -92,6 +99,42 @@ export class ProverNodePublisher {
|
|
|
92
99
|
this.log.verbose('Checkpoint data syncing interrupted', ctx);
|
|
93
100
|
return false;
|
|
94
101
|
}
|
|
102
|
+
async waitUntilStartBuildsOnProven(args) {
|
|
103
|
+
const { epochNumber, fromCheckpoint } = args;
|
|
104
|
+
const provenCheckpoint = await this.getProvenCheckpoint();
|
|
105
|
+
if (this.isStartBuildingOnProven(fromCheckpoint, provenCheckpoint)) {
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
const timeout = await this.getSecondsUntilProofSubmissionWindowEnd(epochNumber);
|
|
109
|
+
this.log.info(`Waiting for proven checkpoint to reach proof start`, {
|
|
110
|
+
epochNumber,
|
|
111
|
+
fromCheckpoint,
|
|
112
|
+
provenCheckpoint,
|
|
113
|
+
timeout
|
|
114
|
+
});
|
|
115
|
+
await retryUntil(async ()=>{
|
|
116
|
+
if (this.interrupted) {
|
|
117
|
+
return true;
|
|
118
|
+
}
|
|
119
|
+
const proven = await this.getProvenCheckpoint();
|
|
120
|
+
this.log.verbose(`Proven checkpoint is at ${proven} (waiting for ${fromCheckpoint - 1})`, {
|
|
121
|
+
epochNumber
|
|
122
|
+
});
|
|
123
|
+
return this.isStartBuildingOnProven(fromCheckpoint, proven) ? true : undefined;
|
|
124
|
+
}, `proven checkpoint to reach ${fromCheckpoint - 1}`, timeout, 4);
|
|
125
|
+
return !this.interrupted;
|
|
126
|
+
}
|
|
127
|
+
async getProvenCheckpoint() {
|
|
128
|
+
return (await this.rollupContract.getTips()).proven;
|
|
129
|
+
}
|
|
130
|
+
isStartBuildingOnProven(fromCheckpoint, provenCheckpoint) {
|
|
131
|
+
return fromCheckpoint - 1 <= provenCheckpoint;
|
|
132
|
+
}
|
|
133
|
+
async getSecondsUntilProofSubmissionWindowEnd(epochNumber) {
|
|
134
|
+
const deadline = getProofSubmissionDeadlineTimestamp(epochNumber, await this.rollupContract.getRollupConstants());
|
|
135
|
+
const now = BigInt(Math.floor(Date.now() / 1000));
|
|
136
|
+
return Math.max(Number(deadline - now), 0.001);
|
|
137
|
+
}
|
|
95
138
|
async validateEpochProofSubmission(args) {
|
|
96
139
|
const { fromCheckpoint, toCheckpoint, publicInputs, batchedBlobInputs } = args;
|
|
97
140
|
// Check that the checkpoint numbers match the expected epoch to be proven
|
|
@@ -102,7 +145,7 @@ export class ProverNodePublisher {
|
|
|
102
145
|
}
|
|
103
146
|
// toCheckpoint can't be greater than pending
|
|
104
147
|
if (toCheckpoint > pending) {
|
|
105
|
-
throw new Error(`Cannot submit epoch proof for ${fromCheckpoint}-${toCheckpoint} as
|
|
148
|
+
throw new Error(`Cannot submit epoch proof for ${fromCheckpoint}-${toCheckpoint} as proposed checkpoint is ${pending}`);
|
|
106
149
|
}
|
|
107
150
|
// Check the archive for the immediate checkpoint before the epoch
|
|
108
151
|
const checkpointLog = await this.rollupContract.getCheckpoint(CheckpointNumber(fromCheckpoint - 1));
|
|
@@ -125,10 +168,65 @@ export class ProverNodePublisher {
|
|
|
125
168
|
...publicInputs.toFields()
|
|
126
169
|
];
|
|
127
170
|
if (!areArraysEqual(rollupPublicInputs, argsPublicInputs, (a, b)=>a.equals(b))) {
|
|
128
|
-
|
|
129
|
-
|
|
171
|
+
throw await reportPublicInputsMismatch({
|
|
172
|
+
rollupPublicInputs,
|
|
173
|
+
argsPublicInputs,
|
|
174
|
+
fromCheckpoint,
|
|
175
|
+
toCheckpoint,
|
|
176
|
+
rollupContract: this.rollupContract,
|
|
177
|
+
log: this.log
|
|
178
|
+
});
|
|
130
179
|
}
|
|
131
180
|
}
|
|
181
|
+
/**
|
|
182
|
+
* Estimates what submitting the epoch proof would have cost on L1 without actually sending it.
|
|
183
|
+
* Runs the same validation as `submitEpochProof`, encodes the calldata, estimates gas, and records metrics.
|
|
184
|
+
* Used when proof publishing is disabled (e.g. PROVER_NODE_DISABLE_PROOF_PUBLISH=true on mainnet).
|
|
185
|
+
*/ async analyzeEpochProofSubmission(args) {
|
|
186
|
+
const { epochNumber, fromCheckpoint, toCheckpoint } = args;
|
|
187
|
+
await this.validateEpochProofSubmission(args);
|
|
188
|
+
const data = this.encodeSubmitEpochProofCalldata(args);
|
|
189
|
+
const senderAddress = this.l1TxUtils.getSenderAddress();
|
|
190
|
+
const [gasLimit, gasPrice, latestBlock] = await Promise.all([
|
|
191
|
+
this.l1TxUtils.estimateGas(senderAddress.toString(), {
|
|
192
|
+
to: this.rollupContract.address,
|
|
193
|
+
data
|
|
194
|
+
}),
|
|
195
|
+
this.l1TxUtils.getGasPrice(),
|
|
196
|
+
this.l1TxUtils.client.getBlock({
|
|
197
|
+
blockTag: 'latest'
|
|
198
|
+
})
|
|
199
|
+
]);
|
|
200
|
+
const baseFeePerGas = latestBlock.baseFeePerGas ?? 0n;
|
|
201
|
+
const { maxPriorityFeePerGas } = gasPrice;
|
|
202
|
+
const effectiveFeePerGas = baseFeePerGas + maxPriorityFeePerGas;
|
|
203
|
+
const estimatedTotalFee = gasLimit * effectiveFeePerGas;
|
|
204
|
+
const stats = {
|
|
205
|
+
gasLimit,
|
|
206
|
+
baseFeePerGas,
|
|
207
|
+
maxPriorityFeePerGas,
|
|
208
|
+
estimatedTotalFee
|
|
209
|
+
};
|
|
210
|
+
this.log.info(`Estimated epoch proof submission cost (not submitted)`, {
|
|
211
|
+
epochNumber,
|
|
212
|
+
fromCheckpoint,
|
|
213
|
+
toCheckpoint,
|
|
214
|
+
gasLimit: gasLimit.toString(),
|
|
215
|
+
baseFeePerGas: formatGwei(baseFeePerGas),
|
|
216
|
+
maxPriorityFeePerGas: formatGwei(maxPriorityFeePerGas),
|
|
217
|
+
estimatedTotalFeeEth: formatEther(estimatedTotalFee)
|
|
218
|
+
});
|
|
219
|
+
this.metrics.recordEstimatedSubmitProof(stats);
|
|
220
|
+
}
|
|
221
|
+
encodeSubmitEpochProofCalldata(args) {
|
|
222
|
+
return encodeFunctionData({
|
|
223
|
+
abi: RollupAbi,
|
|
224
|
+
functionName: 'submitEpochRootProof',
|
|
225
|
+
args: [
|
|
226
|
+
this.getSubmitEpochProofArgs(args)
|
|
227
|
+
]
|
|
228
|
+
});
|
|
229
|
+
}
|
|
132
230
|
async sendSubmitEpochProofTx(args) {
|
|
133
231
|
const txArgs = [
|
|
134
232
|
this.getSubmitEpochProofArgs(args)
|
|
@@ -190,9 +288,106 @@ export class ProverNodePublisher {
|
|
|
190
288
|
end: argsArray[1],
|
|
191
289
|
args: argsArray[2],
|
|
192
290
|
fees: argsArray[3],
|
|
193
|
-
attestations:
|
|
291
|
+
attestations: CommitteeAttestationsAndSigners.packAttestations(args.attestations.map((a)=>CommitteeAttestation.fromViem(a))),
|
|
194
292
|
blobInputs: argsArray[4],
|
|
195
293
|
proof: proofHex
|
|
196
294
|
};
|
|
197
295
|
}
|
|
198
296
|
}
|
|
297
|
+
/**
|
|
298
|
+
* Decodes a `Root rollup public inputs mismatch`, fetches the on-chain CheckpointLog for any
|
|
299
|
+
* mismatching `checkpointHeaderHashes[i]`, emits a structured error log, and returns a thrown-ready
|
|
300
|
+
* Error with a human-readable summary.
|
|
301
|
+
*
|
|
302
|
+
* Layout of `RootRollupPublicInputs.toFields()`:
|
|
303
|
+
* [0] previousArchiveRoot
|
|
304
|
+
* [1] endArchiveRoot
|
|
305
|
+
* [2] outHash
|
|
306
|
+
* [3 .. 3+N-1] checkpointHeaderHashes[i] for i in 0..N-1 (N = MAX_CHECKPOINTS_PER_EPOCH)
|
|
307
|
+
* [3+N .. 3+3N-1] fees[i] = (recipient, value) for i in 0..N-1
|
|
308
|
+
* [3+3N .. 3+3N+4] EpochConstantData (chainId, version, vkTreeRoot, protocolContractsHash, proverId)
|
|
309
|
+
* [3+3N+5 ..] blobPublicInputs (FinalBlobAccumulator)
|
|
310
|
+
*/ async function reportPublicInputsMismatch(input) {
|
|
311
|
+
const { rollupPublicInputs, argsPublicInputs, fromCheckpoint, toCheckpoint, rollupContract, log } = input;
|
|
312
|
+
const N = MAX_CHECKPOINTS_PER_EPOCH;
|
|
313
|
+
const constantsStart = 3 + 3 * N;
|
|
314
|
+
const blobStart = constantsStart + 5;
|
|
315
|
+
const constantLabels = [
|
|
316
|
+
'chainId',
|
|
317
|
+
'version',
|
|
318
|
+
'vkTreeRoot',
|
|
319
|
+
'protocolContractsHash',
|
|
320
|
+
'proverId'
|
|
321
|
+
];
|
|
322
|
+
const diffs = [];
|
|
323
|
+
const len = Math.max(rollupPublicInputs.length, argsPublicInputs.length);
|
|
324
|
+
for(let i = 0; i < len; i++){
|
|
325
|
+
const a = rollupPublicInputs[i] ?? Fr.ZERO;
|
|
326
|
+
const b = argsPublicInputs[i] ?? Fr.ZERO;
|
|
327
|
+
if (a.equals(b)) {
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
330
|
+
let label;
|
|
331
|
+
let checkpointIndex;
|
|
332
|
+
if (i === 0) {
|
|
333
|
+
label = 'previousArchiveRoot';
|
|
334
|
+
} else if (i === 1) {
|
|
335
|
+
label = 'endArchiveRoot';
|
|
336
|
+
} else if (i === 2) {
|
|
337
|
+
label = 'outHash';
|
|
338
|
+
} else if (i < 3 + N) {
|
|
339
|
+
checkpointIndex = i - 3;
|
|
340
|
+
label = `checkpointHeaderHashes[${checkpointIndex}]`;
|
|
341
|
+
} else if (i < 3 + 3 * N) {
|
|
342
|
+
const feePairIndex = i - (3 + N);
|
|
343
|
+
const feeIndex = Math.floor(feePairIndex / 2);
|
|
344
|
+
const sub = feePairIndex % 2 === 0 ? 'recipient' : 'value';
|
|
345
|
+
label = `fees[${feeIndex}].${sub}`;
|
|
346
|
+
} else if (i < blobStart) {
|
|
347
|
+
label = `constants.${constantLabels[i - constantsStart]}`;
|
|
348
|
+
} else {
|
|
349
|
+
label = `blobPublicInputs[${i - blobStart}]`;
|
|
350
|
+
}
|
|
351
|
+
diffs.push({
|
|
352
|
+
index: i,
|
|
353
|
+
label,
|
|
354
|
+
rollup: a,
|
|
355
|
+
computed: b,
|
|
356
|
+
checkpointIndex
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
// For each mismatching checkpointHeaderHash, fetch the L1 CheckpointLog so the operator can
|
|
360
|
+
// see what was published on-chain alongside the prover's recomputed hash.
|
|
361
|
+
const onChainCheckpoints = await Promise.all(diffs.filter((d)=>d.checkpointIndex !== undefined).map(async (d)=>{
|
|
362
|
+
const checkpointNumber = CheckpointNumber(fromCheckpoint + d.checkpointIndex);
|
|
363
|
+
try {
|
|
364
|
+
const cp = await rollupContract.getCheckpoint(checkpointNumber);
|
|
365
|
+
return {
|
|
366
|
+
checkpointIndex: d.checkpointIndex,
|
|
367
|
+
checkpointNumber,
|
|
368
|
+
headerHash: cp.headerHash.toString()
|
|
369
|
+
};
|
|
370
|
+
} catch (err) {
|
|
371
|
+
return {
|
|
372
|
+
checkpointIndex: d.checkpointIndex,
|
|
373
|
+
checkpointNumber,
|
|
374
|
+
error: err.message
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
}));
|
|
378
|
+
log.error(`Root rollup public inputs mismatch`, undefined, {
|
|
379
|
+
fromCheckpoint,
|
|
380
|
+
toCheckpoint,
|
|
381
|
+
numDiffs: diffs.length,
|
|
382
|
+
diffs: diffs.map((d)=>({
|
|
383
|
+
index: d.index,
|
|
384
|
+
label: d.label,
|
|
385
|
+
rollup: d.rollup.toString(),
|
|
386
|
+
computed: d.computed.toString()
|
|
387
|
+
})),
|
|
388
|
+
onChainCheckpoints
|
|
389
|
+
});
|
|
390
|
+
const fmt = (inputs)=>inputs.map((x)=>x.toString()).join(', ');
|
|
391
|
+
const summary = diffs.map((d)=>`[${d.index} ${d.label}] L1=${d.rollup} prover=${d.computed}`).join('\n');
|
|
392
|
+
return new Error(`Root rollup public inputs mismatch (${diffs.length} fields differ):\n${summary}\n` + `Rollup: ${fmt(rollupPublicInputs)}\nComputed:${fmt(argsPublicInputs)}`);
|
|
393
|
+
}
|
package/dest/prover-node.d.ts
CHANGED
|
@@ -2,12 +2,12 @@ import type { RollupContract } from '@aztec/ethereum/contracts';
|
|
|
2
2
|
import type { Delayer } from '@aztec/ethereum/l1-tx-utils';
|
|
3
3
|
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
4
4
|
import { DateProvider } from '@aztec/foundation/timer';
|
|
5
|
-
import type { DataStoreConfig } from '@aztec/kv-store/config';
|
|
6
5
|
import { PublicProcessorFactory } from '@aztec/simulator/server';
|
|
7
6
|
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
8
7
|
import type { ChainConfig } from '@aztec/stdlib/config';
|
|
9
8
|
import type { ContractDataSource } from '@aztec/stdlib/contract';
|
|
10
9
|
import { type EpochProverManager, type ITxProvider, type ProverNodeApi, type Service, type WorldStateSyncStatus, type WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
|
|
10
|
+
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
11
11
|
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
12
12
|
import { L1Metrics, type TelemetryClient, type Traceable, type Tracer } from '@aztec/telemetry-client';
|
|
13
13
|
import type { SpecificProverNodeConfig } from './config.js';
|
|
@@ -44,6 +44,7 @@ export declare class ProverNode implements EpochMonitorHandler, ProverNodeApi, T
|
|
|
44
44
|
private config;
|
|
45
45
|
private jobMetrics;
|
|
46
46
|
private rewardsMetrics;
|
|
47
|
+
private startingProofEpochs;
|
|
47
48
|
readonly tracer: Tracer;
|
|
48
49
|
protected publisher: ProverNodePublisher | undefined;
|
|
49
50
|
constructor(prover: EpochProverManager, publisherFactory: ProverPublisherFactory, l2BlockSource: L2BlockSource & Partial<Service>, l1ToL2MessageSource: L1ToL2MessageSource, contractDataSource: ContractDataSource, worldState: WorldStateSynchronizer, p2pClient: {
|
|
@@ -79,6 +80,8 @@ export declare class ProverNode implements EpochMonitorHandler, ProverNodeApi, T
|
|
|
79
80
|
* Starts a proving process and returns immediately.
|
|
80
81
|
*/
|
|
81
82
|
startProof(epochNumber: EpochNumber): Promise<void>;
|
|
83
|
+
private startProofIfNeeded;
|
|
84
|
+
private startProofInternal;
|
|
82
85
|
private runJob;
|
|
83
86
|
protected tryUploadEpochFailure(job: EpochProvingJob): Promise<string | undefined>;
|
|
84
87
|
/**
|
|
@@ -93,15 +96,12 @@ export declare class ProverNode implements EpochMonitorHandler, ProverNodeApi, T
|
|
|
93
96
|
status: EpochProvingJobState;
|
|
94
97
|
epochNumber: EpochNumber;
|
|
95
98
|
}[]>;
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
status: EpochProvingJobState;
|
|
99
|
-
}[]>;
|
|
99
|
+
private getJobsInternal;
|
|
100
|
+
private activeJobsCoverEpoch;
|
|
100
101
|
private checkMaximumPendingJobs;
|
|
101
102
|
private createProvingJob;
|
|
102
103
|
private getL1Constants;
|
|
103
104
|
private gatherEpochData;
|
|
104
|
-
private gatherCheckpoints;
|
|
105
105
|
private gatherTxs;
|
|
106
106
|
private gatherMessages;
|
|
107
107
|
private gatherPreviousBlockHeader;
|
|
@@ -114,4 +114,4 @@ export declare class ProverNode implements EpochMonitorHandler, ProverNodeApi, T
|
|
|
114
114
|
private validateConfig;
|
|
115
115
|
}
|
|
116
116
|
export {};
|
|
117
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
117
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmVyLW5vZGUuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9wcm92ZXItbm9kZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxPQUFPLEtBQUssRUFBRSxjQUFjLEVBQUUsTUFBTSwyQkFBMkIsQ0FBQztBQUNoRSxPQUFPLEtBQUssRUFBRSxPQUFPLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUMzRCxPQUFPLEVBQWlDLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBSzdGLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUN2RCxPQUFPLEVBQUUsc0JBQXNCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUNqRSxPQUFPLEtBQUssRUFBRSxhQUFhLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUV6RCxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUN4RCxPQUFPLEtBQUssRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHdCQUF3QixDQUFDO0FBRWpFLE9BQU8sRUFDTCxLQUFLLGtCQUFrQixFQUN2QixLQUFLLFdBQVcsRUFDaEIsS0FBSyxhQUFhLEVBQ2xCLEtBQUssT0FBTyxFQUNaLEtBQUssb0JBQW9CLEVBQ3pCLEtBQUssc0JBQXNCLEVBRTVCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxLQUFLLEVBQUUsZUFBZSxFQUFFLE1BQU0sd0JBQXdCLENBQUM7QUFDOUQsT0FBTyxLQUFLLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUVuRSxPQUFPLEVBRUwsU0FBUyxFQUNULEtBQUssZUFBZSxFQUNwQixLQUFLLFNBQVMsRUFDZCxLQUFLLE1BQU0sRUFHWixNQUFNLHlCQUF5QixDQUFDO0FBR2pDLE9BQU8sS0FBSyxFQUFFLHdCQUF3QixFQUFFLE1BQU0sYUFBYSxDQUFDO0FBQzVELE9BQU8sS0FBSyxFQUFFLG1CQUFtQixFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFDM0UsT0FBTyxFQUFFLGVBQWUsRUFBK0IsS0FBSyxvQkFBb0IsRUFBRSxNQUFNLDRCQUE0QixDQUFDO0FBRXJILE9BQU8sS0FBSyxFQUFFLFlBQVksRUFBRSxtQkFBbUIsRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBQ3JGLE9BQU8sS0FBSyxFQUFFLG1CQUFtQixFQUFFLE1BQU0sNEJBQTRCLENBQUM7QUFDdEUsT0FBTyxLQUFLLEVBQUUsc0JBQXNCLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUU1RSxLQUFLLGlCQUFpQixHQUFHLHdCQUF3QixHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxDQUFDO0FBQzlFLEtBQUssZ0JBQWdCLEdBQUcsSUFBSSxDQUFDLGVBQWUsRUFBRSxlQUFlLENBQUMsR0FBRyxJQUFJLENBQUMsV0FBVyxFQUFFLFdBQVcsR0FBRyxlQUFlLENBQUMsQ0FBQztBQUVsSDs7OztHQUlHO0FBQ0gscUJBQWEsVUFBVyxZQUFXLG1CQUFtQixFQUFFLGFBQWEsRUFBRSxTQUFTO0lBYzVFLFNBQVMsQ0FBQyxRQUFRLENBQUMsTUFBTSxFQUFFLGtCQUFrQjtJQUM3QyxTQUFTLENBQUMsUUFBUSxDQUFDLGdCQUFnQixFQUFFLHNCQUFzQjtJQUMzRCxTQUFTLENBQUMsUUFBUSxDQUFDLGFBQWEsRUFBRSxhQUFhLEdBQUcsT0FBTyxDQUFDLE9BQU8sQ0FBQztJQUNsRSxTQUFTLENBQUMsUUFBUSxDQUFDLG1CQUFtQixFQUFFLG1CQUFtQjtJQUMzRCxTQUFTLENBQUMsUUFBUSxDQUFDLGtCQUFrQixFQUFFLGtCQUFrQjtJQUN6RCxTQUFTLENBQUMsUUFBUSxDQUFDLFVBQVUsRUFBRSxzQkFBc0I7SUFDckQsU0FBUyxDQUFDLFFBQVEsQ0FBQyxTQUFTLEVBQUU7UUFBRSxhQUFhLElBQUksV0FBVyxDQUFBO0tBQUUsR0FBRyxPQUFPLENBQUMsT0FBTyxDQUFDO0lBQ2pGLFNBQVMsQ0FBQyxRQUFRLENBQUMsYUFBYSxFQUFFLFlBQVk7SUFDOUMsU0FBUyxDQUFDLFFBQVEsQ0FBQyxjQUFjLEVBQUUsY0FBYztJQUNqRCxTQUFTLENBQUMsUUFBUSxDQUFDLFNBQVMsRUFBRSxTQUFTO0lBRXZDLFNBQVMsQ0FBQyxRQUFRLENBQUMsZUFBZSxFQUFFLGVBQWU7SUFDbkQsT0FBTyxDQUFDLE9BQU8sQ0FBQztJQUNoQixPQUFPLENBQUMsUUFBUSxDQUFDLFlBQVk7SUExQi9CLE9BQU8sQ0FBQyxHQUFHLENBQStCO0lBRTFDLE9BQU8sQ0FBQyxJQUFJLENBQTJDO0lBQ3ZELE9BQU8sQ0FBQyxNQUFNLENBQW9CO0lBQ2xDLE9BQU8sQ0FBQyxVQUFVLENBQXVCO0lBQ3pDLE9BQU8sQ0FBQyxjQUFjLENBQTJCO0lBQ2pELE9BQU8sQ0FBQyxtQkFBbUIsQ0FBK0I7SUFFMUQsU0FBZ0IsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUUvQixTQUFTLENBQUMsU0FBUyxFQUFFLG1CQUFtQixHQUFHLFNBQVMsQ0FBQztJQUVyRCxZQUNxQixNQUFNLEVBQUUsa0JBQWtCLEVBQzFCLGdCQUFnQixFQUFFLHNCQUFzQixFQUN4QyxhQUFhLEVBQUUsYUFBYSxHQUFHLE9BQU8sQ0FBQyxPQUFPLENBQUMsRUFDL0MsbUJBQW1CLEVBQUUsbUJBQW1CLEVBQ3hDLGtCQUFrQixFQUFFLGtCQUFrQixFQUN0QyxVQUFVLEVBQUUsc0JBQXNCLEVBQ2xDLFNBQVMsRUFBRTtRQUFFLGFBQWEsSUFBSSxXQUFXLENBQUE7S0FBRSxHQUFHLE9BQU8sQ0FBQyxPQUFPLENBQUMsRUFDOUQsYUFBYSxFQUFFLFlBQVksRUFDM0IsY0FBYyxFQUFFLGNBQWMsRUFDOUIsU0FBUyxFQUFFLFNBQVMsRUFDdkMsTUFBTSxHQUFFLE9BQU8sQ0FBQyxpQkFBaUIsQ0FBTSxFQUNwQixlQUFlLEdBQUUsZUFBc0MsRUFDbEUsT0FBTyxDQUFDLHFCQUFTLEVBQ1IsWUFBWSxHQUFFLFlBQWlDLEVBdUJqRTtJQUVNLFdBQVcsdURBRWpCO0lBRU0sTUFBTTs7eUJBRVo7SUFFRCw4RUFBOEU7SUFDdkUsVUFBVSxJQUFJLE9BQU8sR0FBRyxTQUFTLENBRXZDO0lBRUQ7Ozs7T0FJRztJQUNHLHVCQUF1QixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQWN4RTtJQUVEOzs7T0FHRztJQUNHLEtBQUssa0JBT1Y7SUFFRDs7O09BR0c7SUFDRyxJQUFJLGtCQVdUO0lBRUQsa0NBQWtDO0lBQ3JCLHVCQUF1QixJQUFJLE9BQU8sQ0FBQyxvQkFBb0IsQ0FBQyxDQUdwRTtJQUVELCtCQUErQjtJQUN4QixTQUFTLGtEQUVmO0lBRUQ7O09BRUc7SUFDVSxVQUFVLENBQUMsV0FBVyxFQUFFLFdBQVcsaUJBRS9DO1lBRWEsa0JBQWtCO1lBMkJsQixrQkFBa0I7WUFLbEIsTUFBTTtJQXlCcEIsVUFBZ0IscUJBQXFCLENBQUMsR0FBRyxFQUFFLGVBQWUsK0JBWXpEO0lBRUQ7O09BRUc7SUFDSSxTQUFTLHVCQUVmO0lBRUQ7O09BRUc7SUFDSSxPQUFPLElBQUksT0FBTyxDQUFDO1FBQUUsSUFBSSxFQUFFLE1BQU0sQ0FBQztRQUFDLE1BQU0sRUFBRSxvQkFBb0IsQ0FBQztRQUFDLFdBQVcsRUFBRSxXQUFXLENBQUE7S0FBRSxFQUFFLENBQUMsQ0FFcEc7SUFFRCxPQUFPLENBQUMsZUFBZTtZQVFULG9CQUFvQjtJQTJCbEMsT0FBTyxDQUFDLHVCQUF1QjtZQVFqQixnQkFBZ0I7SUFxQzlCLE9BQU8sQ0FBQyxjQUFjO1lBS1IsZUFBZTtZQWdCZixTQUFTO1lBZ0JULGNBQWM7WUFXZCx5QkFBeUI7SUFVdkMsc0NBQXNDO0lBQ3RDLFNBQVMsQ0FBQyx1QkFBdUIsQ0FDL0IsSUFBSSxFQUFFLG1CQUFtQixFQUN6QixRQUFRLEVBQUUsSUFBSSxHQUFHLFNBQVMsRUFDMUIsc0JBQXNCLEVBQUUsc0JBQXNCLEVBQzlDLFNBQVMsRUFBRSxtQkFBbUIsRUFDOUIsSUFBSSxHQUFFO1FBQUUsY0FBYyxDQUFDLEVBQUUsT0FBTyxDQUFBO0tBQU8sbUJBZXhDO0lBRUQsc0NBQXNDO0lBQ3RDLFVBQWdCLGVBQWUsa0JBRTlCO0lBRUQsT0FBTyxDQUFDLGNBQWM7Q0FjdkIifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prover-node.d.ts","sourceRoot":"","sources":["../src/prover-node.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAiC,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAK7F,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,
|
|
1
|
+
{"version":3,"file":"prover-node.d.ts","sourceRoot":"","sources":["../src/prover-node.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAiC,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAK7F,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAEjE,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,KAAK,OAAO,EACZ,KAAK,oBAAoB,EACzB,KAAK,sBAAsB,EAE5B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAEnE,OAAO,EAEL,SAAS,EACT,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAGjC,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AAC3E,OAAO,EAAE,eAAe,EAA+B,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAErH,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACrF,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAE5E,KAAK,iBAAiB,GAAG,wBAAwB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAC9E,KAAK,gBAAgB,GAAG,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,WAAW,GAAG,eAAe,CAAC,CAAC;AAElH;;;;GAIG;AACH,qBAAa,UAAW,YAAW,mBAAmB,EAAE,aAAa,EAAE,SAAS;IAc5E,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,kBAAkB;IAC7C,SAAS,CAAC,QAAQ,CAAC,gBAAgB,EAAE,sBAAsB;IAC3D,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IAClE,SAAS,CAAC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB;IAC3D,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,kBAAkB;IACzD,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,sBAAsB;IACrD,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE;QAAE,aAAa,IAAI,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC;IACjF,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,YAAY;IAC9C,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,cAAc;IACjD,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS;IAEvC,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,eAAe;IACnD,OAAO,CAAC,OAAO,CAAC;IAChB,OAAO,CAAC,QAAQ,CAAC,YAAY;IA1B/B,OAAO,CAAC,GAAG,CAA+B;IAE1C,OAAO,CAAC,IAAI,CAA2C;IACvD,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,cAAc,CAA2B;IACjD,OAAO,CAAC,mBAAmB,CAA+B;IAE1D,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,SAAS,CAAC,SAAS,EAAE,mBAAmB,GAAG,SAAS,CAAC;IAErD,YACqB,MAAM,EAAE,kBAAkB,EAC1B,gBAAgB,EAAE,sBAAsB,EACxC,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC,EAC/C,mBAAmB,EAAE,mBAAmB,EACxC,kBAAkB,EAAE,kBAAkB,EACtC,UAAU,EAAE,sBAAsB,EAClC,SAAS,EAAE;QAAE,aAAa,IAAI,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,EAC9D,aAAa,EAAE,YAAY,EAC3B,cAAc,EAAE,cAAc,EAC9B,SAAS,EAAE,SAAS,EACvC,MAAM,GAAE,OAAO,CAAC,iBAAiB,CAAM,EACpB,eAAe,GAAE,eAAsC,EAClE,OAAO,CAAC,qBAAS,EACR,YAAY,GAAE,YAAiC,EAuBjE;IAEM,WAAW,uDAEjB;IAEM,MAAM;;yBAEZ;IAED,8EAA8E;IACvE,UAAU,IAAI,OAAO,GAAG,SAAS,CAEvC;IAED;;;;OAIG;IACG,uBAAuB,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAcxE;IAED;;;OAGG;IACG,KAAK,kBAOV;IAED;;;OAGG;IACG,IAAI,kBAWT;IAED,kCAAkC;IACrB,uBAAuB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAGpE;IAED,+BAA+B;IACxB,SAAS,kDAEf;IAED;;OAEG;IACU,UAAU,CAAC,WAAW,EAAE,WAAW,iBAE/C;YAEa,kBAAkB;YA2BlB,kBAAkB;YAKlB,MAAM;IAyBpB,UAAgB,qBAAqB,CAAC,GAAG,EAAE,eAAe,+BAYzD;IAED;;OAEG;IACI,SAAS,uBAEf;IAED;;OAEG;IACI,OAAO,IAAI,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,oBAAoB,CAAC;QAAC,WAAW,EAAE,WAAW,CAAA;KAAE,EAAE,CAAC,CAEpG;IAED,OAAO,CAAC,eAAe;YAQT,oBAAoB;IA2BlC,OAAO,CAAC,uBAAuB;YAQjB,gBAAgB;IAqC9B,OAAO,CAAC,cAAc;YAKR,eAAe;YAgBf,SAAS;YAgBT,cAAc;YAWd,yBAAyB;IAUvC,sCAAsC;IACtC,SAAS,CAAC,uBAAuB,CAC/B,IAAI,EAAE,mBAAmB,EACzB,QAAQ,EAAE,IAAI,GAAG,SAAS,EAC1B,sBAAsB,EAAE,sBAAsB,EAC9C,SAAS,EAAE,mBAAmB,EAC9B,IAAI,GAAE;QAAE,cAAc,CAAC,EAAE,OAAO,CAAA;KAAO,mBAexC;IAED,sCAAsC;IACtC,UAAgB,eAAe,kBAE9B;IAED,OAAO,CAAC,cAAc;CAcvB"}
|
package/dest/prover-node.js
CHANGED
|
@@ -378,7 +378,7 @@ import { createLogger } from '@aztec/foundation/log';
|
|
|
378
378
|
import { DateProvider } from '@aztec/foundation/timer';
|
|
379
379
|
import { PublicProcessorFactory } from '@aztec/simulator/server';
|
|
380
380
|
import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
381
|
-
import {
|
|
381
|
+
import { tryStop } from '@aztec/stdlib/interfaces/server';
|
|
382
382
|
import { Attributes, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
383
383
|
import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
|
|
384
384
|
import { EpochProvingJob } from './job/epoch-proving-job.js';
|
|
@@ -430,6 +430,7 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
430
430
|
config;
|
|
431
431
|
jobMetrics;
|
|
432
432
|
rewardsMetrics;
|
|
433
|
+
startingProofEpochs;
|
|
433
434
|
tracer;
|
|
434
435
|
publisher;
|
|
435
436
|
constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, epochsMonitor, rollupContract, l1Metrics, config = {}, telemetryClient = getTelemetryClient(), delayer, dateProvider = new DateProvider()){
|
|
@@ -448,10 +449,11 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
448
449
|
this.dateProvider = dateProvider;
|
|
449
450
|
this.log = (_initProto(this), createLogger('prover-node'));
|
|
450
451
|
this.jobs = new Map();
|
|
452
|
+
this.startingProofEpochs = new Set();
|
|
451
453
|
this.config = {
|
|
452
454
|
proverNodePollingIntervalMs: 1_000,
|
|
453
455
|
proverNodeMaxPendingJobs: 100,
|
|
454
|
-
proverNodeMaxParallelBlocksPerEpoch:
|
|
456
|
+
proverNodeMaxParallelBlocksPerEpoch: 0,
|
|
455
457
|
txGatheringIntervalMs: 1_000,
|
|
456
458
|
txGatheringBatchSize: 10,
|
|
457
459
|
txGatheringMaxParallelRequestsPerNode: 100,
|
|
@@ -484,15 +486,7 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
484
486
|
this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
|
|
485
487
|
jobs: Array.from(this.jobs.values()).map((job)=>`${job.getEpochNumber()}:${job.getId()}`)
|
|
486
488
|
});
|
|
487
|
-
|
|
488
|
-
if (activeJobs.length > 0) {
|
|
489
|
-
this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
|
|
490
|
-
activeJobs: activeJobs.map((job)=>job.uuid)
|
|
491
|
-
});
|
|
492
|
-
return true;
|
|
493
|
-
}
|
|
494
|
-
await this.startProof(epochNumber);
|
|
495
|
-
return true;
|
|
489
|
+
return await this.startProofIfNeeded(epochNumber);
|
|
496
490
|
} catch (err) {
|
|
497
491
|
if (err instanceof EmptyEpochError) {
|
|
498
492
|
this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
|
|
@@ -519,10 +513,10 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
519
513
|
*/ async stop() {
|
|
520
514
|
this.log.info('Stopping ProverNode');
|
|
521
515
|
await this.epochsMonitor.stop();
|
|
522
|
-
await this.prover.stop();
|
|
523
|
-
await tryStop(this.publisherFactory);
|
|
524
516
|
this.publisher?.interrupt();
|
|
525
517
|
await Promise.all(Array.from(this.jobs.values()).map((job)=>job.stop()));
|
|
518
|
+
await this.prover.stop();
|
|
519
|
+
await tryStop(this.publisherFactory);
|
|
526
520
|
this.rewardsMetrics.stop();
|
|
527
521
|
this.l1Metrics.stop();
|
|
528
522
|
await this.telemetryClient.stop();
|
|
@@ -538,9 +532,35 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
538
532
|
/**
|
|
539
533
|
* Starts a proving process and returns immediately.
|
|
540
534
|
*/ async startProof(epochNumber) {
|
|
541
|
-
|
|
535
|
+
await this.startProofIfNeeded(epochNumber, {
|
|
542
536
|
skipEpochCheck: true
|
|
543
537
|
});
|
|
538
|
+
}
|
|
539
|
+
async startProofIfNeeded(epochNumber, opts = {}) {
|
|
540
|
+
if (this.startingProofEpochs.has(epochNumber)) {
|
|
541
|
+
this.log.warn(`Not starting proof for ${epochNumber} since a proof is already being started for the epoch`, {
|
|
542
|
+
epochNumber
|
|
543
|
+
});
|
|
544
|
+
return false;
|
|
545
|
+
}
|
|
546
|
+
this.startingProofEpochs.add(epochNumber);
|
|
547
|
+
try {
|
|
548
|
+
const activeJobs = await this.activeJobsCoverEpoch(epochNumber);
|
|
549
|
+
if (activeJobs.length > 0) {
|
|
550
|
+
this.log.warn(`Not starting proof for ${epochNumber} since an active job already covers the epoch`, {
|
|
551
|
+
epochNumber,
|
|
552
|
+
activeJobs
|
|
553
|
+
});
|
|
554
|
+
return true;
|
|
555
|
+
}
|
|
556
|
+
await this.startProofInternal(epochNumber, opts);
|
|
557
|
+
return true;
|
|
558
|
+
} finally{
|
|
559
|
+
this.startingProofEpochs.delete(epochNumber);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
async startProofInternal(epochNumber, opts = {}) {
|
|
563
|
+
const job = await this.createProvingJob(epochNumber, opts);
|
|
544
564
|
void this.runJob(job);
|
|
545
565
|
}
|
|
546
566
|
async runJob(job) {
|
|
@@ -582,15 +602,40 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
582
602
|
/**
|
|
583
603
|
* Returns an array of jobs being processed.
|
|
584
604
|
*/ getJobs() {
|
|
585
|
-
return Promise.resolve(
|
|
605
|
+
return Promise.resolve(this.getJobsInternal());
|
|
606
|
+
}
|
|
607
|
+
getJobsInternal() {
|
|
608
|
+
return Array.from(this.jobs.entries()).map(([uuid, job])=>({
|
|
586
609
|
uuid,
|
|
587
610
|
status: job.getState(),
|
|
588
611
|
epochNumber: job.getEpochNumber()
|
|
589
|
-
}))
|
|
612
|
+
}));
|
|
590
613
|
}
|
|
591
|
-
async
|
|
592
|
-
const
|
|
593
|
-
|
|
614
|
+
async activeJobsCoverEpoch(epochNumber) {
|
|
615
|
+
const checkpoints = await this.l2BlockSource.getCheckpointsData({
|
|
616
|
+
epoch: epochNumber
|
|
617
|
+
});
|
|
618
|
+
if (checkpoints.length === 0) {
|
|
619
|
+
return [];
|
|
620
|
+
}
|
|
621
|
+
const firstCheckpoint = checkpoints.at(0).checkpointNumber;
|
|
622
|
+
const latestCheckpoint = checkpoints.at(-1).checkpointNumber;
|
|
623
|
+
const jobs = [];
|
|
624
|
+
for (const job of this.jobs.values()){
|
|
625
|
+
if (job.getEpochNumber() !== epochNumber) {
|
|
626
|
+
continue;
|
|
627
|
+
}
|
|
628
|
+
const jobCheckpoints = job.getProvingData().checkpoints;
|
|
629
|
+
const checkpointOverlap = jobCheckpoints.at(0).number <= firstCheckpoint && jobCheckpoints.at(-1).number >= latestCheckpoint;
|
|
630
|
+
if (checkpointOverlap && ![
|
|
631
|
+
'failed',
|
|
632
|
+
'stopped',
|
|
633
|
+
'timed-out'
|
|
634
|
+
].includes(job.getState())) {
|
|
635
|
+
jobs.push(job.getId());
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
return jobs;
|
|
594
639
|
}
|
|
595
640
|
checkMaximumPendingJobs() {
|
|
596
641
|
const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
|
|
@@ -606,10 +651,12 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
606
651
|
const fromCheckpoint = epochData.checkpoints[0].number;
|
|
607
652
|
const toCheckpoint = epochData.checkpoints.at(-1).number;
|
|
608
653
|
const fromBlock = epochData.checkpoints[0].blocks[0].number;
|
|
609
|
-
const
|
|
654
|
+
const lastBlock = epochData.checkpoints.at(-1).blocks.at(-1);
|
|
655
|
+
const toBlock = lastBlock.number;
|
|
610
656
|
this.log.verbose(`Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`);
|
|
611
657
|
// Fast forward world state to right before the target block and get a fork
|
|
612
|
-
await
|
|
658
|
+
const lastBlockHash = await lastBlock.header.hash();
|
|
659
|
+
await this.worldState.syncImmediate(toBlock, lastBlockHash);
|
|
613
660
|
// Create a processor factory
|
|
614
661
|
const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.dateProvider, this.telemetryClient, this.log.getBindings());
|
|
615
662
|
// Set deadline for this job to run. It will abort if it takes too long.
|
|
@@ -623,7 +670,14 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
623
670
|
return this.l2BlockSource.getL1Constants();
|
|
624
671
|
}
|
|
625
672
|
async gatherEpochData(epochNumber) {
|
|
626
|
-
const
|
|
673
|
+
const publishedCheckpoints = await this.l2BlockSource.getCheckpoints({
|
|
674
|
+
epoch: epochNumber
|
|
675
|
+
});
|
|
676
|
+
if (publishedCheckpoints.length === 0) {
|
|
677
|
+
throw new EmptyEpochError(epochNumber);
|
|
678
|
+
}
|
|
679
|
+
const checkpoints = publishedCheckpoints.map((p)=>p.checkpoint);
|
|
680
|
+
const attestations = publishedCheckpoints.at(-1)?.attestations ?? [];
|
|
627
681
|
const txArray = await this.gatherTxs(epochNumber, checkpoints);
|
|
628
682
|
const txs = new Map(txArray.map((tx)=>[
|
|
629
683
|
tx.getTxHash().toString(),
|
|
@@ -632,8 +686,6 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
632
686
|
const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
|
|
633
687
|
const [firstBlock] = checkpoints[0].blocks;
|
|
634
688
|
const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
|
|
635
|
-
const [lastPublishedCheckpoint] = await this.l2BlockSource.getCheckpoints(checkpoints.at(-1).number, 1);
|
|
636
|
-
const attestations = lastPublishedCheckpoint?.attestations ?? [];
|
|
637
689
|
return {
|
|
638
690
|
checkpoints,
|
|
639
691
|
txs,
|
|
@@ -643,13 +695,6 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
643
695
|
attestations
|
|
644
696
|
};
|
|
645
697
|
}
|
|
646
|
-
async gatherCheckpoints(epochNumber) {
|
|
647
|
-
const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
|
|
648
|
-
if (checkpoints.length === 0) {
|
|
649
|
-
throw new EmptyEpochError(epochNumber);
|
|
650
|
-
}
|
|
651
|
-
return checkpoints;
|
|
652
|
-
}
|
|
653
698
|
async gatherTxs(epochNumber, checkpoints) {
|
|
654
699
|
const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
|
|
655
700
|
const txProvider = this.p2pClient.getTxProvider();
|
|
@@ -680,12 +725,14 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
680
725
|
return messagesByCheckpoint;
|
|
681
726
|
}
|
|
682
727
|
async gatherPreviousBlockHeader(epochNumber, previousBlockNumber) {
|
|
683
|
-
const
|
|
684
|
-
|
|
728
|
+
const data = await this.l2BlockSource.getBlockData({
|
|
729
|
+
number: BlockNumber(previousBlockNumber)
|
|
730
|
+
});
|
|
731
|
+
if (!data?.header) {
|
|
685
732
|
throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
|
|
686
733
|
}
|
|
687
|
-
this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
|
|
688
|
-
return header;
|
|
734
|
+
this.log.verbose(`Gathered previous block header ${data.header.getBlockNumber()} for epoch ${epochNumber}`);
|
|
735
|
+
return data.header;
|
|
689
736
|
}
|
|
690
737
|
/** Extracted for testing purposes. */ doCreateEpochProvingJob(data, deadline, publicProcessorFactory, publisher, opts = {}) {
|
|
691
738
|
const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config;
|
|
@@ -15,11 +15,11 @@ export declare class ProverPublisherFactory {
|
|
|
15
15
|
telemetry?: TelemetryClient;
|
|
16
16
|
}, bindings?: LoggerBindings | undefined);
|
|
17
17
|
start(): Promise<void>;
|
|
18
|
-
stop(): void
|
|
18
|
+
stop(): Promise<void>;
|
|
19
19
|
/**
|
|
20
20
|
* Creates a new Prover Publisher instance.
|
|
21
21
|
* @returns A new ProverNodePublisher instance.
|
|
22
22
|
*/
|
|
23
23
|
create(): Promise<ProverNodePublisher>;
|
|
24
24
|
}
|
|
25
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
25
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmVyLXB1Ymxpc2hlci1mYWN0b3J5LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvcHJvdmVyLXB1Ymxpc2hlci1mYWN0b3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLGNBQWMsRUFBRSxNQUFNLDJCQUEyQixDQUFDO0FBQ2hFLE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBQzdELE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sbUNBQW1DLENBQUM7QUFDMUUsT0FBTyxLQUFLLEVBQUUsY0FBYyxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDNUQsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUMzRixPQUFPLEtBQUssRUFBRSxlQUFlLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUUvRCxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSw0QkFBNEIsQ0FBQztBQUVqRSxxQkFBYSxzQkFBc0I7SUFFL0IsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsSUFBSTtJQUtaLE9BQU8sQ0FBQyxRQUFRLENBQUM7SUFQbkIsWUFDVSxNQUFNLEVBQUUsb0JBQW9CLEdBQUcscUJBQXFCLEVBQ3BELElBQUksRUFBRTtRQUNaLGNBQWMsRUFBRSxjQUFjLENBQUM7UUFDL0IsZ0JBQWdCLEVBQUUsZ0JBQWdCLENBQUMsU0FBUyxDQUFDLENBQUM7UUFDOUMsU0FBUyxDQUFDLEVBQUUsZUFBZSxDQUFDO0tBQzdCLEVBQ08sUUFBUSxDQUFDLDRCQUFnQixFQUMvQjtJQUVTLEtBQUssa0JBRWpCO0lBRVksSUFBSSxrQkFFaEI7SUFFRDs7O09BR0c7SUFDVSxNQUFNLElBQUksT0FBTyxDQUFDLG1CQUFtQixDQUFDLENBV2xEO0NBQ0YifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prover-publisher-factory.d.ts","sourceRoot":"","sources":["../src/prover-publisher-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ,CAAC;IAPnB,YACU,MAAM,EAAE,oBAAoB,GAAG,qBAAqB,EACpD,IAAI,EAAE;QACZ,cAAc,EAAE,cAAc,CAAC;QAC/B,gBAAgB,EAAE,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC9C,SAAS,CAAC,EAAE,eAAe,CAAC;KAC7B,EACO,QAAQ,CAAC,4BAAgB,EAC/B;IAES,KAAK,kBAEjB;
|
|
1
|
+
{"version":3,"file":"prover-publisher-factory.d.ts","sourceRoot":"","sources":["../src/prover-publisher-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ,CAAC;IAPnB,YACU,MAAM,EAAE,oBAAoB,GAAG,qBAAqB,EACpD,IAAI,EAAE;QACZ,cAAc,EAAE,cAAc,CAAC;QAC/B,gBAAgB,EAAE,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC9C,SAAS,CAAC,EAAE,eAAe,CAAC;KAC7B,EACO,QAAQ,CAAC,4BAAgB,EAC/B;IAES,KAAK,kBAEjB;IAEY,IAAI,kBAEhB;IAED;;;OAGG;IACU,MAAM,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAWlD;CACF"}
|
|
@@ -9,10 +9,10 @@ export class ProverPublisherFactory {
|
|
|
9
9
|
this.bindings = bindings;
|
|
10
10
|
}
|
|
11
11
|
async start() {
|
|
12
|
-
await this.deps.publisherManager.
|
|
12
|
+
await this.deps.publisherManager.start();
|
|
13
13
|
}
|
|
14
|
-
stop() {
|
|
15
|
-
this.deps.publisherManager.
|
|
14
|
+
async stop() {
|
|
15
|
+
await this.deps.publisherManager.stop();
|
|
16
16
|
}
|
|
17
17
|
/**
|
|
18
18
|
* Creates a new Prover Publisher instance.
|