@aztec/prover-node 0.0.0-test.1 → 0.0.1-commit.b655e406
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/actions/download-epoch-proving-job.d.ts +18 -0
- package/dest/actions/download-epoch-proving-job.d.ts.map +1 -0
- package/dest/actions/download-epoch-proving-job.js +37 -0
- package/dest/actions/index.d.ts +3 -0
- package/dest/actions/index.d.ts.map +1 -0
- package/dest/actions/index.js +2 -0
- package/dest/actions/rerun-epoch-proving-job.d.ts +11 -0
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -0
- package/dest/actions/rerun-epoch-proving-job.js +40 -0
- package/dest/actions/upload-epoch-proof-failure.d.ts +15 -0
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -0
- package/dest/actions/upload-epoch-proof-failure.js +78 -0
- package/dest/bin/run-failed-epoch.d.ts +2 -0
- package/dest/bin/run-failed-epoch.d.ts.map +1 -0
- package/dest/bin/run-failed-epoch.js +67 -0
- package/dest/config.d.ts +12 -9
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +81 -14
- package/dest/factory.d.ts +12 -8
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +95 -31
- package/dest/index.d.ts +1 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -1
- package/dest/job/epoch-proving-job-data.d.ts +16 -0
- package/dest/job/epoch-proving-job-data.d.ts.map +1 -0
- package/dest/job/epoch-proving-job-data.js +52 -0
- package/dest/job/epoch-proving-job.d.ts +30 -15
- package/dest/job/epoch-proving-job.d.ts.map +1 -1
- package/dest/job/epoch-proving-job.js +149 -50
- package/dest/metrics.d.ts +28 -4
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +141 -35
- package/dest/monitors/epoch-monitor.d.ts +3 -1
- package/dest/monitors/epoch-monitor.d.ts.map +1 -1
- package/dest/monitors/epoch-monitor.js +15 -2
- package/dest/prover-node-publisher.d.ts +7 -10
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +59 -60
- package/dest/prover-node.d.ts +43 -39
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +171 -100
- package/dest/prover-publisher-factory.d.ts +21 -0
- package/dest/prover-publisher-factory.d.ts.map +1 -0
- package/dest/prover-publisher-factory.js +26 -0
- package/dest/test/index.d.ts +4 -2
- package/dest/test/index.d.ts.map +1 -1
- package/dest/test/index.js +1 -3
- package/package.json +36 -31
- package/src/actions/download-epoch-proving-job.ts +44 -0
- package/src/actions/index.ts +2 -0
- package/src/actions/rerun-epoch-proving-job.ts +61 -0
- package/src/actions/upload-epoch-proof-failure.ts +88 -0
- package/src/bin/run-failed-epoch.ts +77 -0
- package/src/config.ts +108 -24
- package/src/factory.ts +161 -43
- package/src/index.ts +1 -1
- package/src/job/epoch-proving-job-data.ts +76 -0
- package/src/job/epoch-proving-job.ts +215 -50
- package/src/metrics.ts +135 -37
- package/src/monitors/epoch-monitor.ts +16 -5
- package/src/prover-node-publisher.ts +93 -86
- package/src/prover-node.ts +203 -126
- package/src/prover-publisher-factory.ts +37 -0
- package/src/test/index.ts +7 -4
- package/dest/http.d.ts +0 -8
- package/dest/http.d.ts.map +0 -1
- package/dest/http.js +0 -9
- package/dest/prover-coordination/config.d.ts +0 -7
- package/dest/prover-coordination/config.d.ts.map +0 -1
- package/dest/prover-coordination/config.js +0 -11
- package/dest/prover-coordination/factory.d.ts +0 -22
- package/dest/prover-coordination/factory.d.ts.map +0 -1
- package/dest/prover-coordination/factory.js +0 -42
- package/dest/prover-coordination/index.d.ts +0 -3
- package/dest/prover-coordination/index.d.ts.map +0 -1
- package/dest/prover-coordination/index.js +0 -2
- package/src/http.ts +0 -13
- package/src/prover-coordination/config.ts +0 -17
- package/src/prover-coordination/factory.ts +0 -72
- package/src/prover-coordination/index.ts +0 -2
package/src/prover-node.ts
CHANGED
|
@@ -1,28 +1,32 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Archiver } from '@aztec/archiver';
|
|
2
|
+
import type { RollupContract } from '@aztec/ethereum';
|
|
3
|
+
import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
|
|
2
4
|
import { memoize } from '@aztec/foundation/decorators';
|
|
5
|
+
import type { Fr } from '@aztec/foundation/fields';
|
|
3
6
|
import { createLogger } from '@aztec/foundation/log';
|
|
4
|
-
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
5
7
|
import { DateProvider } from '@aztec/foundation/timer';
|
|
6
|
-
import type {
|
|
7
|
-
import type {
|
|
8
|
+
import type { DataStoreConfig } from '@aztec/kv-store/config';
|
|
9
|
+
import type { P2PClient } from '@aztec/p2p';
|
|
8
10
|
import { PublicProcessorFactory } from '@aztec/simulator/server';
|
|
9
11
|
import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
|
|
12
|
+
import type { ChainConfig } from '@aztec/stdlib/config';
|
|
10
13
|
import type { ContractDataSource } from '@aztec/stdlib/contract';
|
|
11
|
-
import {
|
|
14
|
+
import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
12
15
|
import {
|
|
13
16
|
type EpochProverManager,
|
|
14
17
|
EpochProvingJobTerminalState,
|
|
15
|
-
type ProverCoordination,
|
|
16
18
|
type ProverNodeApi,
|
|
17
19
|
type Service,
|
|
20
|
+
type WorldStateSyncStatus,
|
|
18
21
|
type WorldStateSynchronizer,
|
|
19
22
|
tryStop,
|
|
20
23
|
} from '@aztec/stdlib/interfaces/server';
|
|
21
24
|
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
22
25
|
import type { P2PClientType } from '@aztec/stdlib/p2p';
|
|
23
|
-
import type { Tx
|
|
26
|
+
import type { Tx } from '@aztec/stdlib/tx';
|
|
24
27
|
import {
|
|
25
28
|
Attributes,
|
|
29
|
+
L1Metrics,
|
|
26
30
|
type TelemetryClient,
|
|
27
31
|
type Traceable,
|
|
28
32
|
type Tracer,
|
|
@@ -30,66 +34,71 @@ import {
|
|
|
30
34
|
trackSpan,
|
|
31
35
|
} from '@aztec/telemetry-client';
|
|
32
36
|
|
|
37
|
+
import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
|
|
38
|
+
import type { SpecificProverNodeConfig } from './config.js';
|
|
39
|
+
import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
|
|
33
40
|
import { EpochProvingJob, type EpochProvingJobState } from './job/epoch-proving-job.js';
|
|
34
|
-
import {
|
|
41
|
+
import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
|
|
35
42
|
import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js';
|
|
36
43
|
import type { ProverNodePublisher } from './prover-node-publisher.js';
|
|
44
|
+
import type { ProverPublisherFactory } from './prover-publisher-factory.js';
|
|
37
45
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
maxPendingJobs: number;
|
|
41
|
-
maxParallelBlocksPerEpoch: number;
|
|
42
|
-
txGatheringTimeoutMs: number;
|
|
43
|
-
txGatheringIntervalMs: number;
|
|
44
|
-
txGatheringMaxParallelRequests: number;
|
|
45
|
-
};
|
|
46
|
+
type ProverNodeOptions = SpecificProverNodeConfig & Partial<DataStoreOptions>;
|
|
47
|
+
type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'>;
|
|
46
48
|
|
|
47
49
|
/**
|
|
48
|
-
* An Aztec Prover Node is a standalone process that monitors the
|
|
49
|
-
*
|
|
50
|
-
* from a tx source in the p2p network or an external node, re-executes their public functions, creates a rollup
|
|
50
|
+
* An Aztec Prover Node is a standalone process that monitors the unfinalized chain on L1 for unproven epochs,
|
|
51
|
+
* fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
|
|
51
52
|
* proof for the epoch, and submits it to L1.
|
|
52
53
|
*/
|
|
53
54
|
export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable {
|
|
54
55
|
private log = createLogger('prover-node');
|
|
55
56
|
private dateProvider = new DateProvider();
|
|
56
57
|
|
|
57
|
-
private latestEpochWeAreProving: bigint | undefined;
|
|
58
58
|
private jobs: Map<string, EpochProvingJob> = new Map();
|
|
59
|
-
private
|
|
60
|
-
private
|
|
61
|
-
private
|
|
62
|
-
|
|
63
|
-
private txFetcher: RunningPromise;
|
|
64
|
-
private lastBlockNumber: number | undefined;
|
|
59
|
+
private config: ProverNodeOptions;
|
|
60
|
+
private jobMetrics: ProverNodeJobMetrics;
|
|
61
|
+
private rewardsMetrics: ProverNodeRewardsMetrics;
|
|
65
62
|
|
|
66
63
|
public readonly tracer: Tracer;
|
|
67
64
|
|
|
65
|
+
protected publisher: ProverNodePublisher | undefined;
|
|
66
|
+
|
|
68
67
|
constructor(
|
|
69
68
|
protected readonly prover: EpochProverManager,
|
|
70
|
-
protected readonly
|
|
71
|
-
protected readonly l2BlockSource: L2BlockSource &
|
|
69
|
+
protected readonly publisherFactory: ProverPublisherFactory,
|
|
70
|
+
protected readonly l2BlockSource: L2BlockSource & Partial<Service>,
|
|
72
71
|
protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
|
|
73
72
|
protected readonly contractDataSource: ContractDataSource,
|
|
74
73
|
protected readonly worldState: WorldStateSynchronizer,
|
|
75
|
-
protected readonly
|
|
74
|
+
protected readonly p2pClient: Pick<P2PClient<P2PClientType.Prover>, 'getTxProvider'> & Partial<Service>,
|
|
76
75
|
protected readonly epochsMonitor: EpochMonitor,
|
|
77
|
-
|
|
76
|
+
protected readonly rollupContract: RollupContract,
|
|
77
|
+
protected readonly l1Metrics: L1Metrics,
|
|
78
|
+
config: Partial<ProverNodeOptions> = {},
|
|
78
79
|
protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
|
|
79
80
|
) {
|
|
80
|
-
this.
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
txGatheringTimeoutMs: 60_000,
|
|
81
|
+
this.config = {
|
|
82
|
+
proverNodePollingIntervalMs: 1_000,
|
|
83
|
+
proverNodeMaxPendingJobs: 100,
|
|
84
|
+
proverNodeMaxParallelBlocksPerEpoch: 32,
|
|
85
85
|
txGatheringIntervalMs: 1_000,
|
|
86
|
-
|
|
87
|
-
|
|
86
|
+
txGatheringBatchSize: 10,
|
|
87
|
+
txGatheringMaxParallelRequestsPerNode: 100,
|
|
88
|
+
txGatheringTimeoutMs: 120_000,
|
|
89
|
+
proverNodeFailedEpochStore: undefined,
|
|
90
|
+
proverNodeEpochProvingDelayMs: undefined,
|
|
91
|
+
...compact(config),
|
|
88
92
|
};
|
|
89
93
|
|
|
90
|
-
this.
|
|
94
|
+
this.validateConfig();
|
|
95
|
+
|
|
96
|
+
const meter = telemetryClient.getMeter('ProverNode');
|
|
91
97
|
this.tracer = telemetryClient.getTracer('ProverNode');
|
|
92
|
-
|
|
98
|
+
|
|
99
|
+
this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
|
|
100
|
+
|
|
101
|
+
this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
|
|
93
102
|
}
|
|
94
103
|
|
|
95
104
|
public getProverId() {
|
|
@@ -97,44 +106,49 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
97
106
|
}
|
|
98
107
|
|
|
99
108
|
public getP2P() {
|
|
100
|
-
|
|
101
|
-
if (typeof asP2PClient.isP2PClient === 'function' && asP2PClient.isP2PClient()) {
|
|
102
|
-
return asP2PClient;
|
|
103
|
-
}
|
|
104
|
-
return undefined;
|
|
109
|
+
return this.p2pClient;
|
|
105
110
|
}
|
|
106
111
|
|
|
107
112
|
/**
|
|
108
113
|
* Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
|
|
109
114
|
* @param epochNumber - The epoch number that was just completed.
|
|
115
|
+
* @returns false if there is an error, true otherwise
|
|
110
116
|
*/
|
|
111
|
-
async handleEpochReadyToProve(epochNumber: bigint): Promise<
|
|
117
|
+
async handleEpochReadyToProve(epochNumber: bigint): Promise<boolean> {
|
|
112
118
|
try {
|
|
113
|
-
this.log.debug(
|
|
119
|
+
this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
|
|
120
|
+
jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
|
|
121
|
+
});
|
|
114
122
|
const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
|
|
115
123
|
if (activeJobs.length > 0) {
|
|
116
|
-
this.log.
|
|
117
|
-
|
|
124
|
+
this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
|
|
125
|
+
activeJobs: activeJobs.map(job => job.uuid),
|
|
126
|
+
});
|
|
127
|
+
return true;
|
|
118
128
|
}
|
|
119
|
-
// TODO: we probably want to skip starting a proof if we are too far into the current epoch
|
|
120
129
|
await this.startProof(epochNumber);
|
|
130
|
+
return true;
|
|
121
131
|
} catch (err) {
|
|
122
132
|
if (err instanceof EmptyEpochError) {
|
|
123
133
|
this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
|
|
124
134
|
} else {
|
|
125
135
|
this.log.error(`Error handling epoch completed`, err);
|
|
126
136
|
}
|
|
137
|
+
return false;
|
|
127
138
|
}
|
|
128
139
|
}
|
|
129
140
|
|
|
130
141
|
/**
|
|
131
|
-
* Starts the prover node so it periodically checks for unproven epochs in the
|
|
142
|
+
* Starts the prover node so it periodically checks for unproven epochs in the unfinalized chain from L1 and
|
|
132
143
|
* starts proving jobs for them.
|
|
133
144
|
*/
|
|
134
|
-
start() {
|
|
135
|
-
this.txFetcher.start();
|
|
145
|
+
async start() {
|
|
136
146
|
this.epochsMonitor.start(this);
|
|
137
|
-
this.
|
|
147
|
+
await this.publisherFactory.start();
|
|
148
|
+
this.publisher = await this.publisherFactory.create();
|
|
149
|
+
await this.rewardsMetrics.start();
|
|
150
|
+
this.l1Metrics.start();
|
|
151
|
+
this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
|
|
138
152
|
}
|
|
139
153
|
|
|
140
154
|
/**
|
|
@@ -142,32 +156,76 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
142
156
|
*/
|
|
143
157
|
async stop() {
|
|
144
158
|
this.log.info('Stopping ProverNode');
|
|
145
|
-
await this.txFetcher.stop();
|
|
146
159
|
await this.epochsMonitor.stop();
|
|
147
160
|
await this.prover.stop();
|
|
161
|
+
await tryStop(this.p2pClient);
|
|
148
162
|
await tryStop(this.l2BlockSource);
|
|
149
|
-
this.
|
|
163
|
+
await tryStop(this.publisherFactory);
|
|
164
|
+
this.publisher?.interrupt();
|
|
150
165
|
await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
|
|
151
166
|
await this.worldState.stop();
|
|
152
|
-
|
|
167
|
+
this.rewardsMetrics.stop();
|
|
168
|
+
this.l1Metrics.stop();
|
|
153
169
|
await this.telemetryClient.stop();
|
|
154
170
|
this.log.info('Stopped ProverNode');
|
|
155
171
|
}
|
|
156
172
|
|
|
157
|
-
/**
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
173
|
+
/** Returns world state status. */
|
|
174
|
+
public async getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
|
|
175
|
+
const { syncSummary } = await this.worldState.status();
|
|
176
|
+
return syncSummary;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Returns archiver status. */
|
|
180
|
+
public getL2Tips() {
|
|
181
|
+
return this.l2BlockSource.getL2Tips();
|
|
163
182
|
}
|
|
164
183
|
|
|
165
184
|
/**
|
|
166
185
|
* Starts a proving process and returns immediately.
|
|
167
186
|
*/
|
|
168
187
|
public async startProof(epochNumber: number | bigint) {
|
|
169
|
-
const job = await this.createProvingJob(BigInt(epochNumber));
|
|
170
|
-
void
|
|
188
|
+
const job = await this.createProvingJob(BigInt(epochNumber), { skipEpochCheck: true });
|
|
189
|
+
void this.runJob(job);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
private async runJob(job: EpochProvingJob) {
|
|
193
|
+
const epochNumber = job.getEpochNumber();
|
|
194
|
+
const ctx = { id: job.getId(), epochNumber, state: undefined as EpochProvingJobState | undefined };
|
|
195
|
+
|
|
196
|
+
try {
|
|
197
|
+
await job.run();
|
|
198
|
+
const state = job.getState();
|
|
199
|
+
ctx.state = state;
|
|
200
|
+
|
|
201
|
+
if (state === 'reorg') {
|
|
202
|
+
this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
|
|
203
|
+
await this.createProvingJob(epochNumber);
|
|
204
|
+
} else if (state === 'failed') {
|
|
205
|
+
this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
|
|
206
|
+
await this.tryUploadEpochFailure(job);
|
|
207
|
+
} else {
|
|
208
|
+
this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
|
|
209
|
+
}
|
|
210
|
+
} catch (err) {
|
|
211
|
+
this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
|
|
212
|
+
} finally {
|
|
213
|
+
this.jobs.delete(job.getId());
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
protected async tryUploadEpochFailure(job: EpochProvingJob) {
|
|
218
|
+
if (this.config.proverNodeFailedEpochStore) {
|
|
219
|
+
return await uploadEpochProofFailure(
|
|
220
|
+
this.config.proverNodeFailedEpochStore,
|
|
221
|
+
job.getId(),
|
|
222
|
+
job.getProvingData(),
|
|
223
|
+
this.l2BlockSource as Archiver,
|
|
224
|
+
this.worldState,
|
|
225
|
+
assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
|
|
226
|
+
this.log,
|
|
227
|
+
);
|
|
228
|
+
}
|
|
171
229
|
}
|
|
172
230
|
|
|
173
231
|
/**
|
|
@@ -199,43 +257,40 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
199
257
|
}
|
|
200
258
|
|
|
201
259
|
private checkMaximumPendingJobs() {
|
|
202
|
-
const { maxPendingJobs } = this.
|
|
203
|
-
|
|
260
|
+
const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
|
|
261
|
+
if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
|
|
262
|
+
throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
|
|
263
|
+
}
|
|
204
264
|
}
|
|
205
265
|
|
|
206
266
|
@trackSpan('ProverNode.createProvingJob', epochNumber => ({ [Attributes.EPOCH_NUMBER]: Number(epochNumber) }))
|
|
207
|
-
private async createProvingJob(epochNumber: bigint) {
|
|
208
|
-
|
|
209
|
-
throw new Error(`Maximum pending proving jobs ${this.options.maxPendingJobs} reached. Cannot create new job.`);
|
|
210
|
-
}
|
|
267
|
+
private async createProvingJob(epochNumber: bigint, opts: { skipEpochCheck?: boolean } = {}) {
|
|
268
|
+
this.checkMaximumPendingJobs();
|
|
211
269
|
|
|
212
|
-
|
|
213
|
-
const cachedEpochData = this.cachedEpochData?.epochNumber === epochNumber ? this.cachedEpochData : undefined;
|
|
214
|
-
const { blocks, txs } = cachedEpochData ?? (await this.gatherEpochData(epochNumber));
|
|
270
|
+
this.publisher = await this.publisherFactory.create();
|
|
215
271
|
|
|
216
|
-
|
|
217
|
-
const
|
|
272
|
+
// Gather all data for this epoch
|
|
273
|
+
const epochData = await this.gatherEpochData(epochNumber);
|
|
218
274
|
|
|
219
|
-
|
|
275
|
+
const fromBlock = epochData.blocks[0].number;
|
|
276
|
+
const toBlock = epochData.blocks.at(-1)!.number;
|
|
220
277
|
this.log.verbose(`Creating proving job for epoch ${epochNumber} for block range ${fromBlock} to ${toBlock}`);
|
|
278
|
+
|
|
279
|
+
// Fast forward world state to right before the target block and get a fork
|
|
221
280
|
await this.worldState.syncImmediate(toBlock);
|
|
222
281
|
|
|
223
|
-
// Create a processor
|
|
282
|
+
// Create a processor factory
|
|
224
283
|
const publicProcessorFactory = new PublicProcessorFactory(
|
|
225
284
|
this.contractDataSource,
|
|
226
285
|
this.dateProvider,
|
|
227
286
|
this.telemetryClient,
|
|
228
287
|
);
|
|
229
288
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
};
|
|
289
|
+
// Set deadline for this job to run. It will abort if it takes too long.
|
|
290
|
+
const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
|
|
291
|
+
const deadline = new Date(Number(deadlineTs) * 1000);
|
|
234
292
|
|
|
235
|
-
const
|
|
236
|
-
const deadline = new Date(Number(endTimestamp) * 1000);
|
|
237
|
-
|
|
238
|
-
const job = this.doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory, cleanUp);
|
|
293
|
+
const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
|
|
239
294
|
this.jobs.set(job.getId(), job);
|
|
240
295
|
return job;
|
|
241
296
|
}
|
|
@@ -245,29 +300,17 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
245
300
|
return this.l2BlockSource.getL1Constants();
|
|
246
301
|
}
|
|
247
302
|
|
|
248
|
-
/** Monitors for new blocks and requests their txs from the p2p layer to ensure they are available for proving. */
|
|
249
|
-
@trackSpan('ProverNode.checkForTxs')
|
|
250
|
-
private async checkForTxs() {
|
|
251
|
-
const blockNumber = await this.l2BlockSource.getBlockNumber();
|
|
252
|
-
if (this.lastBlockNumber === undefined || blockNumber > this.lastBlockNumber) {
|
|
253
|
-
const block = await this.l2BlockSource.getBlock(blockNumber);
|
|
254
|
-
if (!block) {
|
|
255
|
-
return;
|
|
256
|
-
}
|
|
257
|
-
const txHashes = block.body.txEffects.map(tx => tx.txHash);
|
|
258
|
-
this.log.verbose(`Fetching ${txHashes.length} tx hashes for block number ${blockNumber} from coordination`);
|
|
259
|
-
await this.coordination.getTxsByHash(txHashes); // This stores the txs in the tx pool, no need to persist them here
|
|
260
|
-
this.lastBlockNumber = blockNumber;
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
|
|
264
303
|
@trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: Number(epochNumber) }))
|
|
265
|
-
private async gatherEpochData(epochNumber: bigint) {
|
|
266
|
-
// Gather blocks for this epoch and their txs
|
|
304
|
+
private async gatherEpochData(epochNumber: bigint): Promise<EpochProvingJobData> {
|
|
267
305
|
const blocks = await this.gatherBlocks(epochNumber);
|
|
268
|
-
const
|
|
269
|
-
|
|
270
|
-
|
|
306
|
+
const txArray = await this.gatherTxs(epochNumber, blocks);
|
|
307
|
+
const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
|
|
308
|
+
const l1ToL2Messages = await this.gatherMessages(epochNumber, blocks);
|
|
309
|
+
const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, blocks[0]);
|
|
310
|
+
const [lastBlock] = await this.l2BlockSource.getPublishedBlocks(blocks.at(-1)!.number, 1);
|
|
311
|
+
const attestations = lastBlock?.attestations ?? [];
|
|
312
|
+
|
|
313
|
+
return { blocks, txs, l1ToL2Messages, epochNumber, previousBlockHeader, attestations };
|
|
271
314
|
}
|
|
272
315
|
|
|
273
316
|
private async gatherBlocks(epochNumber: bigint) {
|
|
@@ -279,45 +322,64 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
279
322
|
}
|
|
280
323
|
|
|
281
324
|
private async gatherTxs(epochNumber: bigint, blocks: L2Block[]) {
|
|
282
|
-
const
|
|
283
|
-
const
|
|
325
|
+
const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
|
|
326
|
+
const txProvider = this.p2pClient.getTxProvider();
|
|
327
|
+
const txsByBlock = await Promise.all(blocks.map(block => txProvider.getTxsForBlock(block, { deadline })));
|
|
328
|
+
const txs = txsByBlock.map(({ txs }) => txs).flat();
|
|
329
|
+
const missingTxs = txsByBlock.map(({ missingTxs }) => missingTxs).flat();
|
|
284
330
|
|
|
285
|
-
if (
|
|
331
|
+
if (missingTxs.length === 0) {
|
|
286
332
|
this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, { epochNumber });
|
|
287
333
|
return txs;
|
|
288
334
|
}
|
|
289
335
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
.filter(txHashToFind => !txHashesFound.some(txHashFound => txHashToFind.equals(txHashFound)))
|
|
293
|
-
.join(', ');
|
|
336
|
+
throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxs.map(hash => hash.toString()).join(', ')}`);
|
|
337
|
+
}
|
|
294
338
|
|
|
295
|
-
|
|
339
|
+
private async gatherMessages(epochNumber: bigint, blocks: L2Block[]) {
|
|
340
|
+
const messages = await Promise.all(blocks.map(b => this.l1ToL2MessageSource.getL1ToL2Messages(b.number)));
|
|
341
|
+
const messageCount = sum(messages.map(m => m.length));
|
|
342
|
+
this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, { epochNumber });
|
|
343
|
+
const messagesByBlock: Record<number, Fr[]> = {};
|
|
344
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
345
|
+
messagesByBlock[blocks[i].number] = messages[i];
|
|
346
|
+
}
|
|
347
|
+
return messagesByBlock;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
private async gatherPreviousBlockHeader(epochNumber: bigint, initialBlock: L2Block) {
|
|
351
|
+
const previousBlockNumber = initialBlock.number - 1;
|
|
352
|
+
const header = await (previousBlockNumber === 0
|
|
353
|
+
? this.worldState.getCommitted().getInitialHeader()
|
|
354
|
+
: this.l2BlockSource.getBlockHeader(previousBlockNumber));
|
|
355
|
+
|
|
356
|
+
if (!header) {
|
|
357
|
+
throw new Error(`Previous block header ${initialBlock.number} not found for proving epoch ${epochNumber}`);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
|
|
361
|
+
return header;
|
|
296
362
|
}
|
|
297
363
|
|
|
298
364
|
/** Extracted for testing purposes. */
|
|
299
365
|
protected doCreateEpochProvingJob(
|
|
300
|
-
|
|
366
|
+
data: EpochProvingJobData,
|
|
301
367
|
deadline: Date | undefined,
|
|
302
|
-
blocks: L2Block[],
|
|
303
|
-
txs: Tx[],
|
|
304
368
|
publicProcessorFactory: PublicProcessorFactory,
|
|
305
|
-
|
|
369
|
+
publisher: ProverNodePublisher,
|
|
370
|
+
opts: { skipEpochCheck?: boolean } = {},
|
|
306
371
|
) {
|
|
372
|
+
const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config;
|
|
307
373
|
return new EpochProvingJob(
|
|
374
|
+
data,
|
|
308
375
|
this.worldState,
|
|
309
|
-
epochNumber,
|
|
310
|
-
blocks,
|
|
311
|
-
txs,
|
|
312
376
|
this.prover.createEpochProver(),
|
|
313
377
|
publicProcessorFactory,
|
|
314
|
-
|
|
378
|
+
publisher,
|
|
315
379
|
this.l2BlockSource,
|
|
316
|
-
this.
|
|
317
|
-
this.metrics,
|
|
380
|
+
this.jobMetrics,
|
|
318
381
|
deadline,
|
|
319
|
-
{ parallelBlockLimit:
|
|
320
|
-
cleanUp,
|
|
382
|
+
{ parallelBlockLimit, skipSubmitProof: proverNodeDisableProofPublish, ...opts },
|
|
321
383
|
);
|
|
322
384
|
}
|
|
323
385
|
|
|
@@ -325,6 +387,21 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
325
387
|
protected async triggerMonitors() {
|
|
326
388
|
await this.epochsMonitor.work();
|
|
327
389
|
}
|
|
390
|
+
|
|
391
|
+
private validateConfig() {
|
|
392
|
+
if (
|
|
393
|
+
this.config.proverNodeFailedEpochStore &&
|
|
394
|
+
(!this.config.dataDirectory || !this.config.l1ChainId || !this.config.rollupVersion)
|
|
395
|
+
) {
|
|
396
|
+
this.log.warn(
|
|
397
|
+
`Invalid prover-node config (missing dataDirectory, l1ChainId, or rollupVersion)`,
|
|
398
|
+
pick(this.config, 'proverNodeFailedEpochStore', 'dataDirectory', 'l1ChainId', 'rollupVersion'),
|
|
399
|
+
);
|
|
400
|
+
throw new Error(
|
|
401
|
+
'All of dataDirectory, l1ChainId, and rollupVersion are required if proverNodeFailedEpochStore is set.',
|
|
402
|
+
);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
328
405
|
}
|
|
329
406
|
|
|
330
407
|
class EmptyEpochError extends Error {
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { L1TxUtils, PublisherManager, RollupContract } from '@aztec/ethereum';
|
|
2
|
+
import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
|
|
3
|
+
import type { TelemetryClient } from '@aztec/telemetry-client';
|
|
4
|
+
|
|
5
|
+
import { ProverNodePublisher } from './prover-node-publisher.js';
|
|
6
|
+
|
|
7
|
+
export class ProverPublisherFactory {
|
|
8
|
+
constructor(
|
|
9
|
+
private config: TxSenderConfig & PublisherConfig,
|
|
10
|
+
private deps: {
|
|
11
|
+
rollupContract: RollupContract;
|
|
12
|
+
publisherManager: PublisherManager<L1TxUtils>;
|
|
13
|
+
telemetry?: TelemetryClient;
|
|
14
|
+
},
|
|
15
|
+
) {}
|
|
16
|
+
|
|
17
|
+
public async start() {
|
|
18
|
+
await this.deps.publisherManager.loadState();
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
public stop() {
|
|
22
|
+
this.deps.publisherManager.interrupt();
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Creates a new Prover Publisher instance.
|
|
27
|
+
* @returns A new ProverNodePublisher instance.
|
|
28
|
+
*/
|
|
29
|
+
public async create(): Promise<ProverNodePublisher> {
|
|
30
|
+
const l1Publisher = await this.deps.publisherManager.getAvailablePublisher();
|
|
31
|
+
return new ProverNodePublisher(this.config, {
|
|
32
|
+
rollupContract: this.deps.rollupContract,
|
|
33
|
+
l1TxUtils: l1Publisher,
|
|
34
|
+
telemetry: this.deps.telemetry,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
}
|
package/src/test/index.ts
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import type { EpochProverManager } from '@aztec/stdlib/interfaces/server';
|
|
2
2
|
|
|
3
|
+
import type { EpochProvingJob } from '../job/epoch-proving-job.js';
|
|
3
4
|
import type { ProverNodePublisher } from '../prover-node-publisher.js';
|
|
4
5
|
import { ProverNode } from '../prover-node.js';
|
|
5
6
|
|
|
6
|
-
class
|
|
7
|
-
public
|
|
8
|
-
public
|
|
7
|
+
abstract class TestProverNodeClass extends ProverNode {
|
|
8
|
+
declare public prover: EpochProverManager;
|
|
9
|
+
declare public publisher: ProverNodePublisher;
|
|
10
|
+
|
|
11
|
+
public abstract override tryUploadEpochFailure(job: EpochProvingJob): Promise<string | undefined>;
|
|
9
12
|
}
|
|
10
13
|
|
|
11
|
-
export type TestProverNode =
|
|
14
|
+
export type TestProverNode = TestProverNodeClass;
|
package/dest/http.d.ts
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { ProverNode } from './prover-node.js';
|
|
2
|
-
/**
|
|
3
|
-
* Wrap a ProverNode instance with a JSON RPC HTTP server.
|
|
4
|
-
* @param node - The ProverNode
|
|
5
|
-
* @returns An JSON-RPC HTTP server
|
|
6
|
-
*/
|
|
7
|
-
export declare function createProverNodeRpcServer(node: ProverNode): import("@aztec/foundation/json-rpc/server").SafeJsonRpcServer;
|
|
8
|
-
//# sourceMappingURL=http.d.ts.map
|
package/dest/http.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,UAAU,iEAEzD"}
|
package/dest/http.js
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { ProverNodeApiSchema } from '@aztec/stdlib/interfaces/server';
|
|
2
|
-
import { createTracedJsonRpcServer } from '@aztec/telemetry-client';
|
|
3
|
-
/**
|
|
4
|
-
* Wrap a ProverNode instance with a JSON RPC HTTP server.
|
|
5
|
-
* @param node - The ProverNode
|
|
6
|
-
* @returns An JSON-RPC HTTP server
|
|
7
|
-
*/ export function createProverNodeRpcServer(node) {
|
|
8
|
-
return createTracedJsonRpcServer(node, ProverNodeApiSchema);
|
|
9
|
-
}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import { type ConfigMappingsType } from '@aztec/foundation/config';
|
|
2
|
-
export type ProverCoordinationConfig = {
|
|
3
|
-
proverCoordinationNodeUrl: string | undefined;
|
|
4
|
-
};
|
|
5
|
-
export declare const proverCoordinationConfigMappings: ConfigMappingsType<ProverCoordinationConfig>;
|
|
6
|
-
export declare function getTxProviderConfigFromEnv(): ProverCoordinationConfig;
|
|
7
|
-
//# sourceMappingURL=config.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/prover-coordination/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAAyB,MAAM,0BAA0B,CAAC;AAE1F,MAAM,MAAM,wBAAwB,GAAG;IACrC,yBAAyB,EAAE,MAAM,GAAG,SAAS,CAAC;CAC/C,CAAC;AAEF,eAAO,MAAM,gCAAgC,EAAE,kBAAkB,CAAC,wBAAwB,CAMzF,CAAC;AAEF,wBAAgB,0BAA0B,IAAI,wBAAwB,CAErE"}
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { getConfigFromMappings } from '@aztec/foundation/config';
|
|
2
|
-
export const proverCoordinationConfigMappings = {
|
|
3
|
-
proverCoordinationNodeUrl: {
|
|
4
|
-
env: 'PROVER_COORDINATION_NODE_URL',
|
|
5
|
-
description: 'The URL of the tx provider node',
|
|
6
|
-
parseEnv: (val)=>val
|
|
7
|
-
}
|
|
8
|
-
};
|
|
9
|
-
export function getTxProviderConfigFromEnv() {
|
|
10
|
-
return getConfigFromMappings(proverCoordinationConfigMappings);
|
|
11
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import type { ArchiveSource, Archiver } from '@aztec/archiver';
|
|
2
|
-
import type { EpochCache } from '@aztec/epoch-cache';
|
|
3
|
-
import type { DataStoreConfig } from '@aztec/kv-store/config';
|
|
4
|
-
import type { ProverCoordination, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
|
|
5
|
-
import { type TelemetryClient } from '@aztec/telemetry-client';
|
|
6
|
-
import type { ProverNodeConfig } from '../config.js';
|
|
7
|
-
type ProverCoordinationDeps = {
|
|
8
|
-
aztecNodeTxProvider?: ProverCoordination;
|
|
9
|
-
worldStateSynchronizer?: WorldStateSynchronizer;
|
|
10
|
-
archiver?: Archiver | ArchiveSource;
|
|
11
|
-
telemetry?: TelemetryClient;
|
|
12
|
-
epochCache?: EpochCache;
|
|
13
|
-
};
|
|
14
|
-
/**
|
|
15
|
-
* Creates a prover coordination service.
|
|
16
|
-
* If p2p is enabled, prover coordination is done via p2p.
|
|
17
|
-
* If an Aztec node URL is provided, prover coordination is done via the Aztec node over http.
|
|
18
|
-
* If an aztec node is provided, it is returned directly.
|
|
19
|
-
*/
|
|
20
|
-
export declare function createProverCoordination(config: ProverNodeConfig & DataStoreConfig, deps: ProverCoordinationDeps): Promise<ProverCoordination>;
|
|
21
|
-
export {};
|
|
22
|
-
//# sourceMappingURL=factory.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/prover-coordination/factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE/D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAK9D,OAAO,KAAK,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAGlG,OAAO,EAAE,KAAK,eAAe,EAAmB,MAAM,yBAAyB,CAAC;AAEhF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGrD,KAAK,sBAAsB,GAAG;IAC5B,mBAAmB,CAAC,EAAE,kBAAkB,CAAC;IACzC,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;IAChD,QAAQ,CAAC,EAAE,QAAQ,GAAG,aAAa,CAAC;IACpC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB,CAAC;AAEF;;;;;GAKG;AACH,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,gBAAgB,GAAG,eAAe,EAC1C,IAAI,EAAE,sBAAsB,GAC3B,OAAO,CAAC,kBAAkB,CAAC,CAqC7B"}
|