@aztec/prover-node 0.85.0 → 0.86.0-nightly.20250425
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/actions/download-epoch-proving-job.d.ts +15 -0
- package/dest/actions/download-epoch-proving-job.d.ts.map +1 -0
- package/dest/actions/download-epoch-proving-job.js +35 -0
- package/dest/actions/index.d.ts +3 -0
- package/dest/actions/index.d.ts.map +1 -0
- package/dest/actions/index.js +2 -0
- package/dest/actions/rerun-epoch-proving-job.d.ts +11 -0
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -0
- package/dest/actions/rerun-epoch-proving-job.js +37 -0
- package/dest/actions/upload-epoch-proof-failure.d.ts +15 -0
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -0
- package/dest/actions/upload-epoch-proof-failure.js +78 -0
- package/dest/config.d.ts +4 -4
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +12 -7
- package/dest/factory.d.ts +3 -2
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +13 -13
- package/dest/index.d.ts +1 -0
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/job/epoch-proving-job-data.d.ts +17 -0
- package/dest/job/epoch-proving-job-data.d.ts.map +1 -0
- package/dest/job/epoch-proving-job-data.js +45 -0
- package/dest/job/epoch-proving-job.d.ts +9 -9
- package/dest/job/epoch-proving-job.d.ts.map +1 -1
- package/dest/job/epoch-proving-job.js +38 -21
- package/dest/metrics.d.ts +27 -4
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +104 -36
- package/dest/monitors/epoch-monitor.d.ts +1 -1
- package/dest/monitors/epoch-monitor.d.ts.map +1 -1
- package/dest/monitors/epoch-monitor.js +7 -2
- package/dest/prover-coordination/combined-prover-coordination.d.ts +22 -0
- package/dest/prover-coordination/combined-prover-coordination.d.ts.map +1 -0
- package/dest/prover-coordination/combined-prover-coordination.js +136 -0
- package/dest/prover-coordination/config.d.ts +1 -1
- package/dest/prover-coordination/config.d.ts.map +1 -1
- package/dest/prover-coordination/config.js +5 -4
- package/dest/prover-coordination/factory.d.ts +5 -4
- package/dest/prover-coordination/factory.d.ts.map +1 -1
- package/dest/prover-coordination/factory.js +20 -14
- package/dest/prover-node-publisher.d.ts +1 -2
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +7 -13
- package/dest/prover-node.d.ts +26 -24
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +103 -51
- package/dest/test/index.d.ts +4 -2
- package/dest/test/index.d.ts.map +1 -1
- package/dest/test/index.js +1 -1
- package/package.json +20 -22
- package/src/actions/download-epoch-proving-job.ts +46 -0
- package/src/actions/index.ts +2 -0
- package/src/actions/rerun-epoch-proving-job.ts +59 -0
- package/src/actions/upload-epoch-proof-failure.ts +88 -0
- package/src/config.ts +16 -10
- package/src/factory.ts +31 -17
- package/src/index.ts +1 -0
- package/src/job/epoch-proving-job-data.ts +68 -0
- package/src/job/epoch-proving-job.ts +53 -23
- package/src/metrics.ts +111 -38
- package/src/monitors/epoch-monitor.ts +5 -3
- package/src/prover-coordination/combined-prover-coordination.ts +160 -0
- package/src/prover-coordination/config.ts +6 -5
- package/src/prover-coordination/factory.ts +36 -25
- package/src/prover-node-publisher.ts +9 -18
- package/src/prover-node.ts +151 -74
- package/src/test/index.ts +5 -2
package/src/metrics.ts
CHANGED
|
@@ -1,107 +1,187 @@
|
|
|
1
|
+
import type { RollupContract } from '@aztec/ethereum';
|
|
2
|
+
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
1
3
|
import { createLogger } from '@aztec/foundation/log';
|
|
2
4
|
import type { L1PublishProofStats, L1PublishStats } from '@aztec/stdlib/stats';
|
|
3
5
|
import {
|
|
4
6
|
Attributes,
|
|
7
|
+
type BatchObservableResult,
|
|
5
8
|
type Gauge,
|
|
6
9
|
type Histogram,
|
|
10
|
+
type Meter,
|
|
7
11
|
Metrics,
|
|
12
|
+
type ObservableGauge,
|
|
8
13
|
type TelemetryClient,
|
|
14
|
+
type Tracer,
|
|
9
15
|
type UpDownCounter,
|
|
10
16
|
ValueType,
|
|
11
17
|
} from '@aztec/telemetry-client';
|
|
12
18
|
|
|
13
|
-
import { formatEther } from 'viem';
|
|
19
|
+
import { formatEther, formatUnits } from 'viem';
|
|
14
20
|
|
|
15
|
-
export class
|
|
21
|
+
export class ProverNodeJobMetrics {
|
|
16
22
|
proverEpochExecutionDuration: Histogram;
|
|
17
23
|
provingJobDuration: Histogram;
|
|
18
24
|
provingJobBlocks: Gauge;
|
|
19
25
|
provingJobTransactions: Gauge;
|
|
20
26
|
|
|
21
|
-
gasPrice: Histogram;
|
|
22
|
-
txCount: UpDownCounter;
|
|
23
|
-
txDuration: Histogram;
|
|
24
|
-
txGas: Histogram;
|
|
25
|
-
txCalldataSize: Histogram;
|
|
26
|
-
txCalldataGas: Histogram;
|
|
27
|
-
txBlobDataGasUsed: Histogram;
|
|
28
|
-
txBlobDataGasCost: Histogram;
|
|
29
|
-
txTotalFee: Histogram;
|
|
30
|
-
|
|
31
|
-
private senderBalance: Gauge;
|
|
32
|
-
|
|
33
27
|
constructor(
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
private meter: Meter,
|
|
29
|
+
public readonly tracer: Tracer,
|
|
36
30
|
private logger = createLogger('prover-node:publisher:metrics'),
|
|
37
31
|
) {
|
|
38
|
-
|
|
39
|
-
this.proverEpochExecutionDuration = meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
|
|
32
|
+
this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
|
|
40
33
|
description: 'Duration of execution of an epoch by the prover',
|
|
41
34
|
unit: 'ms',
|
|
42
35
|
valueType: ValueType.INT,
|
|
43
36
|
});
|
|
44
|
-
this.provingJobDuration = meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
|
|
37
|
+
this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
|
|
45
38
|
description: 'Duration of proving job',
|
|
46
39
|
unit: 's',
|
|
47
40
|
valueType: ValueType.DOUBLE,
|
|
48
41
|
});
|
|
49
|
-
this.provingJobBlocks = meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
|
|
42
|
+
this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
|
|
50
43
|
description: 'Number of blocks in a proven epoch',
|
|
51
44
|
valueType: ValueType.INT,
|
|
52
45
|
});
|
|
53
|
-
this.provingJobTransactions = meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
|
|
46
|
+
this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
|
|
54
47
|
description: 'Number of transactions in a proven epoch',
|
|
55
48
|
valueType: ValueType.INT,
|
|
56
49
|
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
|
|
53
|
+
this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
|
|
54
|
+
this.provingJobDuration.record(totalTimeMs / 1000);
|
|
55
|
+
this.provingJobBlocks.record(Math.floor(numBlocks));
|
|
56
|
+
this.provingJobTransactions.record(Math.floor(numTxs));
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export class ProverNodeRewardsMetrics {
|
|
61
|
+
private rewards: ObservableGauge;
|
|
62
|
+
private accumulatedRewards: UpDownCounter;
|
|
63
|
+
private prevEpoch = -1n;
|
|
64
|
+
private proofSubmissionWindow = 0n;
|
|
65
|
+
|
|
66
|
+
constructor(
|
|
67
|
+
private meter: Meter,
|
|
68
|
+
private coinbase: EthAddress,
|
|
69
|
+
private rollup: RollupContract,
|
|
70
|
+
private logger = createLogger('prover-node:publisher:metrics'),
|
|
71
|
+
) {
|
|
72
|
+
this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH, {
|
|
73
|
+
valueType: ValueType.DOUBLE,
|
|
74
|
+
description: 'The rewards earned',
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, {
|
|
78
|
+
valueType: ValueType.DOUBLE,
|
|
79
|
+
description: 'The rewards earned (total)',
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
public async start() {
|
|
84
|
+
this.proofSubmissionWindow = await this.rollup.getProofSubmissionWindow();
|
|
85
|
+
this.meter.addBatchObservableCallback(this.observe, [this.rewards]);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
public stop() {
|
|
89
|
+
this.meter.removeBatchObservableCallback(this.observe, [this.rewards]);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private observe = async (observer: BatchObservableResult): Promise<void> => {
|
|
93
|
+
const slot = await this.rollup.getSlotNumber();
|
|
94
|
+
|
|
95
|
+
// look at the prev epoch so that we get an accurate value, after proof submission window has closed
|
|
96
|
+
if (slot > this.proofSubmissionWindow) {
|
|
97
|
+
const closedEpoch = await this.rollup.getEpochNumberForSlotNumber(slot - this.proofSubmissionWindow);
|
|
98
|
+
const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase);
|
|
99
|
+
|
|
100
|
+
const fmt = parseFloat(formatUnits(rewards, 18));
|
|
101
|
+
|
|
102
|
+
observer.observe(this.rewards, fmt, {
|
|
103
|
+
[Attributes.COINBASE]: this.coinbase.toString(),
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
// only accumulate once per epoch
|
|
107
|
+
if (closedEpoch > this.prevEpoch) {
|
|
108
|
+
this.prevEpoch = closedEpoch;
|
|
109
|
+
this.accumulatedRewards.add(fmt, {
|
|
110
|
+
[Attributes.COINBASE]: this.coinbase.toString(),
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
}
|
|
57
116
|
|
|
58
|
-
|
|
117
|
+
export class ProverNodePublisherMetrics {
|
|
118
|
+
gasPrice: Histogram;
|
|
119
|
+
txCount: UpDownCounter;
|
|
120
|
+
txDuration: Histogram;
|
|
121
|
+
txGas: Histogram;
|
|
122
|
+
txCalldataSize: Histogram;
|
|
123
|
+
txCalldataGas: Histogram;
|
|
124
|
+
txBlobDataGasUsed: Histogram;
|
|
125
|
+
txBlobDataGasCost: Histogram;
|
|
126
|
+
txTotalFee: Histogram;
|
|
127
|
+
|
|
128
|
+
private senderBalance: Gauge;
|
|
129
|
+
private meter: Meter;
|
|
130
|
+
|
|
131
|
+
constructor(
|
|
132
|
+
public readonly client: TelemetryClient,
|
|
133
|
+
name = 'ProverNode',
|
|
134
|
+
private logger = createLogger('prover-node:publisher:metrics'),
|
|
135
|
+
) {
|
|
136
|
+
this.meter = client.getMeter(name);
|
|
137
|
+
|
|
138
|
+
this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
|
|
59
139
|
description: 'The gas price used for transactions',
|
|
60
140
|
unit: 'gwei',
|
|
61
141
|
valueType: ValueType.DOUBLE,
|
|
62
142
|
});
|
|
63
143
|
|
|
64
|
-
this.txCount = meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
|
|
144
|
+
this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
|
|
65
145
|
description: 'The number of transactions processed',
|
|
66
146
|
});
|
|
67
147
|
|
|
68
|
-
this.txDuration = meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
|
|
148
|
+
this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
|
|
69
149
|
description: 'The duration of transaction processing',
|
|
70
150
|
unit: 'ms',
|
|
71
151
|
valueType: ValueType.INT,
|
|
72
152
|
});
|
|
73
153
|
|
|
74
|
-
this.txGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
|
|
154
|
+
this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
|
|
75
155
|
description: 'The gas consumed by transactions',
|
|
76
156
|
unit: 'gas',
|
|
77
157
|
valueType: ValueType.INT,
|
|
78
158
|
});
|
|
79
159
|
|
|
80
|
-
this.txCalldataSize = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
|
|
160
|
+
this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
|
|
81
161
|
description: 'The size of the calldata in transactions',
|
|
82
162
|
unit: 'By',
|
|
83
163
|
valueType: ValueType.INT,
|
|
84
164
|
});
|
|
85
165
|
|
|
86
|
-
this.txCalldataGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
|
|
166
|
+
this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
|
|
87
167
|
description: 'The gas consumed by the calldata in transactions',
|
|
88
168
|
unit: 'gas',
|
|
89
169
|
valueType: ValueType.INT,
|
|
90
170
|
});
|
|
91
171
|
|
|
92
|
-
this.txBlobDataGasUsed = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
|
|
172
|
+
this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
|
|
93
173
|
description: 'The amount of blob gas used in transactions',
|
|
94
174
|
unit: 'gas',
|
|
95
175
|
valueType: ValueType.INT,
|
|
96
176
|
});
|
|
97
177
|
|
|
98
|
-
this.txBlobDataGasCost = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
|
|
178
|
+
this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
|
|
99
179
|
description: 'The gas cost of blobs in transactions',
|
|
100
180
|
unit: 'gwei',
|
|
101
181
|
valueType: ValueType.INT,
|
|
102
182
|
});
|
|
103
183
|
|
|
104
|
-
this.txTotalFee = meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
|
|
184
|
+
this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
|
|
105
185
|
description: 'How much L1 tx costs',
|
|
106
186
|
unit: 'gwei',
|
|
107
187
|
valueType: ValueType.DOUBLE,
|
|
@@ -112,7 +192,7 @@ export class ProverNodeMetrics {
|
|
|
112
192
|
},
|
|
113
193
|
});
|
|
114
194
|
|
|
115
|
-
this.senderBalance = meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
|
|
195
|
+
this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
|
|
116
196
|
unit: 'eth',
|
|
117
197
|
description: 'The balance of the sender address',
|
|
118
198
|
valueType: ValueType.DOUBLE,
|
|
@@ -130,13 +210,6 @@ export class ProverNodeMetrics {
|
|
|
130
210
|
this.recordTx(durationMs, stats);
|
|
131
211
|
}
|
|
132
212
|
|
|
133
|
-
public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
|
|
134
|
-
this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
|
|
135
|
-
this.provingJobDuration.record(totalTimeMs / 1000);
|
|
136
|
-
this.provingJobBlocks.record(Math.floor(numBlocks));
|
|
137
|
-
this.provingJobTransactions.record(Math.floor(numTxs));
|
|
138
|
-
}
|
|
139
|
-
|
|
140
213
|
public recordSenderBalance(wei: bigint, senderAddress: string) {
|
|
141
214
|
const eth = parseFloat(formatEther(wei, 'wei'));
|
|
142
215
|
this.senderBalance.record(eth, {
|
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
} from '@aztec/telemetry-client';
|
|
12
12
|
|
|
13
13
|
export interface EpochMonitorHandler {
|
|
14
|
-
handleEpochReadyToProve(epochNumber: bigint): Promise<
|
|
14
|
+
handleEpochReadyToProve(epochNumber: bigint): Promise<boolean>;
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
/**
|
|
@@ -71,6 +71,7 @@ export class EpochMonitor implements Traceable {
|
|
|
71
71
|
@trackSpan('EpochMonitor.work')
|
|
72
72
|
public async work() {
|
|
73
73
|
const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
|
|
74
|
+
this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
|
|
74
75
|
if (epochToProve === undefined) {
|
|
75
76
|
this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
|
|
76
77
|
return;
|
|
@@ -87,8 +88,9 @@ export class EpochMonitor implements Traceable {
|
|
|
87
88
|
}
|
|
88
89
|
|
|
89
90
|
this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
|
|
90
|
-
await this.handler?.handleEpochReadyToProve(epochToProve)
|
|
91
|
-
|
|
91
|
+
if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
|
|
92
|
+
this.latestEpochNumber = epochToProve;
|
|
93
|
+
}
|
|
92
94
|
}
|
|
93
95
|
|
|
94
96
|
private async getEpochNumberToProve() {
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { asyncPool } from '@aztec/foundation/async-pool';
|
|
2
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
3
|
+
import type { P2P } from '@aztec/p2p';
|
|
4
|
+
import type { P2PClient, ProverCoordination } from '@aztec/stdlib/interfaces/server';
|
|
5
|
+
import { type Tx, TxHash } from '@aztec/stdlib/tx';
|
|
6
|
+
|
|
7
|
+
export type CombinedCoordinationOptions = {
|
|
8
|
+
// These options apply to http tx gathering only
|
|
9
|
+
txGatheringBatchSize: number;
|
|
10
|
+
txGatheringMaxParallelRequestsPerNode: number;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
interface CoordinationPool {
|
|
14
|
+
getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
|
|
15
|
+
hasTxsInPool(txHashes: TxHash[]): Promise<boolean[]>;
|
|
16
|
+
getTxsByHashFromPool(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
|
|
17
|
+
addTxs(txs: Tx[]): Promise<void>;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface TxSource {
|
|
21
|
+
getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Wraps the p2p client into a coordination pool
|
|
25
|
+
class P2PCoordinationPool implements CoordinationPool {
|
|
26
|
+
constructor(private readonly p2p: P2P) {}
|
|
27
|
+
getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
|
|
28
|
+
return this.p2p.getTxsByHash(txHashes);
|
|
29
|
+
}
|
|
30
|
+
hasTxsInPool(txHashes: TxHash[]): Promise<boolean[]> {
|
|
31
|
+
return this.p2p.hasTxsInPool(txHashes);
|
|
32
|
+
}
|
|
33
|
+
getTxsByHashFromPool(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
|
|
34
|
+
return this.p2p.getTxsByHashFromPool(txHashes);
|
|
35
|
+
}
|
|
36
|
+
addTxs(txs: Tx[]): Promise<void> {
|
|
37
|
+
return this.p2p.addTxs(txs);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Wraps an in memory tx pool into a coordination pool. Used for testing when no p2p/tx pool is available.
|
|
42
|
+
class InMemoryCoordinationPool implements CoordinationPool {
|
|
43
|
+
private txs: Map<string, Tx> = new Map();
|
|
44
|
+
getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
|
|
45
|
+
return Promise.resolve(txHashes.map(hash => this.txs.get(hash.toString())));
|
|
46
|
+
}
|
|
47
|
+
hasTxsInPool(txHashes: TxHash[]): Promise<boolean[]> {
|
|
48
|
+
return Promise.resolve(txHashes.map(hash => this.txs.has(hash.toString())));
|
|
49
|
+
}
|
|
50
|
+
getTxsByHashFromPool(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
|
|
51
|
+
return this.getTxsByHash(txHashes);
|
|
52
|
+
}
|
|
53
|
+
async addTxs(txs: Tx[]): Promise<void> {
|
|
54
|
+
const hashes = await Promise.all(txs.map(tx => tx.getTxHash()));
|
|
55
|
+
txs.forEach((tx, index) => this.txs.set(hashes[index].toString(), tx));
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Class to implement combined transaction retrieval from p2p and any available nodes
|
|
60
|
+
export class CombinedProverCoordination implements ProverCoordination {
|
|
61
|
+
constructor(
|
|
62
|
+
public readonly p2p: P2P | undefined,
|
|
63
|
+
public readonly aztecNodes: TxSource[],
|
|
64
|
+
private readonly options: CombinedCoordinationOptions = {
|
|
65
|
+
txGatheringBatchSize: 10,
|
|
66
|
+
txGatheringMaxParallelRequestsPerNode: 10,
|
|
67
|
+
},
|
|
68
|
+
private readonly log = createLogger('prover-node:combined-prover-coordination'),
|
|
69
|
+
) {}
|
|
70
|
+
|
|
71
|
+
public getP2PClient(): P2PClient | undefined {
|
|
72
|
+
return this.p2p;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
public async getTxsByHash(txHashes: TxHash[]): Promise<Tx[]> {
|
|
76
|
+
const pool = this.p2p ? new P2PCoordinationPool(this.p2p) : new InMemoryCoordinationPool();
|
|
77
|
+
await this.#gatherTxs(txHashes, pool);
|
|
78
|
+
const availability = await pool.hasTxsInPool(txHashes);
|
|
79
|
+
const notFound = txHashes.filter((_, index) => !availability[index]);
|
|
80
|
+
if (notFound.length > 0) {
|
|
81
|
+
throw new Error(`Could not find txs: ${notFound.map(tx => tx.toString())}`);
|
|
82
|
+
}
|
|
83
|
+
const txs = await pool.getTxsByHashFromPool(txHashes);
|
|
84
|
+
return txs.filter(tx => tx !== undefined) as Tx[];
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
public async gatherTxs(txHashes: TxHash[]): Promise<void> {
|
|
88
|
+
const pool = this.p2p ? new P2PCoordinationPool(this.p2p) : new InMemoryCoordinationPool();
|
|
89
|
+
await this.#gatherTxs(txHashes, pool);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async #gatherTxs(txHashes: TxHash[], pool: CoordinationPool): Promise<void> {
|
|
93
|
+
const availability = await pool.hasTxsInPool(txHashes);
|
|
94
|
+
const notFound = txHashes.filter((_, index) => !availability[index]);
|
|
95
|
+
const txsToFind = new Set(notFound.map(tx => tx.toString()));
|
|
96
|
+
if (txsToFind.size === 0) {
|
|
97
|
+
this.log.info(`Check for ${txHashes.length} txs found all in the pool}`);
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
this.log.info(`Check for ${txHashes.length} txs found ${txsToFind.size} missing. Will gather from nodes and p2p`);
|
|
101
|
+
const originalToFind = txsToFind.size;
|
|
102
|
+
await this.#gatherTxsFromAllNodes(txsToFind, pool);
|
|
103
|
+
if (txsToFind.size === 0) {
|
|
104
|
+
this.log.info(`Found all ${originalToFind} txs directly from nodes`);
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
const toFindFromP2P = txsToFind.size;
|
|
108
|
+
this.log.verbose(`Gathering ${toFindFromP2P} txs from p2p network`);
|
|
109
|
+
const foundFromP2P = await pool.getTxsByHash([...txsToFind].map(tx => TxHash.fromString(tx)));
|
|
110
|
+
const numFoundFromNodes = originalToFind - toFindFromP2P;
|
|
111
|
+
const numNotFound = toFindFromP2P - foundFromP2P.length;
|
|
112
|
+
if (numNotFound === 0) {
|
|
113
|
+
this.log.info(
|
|
114
|
+
`Found all ${originalToFind} txs. ${numFoundFromNodes} from nodes, ${foundFromP2P.length} from p2p`,
|
|
115
|
+
);
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
this.log.warn(
|
|
119
|
+
`Failed to find ${numNotFound} txs from any source. Found ${foundFromP2P.length} from p2p and ${numFoundFromNodes} from nodes`,
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async #gatherTxsFromAllNodes(txsToFind: Set<string>, pool: CoordinationPool) {
|
|
124
|
+
if (txsToFind.size === 0 || this.aztecNodes.length === 0) {
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
await Promise.all(this.aztecNodes.map(aztecNode => this.#gatherTxsFromNode(txsToFind, aztecNode, pool)));
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async #gatherTxsFromNode(txsToFind: Set<string>, aztecNode: TxSource, pool: CoordinationPool) {
|
|
131
|
+
const totalTxsRequired = txsToFind.size;
|
|
132
|
+
|
|
133
|
+
// It's possible that the set is empty as we already found the txs
|
|
134
|
+
if (totalTxsRequired === 0) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
let totalTxsGathered = 0;
|
|
138
|
+
|
|
139
|
+
const batches: string[][] = [];
|
|
140
|
+
const allTxs: string[] = [...txsToFind];
|
|
141
|
+
while (allTxs.length) {
|
|
142
|
+
const batch = allTxs.splice(0, this.options.txGatheringBatchSize);
|
|
143
|
+
batches.push(batch);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
await asyncPool(this.options.txGatheringMaxParallelRequestsPerNode, batches, async batch => {
|
|
147
|
+
try {
|
|
148
|
+
const txs = (await aztecNode.getTxsByHash(batch.map(b => TxHash.fromString(b)))).filter((tx): tx is Tx => !!tx);
|
|
149
|
+
const hashes = await Promise.all(txs.map(tx => tx.getTxHash()));
|
|
150
|
+
await pool.addTxs(txs);
|
|
151
|
+
hashes.forEach(hash => txsToFind.delete(hash.toString()));
|
|
152
|
+
totalTxsGathered += txs.length;
|
|
153
|
+
} catch (err) {
|
|
154
|
+
this.log.error(`Error gathering txs from aztec node: ${err}`);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
this.log.verbose(`Gathered ${totalTxsGathered} of ${totalTxsRequired} txs from a node`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config';
|
|
2
2
|
|
|
3
3
|
export type ProverCoordinationConfig = {
|
|
4
|
-
|
|
4
|
+
proverCoordinationNodeUrls: string[];
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
export const proverCoordinationConfigMappings: ConfigMappingsType<ProverCoordinationConfig> = {
|
|
8
|
-
|
|
9
|
-
env: '
|
|
10
|
-
description: 'The
|
|
11
|
-
parseEnv: (val: string) => val,
|
|
8
|
+
proverCoordinationNodeUrls: {
|
|
9
|
+
env: 'PROVER_COORDINATION_NODE_URLS',
|
|
10
|
+
description: 'The URLs of the tx provider nodes',
|
|
11
|
+
parseEnv: (val: string) => val.split(',').map(url => url.trim().replace(/\/$/, '')),
|
|
12
|
+
defaultValue: [],
|
|
12
13
|
},
|
|
13
14
|
};
|
|
14
15
|
|
|
@@ -6,21 +6,26 @@ import type { DataStoreConfig } from '@aztec/kv-store/config';
|
|
|
6
6
|
import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
|
|
7
7
|
import { createP2PClient } from '@aztec/p2p';
|
|
8
8
|
import { protocolContractTreeRoot } from '@aztec/protocol-contracts';
|
|
9
|
-
import { createAztecNodeClient } from '@aztec/stdlib/interfaces/client';
|
|
9
|
+
import { type AztecNode, createAztecNodeClient } from '@aztec/stdlib/interfaces/client';
|
|
10
10
|
import type { ProverCoordination, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
|
|
11
11
|
import { P2PClientType } from '@aztec/stdlib/p2p';
|
|
12
12
|
import { getComponentsVersionsFromConfig } from '@aztec/stdlib/versioning';
|
|
13
13
|
import { type TelemetryClient, makeTracedFetch } from '@aztec/telemetry-client';
|
|
14
14
|
|
|
15
15
|
import type { ProverNodeConfig } from '../config.js';
|
|
16
|
+
import {
|
|
17
|
+
type CombinedCoordinationOptions,
|
|
18
|
+
CombinedProverCoordination,
|
|
19
|
+
type TxSource,
|
|
20
|
+
} from './combined-prover-coordination.js';
|
|
16
21
|
|
|
17
22
|
// We return a reference to the P2P client so that the prover node can stop the service when it shuts down.
|
|
18
23
|
type ProverCoordinationDeps = {
|
|
19
|
-
aztecNodeTxProvider?:
|
|
20
|
-
worldStateSynchronizer
|
|
21
|
-
archiver
|
|
24
|
+
aztecNodeTxProvider?: TxSource;
|
|
25
|
+
worldStateSynchronizer: WorldStateSynchronizer;
|
|
26
|
+
archiver: Archiver | ArchiveSource;
|
|
22
27
|
telemetry?: TelemetryClient;
|
|
23
|
-
epochCache
|
|
28
|
+
epochCache: EpochCache;
|
|
24
29
|
};
|
|
25
30
|
|
|
26
31
|
/**
|
|
@@ -35,9 +40,14 @@ export async function createProverCoordination(
|
|
|
35
40
|
): Promise<ProverCoordination> {
|
|
36
41
|
const log = createLogger('prover-node:prover-coordination');
|
|
37
42
|
|
|
43
|
+
const coordinationConfig: CombinedCoordinationOptions = {
|
|
44
|
+
txGatheringBatchSize: config.txGatheringBatchSize ?? 10,
|
|
45
|
+
txGatheringMaxParallelRequestsPerNode: config.txGatheringMaxParallelRequestsPerNode ?? 10,
|
|
46
|
+
};
|
|
47
|
+
|
|
38
48
|
if (deps.aztecNodeTxProvider) {
|
|
39
49
|
log.info('Using prover coordination via aztec node');
|
|
40
|
-
return deps.aztecNodeTxProvider;
|
|
50
|
+
return new CombinedProverCoordination(undefined, [deps.aztecNodeTxProvider!], coordinationConfig);
|
|
41
51
|
}
|
|
42
52
|
|
|
43
53
|
if (config.p2pEnabled) {
|
|
@@ -46,27 +56,28 @@ export async function createProverCoordination(
|
|
|
46
56
|
if (!deps.archiver || !deps.worldStateSynchronizer || !deps.telemetry || !deps.epochCache) {
|
|
47
57
|
throw new Error('Missing dependencies for p2p prover coordination');
|
|
48
58
|
}
|
|
49
|
-
|
|
50
|
-
const proofVerifier = config.realProofs ? await BBCircuitVerifier.new(config) : new TestCircuitVerifier();
|
|
51
|
-
const p2pClient = await createP2PClient(
|
|
52
|
-
P2PClientType.Prover,
|
|
53
|
-
config,
|
|
54
|
-
deps.archiver,
|
|
55
|
-
proofVerifier,
|
|
56
|
-
deps.worldStateSynchronizer,
|
|
57
|
-
deps.epochCache,
|
|
58
|
-
deps.telemetry,
|
|
59
|
-
);
|
|
60
|
-
await p2pClient.start();
|
|
61
|
-
|
|
62
|
-
return p2pClient;
|
|
63
59
|
}
|
|
64
60
|
|
|
65
|
-
|
|
66
|
-
|
|
61
|
+
let nodes: AztecNode[] = [];
|
|
62
|
+
if (config.proverCoordinationNodeUrls && config.proverCoordinationNodeUrls.length > 0) {
|
|
63
|
+
log.info('Using prover coordination via node urls');
|
|
67
64
|
const versions = getComponentsVersionsFromConfig(config, protocolContractTreeRoot, getVKTreeRoot());
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
65
|
+
nodes = config.proverCoordinationNodeUrls.map(url =>
|
|
66
|
+
createAztecNodeClient(url, versions, makeTracedFetch([1, 2, 3], false)),
|
|
67
|
+
);
|
|
71
68
|
}
|
|
69
|
+
|
|
70
|
+
const proofVerifier = config.realProofs ? await BBCircuitVerifier.new(config) : new TestCircuitVerifier();
|
|
71
|
+
const p2pClient = await createP2PClient(
|
|
72
|
+
P2PClientType.Prover,
|
|
73
|
+
config,
|
|
74
|
+
deps.archiver,
|
|
75
|
+
proofVerifier,
|
|
76
|
+
deps.worldStateSynchronizer,
|
|
77
|
+
deps.epochCache,
|
|
78
|
+
deps.telemetry,
|
|
79
|
+
);
|
|
80
|
+
await p2pClient.start();
|
|
81
|
+
|
|
82
|
+
return new CombinedProverCoordination(p2pClient, nodes, coordinationConfig);
|
|
72
83
|
}
|
|
@@ -17,7 +17,7 @@ import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-clien
|
|
|
17
17
|
|
|
18
18
|
import { type Hex, type TransactionReceipt, encodeFunctionData } from 'viem';
|
|
19
19
|
|
|
20
|
-
import {
|
|
20
|
+
import { ProverNodePublisherMetrics } from './metrics.js';
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
23
|
* Stats for a sent transaction.
|
|
@@ -27,8 +27,6 @@ export type L1SubmitEpochProofArgs = {
|
|
|
27
27
|
epochSize: number;
|
|
28
28
|
previousArchive: Fr;
|
|
29
29
|
endArchive: Fr;
|
|
30
|
-
previousBlockHash: Fr;
|
|
31
|
-
endBlockHash: Fr;
|
|
32
30
|
endTimestamp: Fr;
|
|
33
31
|
outHash: Fr;
|
|
34
32
|
proverId: Fr;
|
|
@@ -40,7 +38,7 @@ export class ProverNodePublisher {
|
|
|
40
38
|
private interruptibleSleep = new InterruptibleSleep();
|
|
41
39
|
private sleepTimeMs: number;
|
|
42
40
|
private interrupted = false;
|
|
43
|
-
private metrics:
|
|
41
|
+
private metrics: ProverNodePublisherMetrics;
|
|
44
42
|
|
|
45
43
|
protected log = createLogger('prover-node:l1-tx-publisher');
|
|
46
44
|
|
|
@@ -60,12 +58,16 @@ export class ProverNodePublisher {
|
|
|
60
58
|
|
|
61
59
|
const telemetry = deps.telemetry ?? getTelemetryClient();
|
|
62
60
|
|
|
63
|
-
this.metrics = new
|
|
61
|
+
this.metrics = new ProverNodePublisherMetrics(telemetry, 'ProverNode');
|
|
64
62
|
|
|
65
63
|
this.rollupContract = deps.rollupContract;
|
|
66
64
|
this.l1TxUtils = deps.l1TxUtils;
|
|
67
65
|
}
|
|
68
66
|
|
|
67
|
+
public getRollupContract() {
|
|
68
|
+
return this.rollupContract;
|
|
69
|
+
}
|
|
70
|
+
|
|
69
71
|
/**
|
|
70
72
|
* Calling `interrupt` will cause any in progress call to `publishRollup` to return `false` asap.
|
|
71
73
|
* Be warned, the call may return false even if the tx subsequently gets successfully mined.
|
|
@@ -158,30 +160,21 @@ export class ProverNodePublisher {
|
|
|
158
160
|
throw new Error(`Cannot submit epoch proof for ${fromBlock}-${toBlock} as pending block is ${pending}`);
|
|
159
161
|
}
|
|
160
162
|
|
|
161
|
-
// Check the
|
|
163
|
+
// Check the archive for the immediate block before the epoch
|
|
162
164
|
const blockLog = await this.rollupContract.getBlock(BigInt(fromBlock - 1));
|
|
163
165
|
if (publicInputs.previousArchive.root.toString() !== blockLog.archive) {
|
|
164
166
|
throw new Error(
|
|
165
167
|
`Previous archive root mismatch: ${publicInputs.previousArchive.root.toString()} !== ${blockLog.archive}`,
|
|
166
168
|
);
|
|
167
169
|
}
|
|
168
|
-
// TODO: Remove zero check once we inject the proper zero blockhash
|
|
169
|
-
if (blockLog.blockHash !== Fr.ZERO.toString() && publicInputs.previousBlockHash.toString() !== blockLog.blockHash) {
|
|
170
|
-
throw new Error(
|
|
171
|
-
`Previous block hash mismatch: ${publicInputs.previousBlockHash.toString()} !== ${blockLog.blockHash}`,
|
|
172
|
-
);
|
|
173
|
-
}
|
|
174
170
|
|
|
175
|
-
// Check the
|
|
171
|
+
// Check the archive for the last block in the epoch
|
|
176
172
|
const endBlockLog = await this.rollupContract.getBlock(BigInt(toBlock));
|
|
177
173
|
if (publicInputs.endArchive.root.toString() !== endBlockLog.archive) {
|
|
178
174
|
throw new Error(
|
|
179
175
|
`End archive root mismatch: ${publicInputs.endArchive.root.toString()} !== ${endBlockLog.archive}`,
|
|
180
176
|
);
|
|
181
177
|
}
|
|
182
|
-
if (publicInputs.endBlockHash.toString() !== endBlockLog.blockHash) {
|
|
183
|
-
throw new Error(`End block hash mismatch: ${publicInputs.endBlockHash.toString()} !== ${endBlockLog.blockHash}`);
|
|
184
|
-
}
|
|
185
178
|
|
|
186
179
|
// Compare the public inputs computed by the contract with the ones injected
|
|
187
180
|
const rollupPublicInputs = await this.rollupContract.getEpochProofPublicInputs(this.getSubmitEpochProofArgs(args));
|
|
@@ -258,8 +251,6 @@ export class ProverNodePublisher {
|
|
|
258
251
|
{
|
|
259
252
|
previousArchive: args.publicInputs.previousArchive.root.toString(),
|
|
260
253
|
endArchive: args.publicInputs.endArchive.root.toString(),
|
|
261
|
-
previousBlockHash: args.publicInputs.previousBlockHash.toString(),
|
|
262
|
-
endBlockHash: args.publicInputs.endBlockHash.toString(),
|
|
263
254
|
endTimestamp: args.publicInputs.endTimestamp.toBigInt(),
|
|
264
255
|
outHash: args.publicInputs.outHash.toString(),
|
|
265
256
|
proverId: EthAddress.fromField(args.publicInputs.proverId).toString(),
|