@aztec/prover-node 0.0.0-test.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dest/config.d.ts +28 -0
- package/dest/config.d.ts.map +1 -0
- package/dest/config.js +71 -0
- package/dest/factory.d.ts +25 -0
- package/dest/factory.d.ts.map +1 -0
- package/dest/factory.js +60 -0
- package/dest/http.d.ts +8 -0
- package/dest/http.d.ts.map +1 -0
- package/dest/http.js +9 -0
- package/dest/index.d.ts +6 -0
- package/dest/index.d.ts.map +1 -0
- package/dest/index.js +5 -0
- package/dest/job/epoch-proving-job.d.ts +54 -0
- package/dest/job/epoch-proving-job.d.ts.map +1 -0
- package/dest/job/epoch-proving-job.js +255 -0
- package/dest/metrics.d.ts +26 -0
- package/dest/metrics.d.ts.map +1 -0
- package/dest/metrics.js +129 -0
- package/dest/monitors/epoch-monitor.d.ts +40 -0
- package/dest/monitors/epoch-monitor.d.ts.map +1 -0
- package/dest/monitors/epoch-monitor.js +104 -0
- package/dest/monitors/index.d.ts +2 -0
- package/dest/monitors/index.d.ts.map +1 -0
- package/dest/monitors/index.js +1 -0
- package/dest/prover-coordination/config.d.ts +7 -0
- package/dest/prover-coordination/config.d.ts.map +1 -0
- package/dest/prover-coordination/config.js +11 -0
- package/dest/prover-coordination/factory.d.ts +22 -0
- package/dest/prover-coordination/factory.d.ts.map +1 -0
- package/dest/prover-coordination/factory.js +42 -0
- package/dest/prover-coordination/index.d.ts +3 -0
- package/dest/prover-coordination/index.d.ts.map +1 -0
- package/dest/prover-coordination/index.js +2 -0
- package/dest/prover-node-publisher.d.ts +61 -0
- package/dest/prover-node-publisher.d.ts.map +1 -0
- package/dest/prover-node-publisher.js +192 -0
- package/dest/prover-node.d.ts +102 -0
- package/dest/prover-node.d.ts.map +1 -0
- package/dest/prover-node.js +258 -0
- package/dest/test/index.d.ts +10 -0
- package/dest/test/index.d.ts.map +1 -0
- package/dest/test/index.js +5 -0
- package/package.json +98 -0
- package/src/config.ts +115 -0
- package/src/factory.ts +99 -0
- package/src/http.ts +13 -0
- package/src/index.ts +5 -0
- package/src/job/epoch-proving-job.ts +255 -0
- package/src/metrics.ts +164 -0
- package/src/monitors/epoch-monitor.ts +106 -0
- package/src/monitors/index.ts +1 -0
- package/src/prover-coordination/config.ts +17 -0
- package/src/prover-coordination/factory.ts +72 -0
- package/src/prover-coordination/index.ts +2 -0
- package/src/prover-node-publisher.ts +286 -0
- package/src/prover-node.ts +335 -0
- package/src/test/index.ts +11 -0
package/src/factory.ts
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { type Archiver, createArchiver } from '@aztec/archiver';
|
|
2
|
+
import { type BlobSinkClientInterface, createBlobSinkClient } from '@aztec/blob-sink/client';
|
|
3
|
+
import { EpochCache } from '@aztec/epoch-cache';
|
|
4
|
+
import { L1TxUtils, RollupContract, createEthereumChain, createL1Clients } from '@aztec/ethereum';
|
|
5
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
6
|
+
import type { DataStoreConfig } from '@aztec/kv-store/config';
|
|
7
|
+
import { createProverClient } from '@aztec/prover-client';
|
|
8
|
+
import { createAndStartProvingBroker } from '@aztec/prover-client/broker';
|
|
9
|
+
import type { ProverCoordination, ProvingJobBroker } from '@aztec/stdlib/interfaces/server';
|
|
10
|
+
import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
11
|
+
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
12
|
+
import { createWorldStateSynchronizer } from '@aztec/world-state';
|
|
13
|
+
|
|
14
|
+
import type { ProverNodeConfig } from './config.js';
|
|
15
|
+
import { EpochMonitor } from './monitors/epoch-monitor.js';
|
|
16
|
+
import { createProverCoordination } from './prover-coordination/factory.js';
|
|
17
|
+
import { ProverNodePublisher } from './prover-node-publisher.js';
|
|
18
|
+
import { ProverNode, type ProverNodeOptions } from './prover-node.js';
|
|
19
|
+
|
|
20
|
+
/** Creates a new prover node given a config. */
|
|
21
|
+
export async function createProverNode(
|
|
22
|
+
config: ProverNodeConfig & DataStoreConfig,
|
|
23
|
+
deps: {
|
|
24
|
+
telemetry?: TelemetryClient;
|
|
25
|
+
log?: Logger;
|
|
26
|
+
aztecNodeTxProvider?: ProverCoordination;
|
|
27
|
+
archiver?: Archiver;
|
|
28
|
+
publisher?: ProverNodePublisher;
|
|
29
|
+
blobSinkClient?: BlobSinkClientInterface;
|
|
30
|
+
broker?: ProvingJobBroker;
|
|
31
|
+
l1TxUtils?: L1TxUtils;
|
|
32
|
+
} = {},
|
|
33
|
+
options: {
|
|
34
|
+
prefilledPublicData?: PublicDataTreeLeaf[];
|
|
35
|
+
} = {},
|
|
36
|
+
) {
|
|
37
|
+
const telemetry = deps.telemetry ?? getTelemetryClient();
|
|
38
|
+
const blobSinkClient = deps.blobSinkClient ?? createBlobSinkClient(config);
|
|
39
|
+
const log = deps.log ?? createLogger('prover-node');
|
|
40
|
+
const archiver = deps.archiver ?? (await createArchiver(config, blobSinkClient, { blockUntilSync: true }, telemetry));
|
|
41
|
+
log.verbose(`Created archiver and synced to block ${await archiver.getBlockNumber()}`);
|
|
42
|
+
|
|
43
|
+
const worldStateConfig = { ...config, worldStateProvenBlocksOnly: false };
|
|
44
|
+
const worldStateSynchronizer = await createWorldStateSynchronizer(
|
|
45
|
+
worldStateConfig,
|
|
46
|
+
archiver,
|
|
47
|
+
options.prefilledPublicData,
|
|
48
|
+
telemetry,
|
|
49
|
+
);
|
|
50
|
+
await worldStateSynchronizer.start();
|
|
51
|
+
|
|
52
|
+
const broker = deps.broker ?? (await createAndStartProvingBroker(config, telemetry));
|
|
53
|
+
const prover = await createProverClient(config, worldStateSynchronizer, broker, telemetry);
|
|
54
|
+
|
|
55
|
+
const { l1RpcUrls: rpcUrls, l1ChainId: chainId, publisherPrivateKey } = config;
|
|
56
|
+
const chain = createEthereumChain(rpcUrls, chainId);
|
|
57
|
+
const { publicClient, walletClient } = createL1Clients(rpcUrls, publisherPrivateKey, chain.chainInfo);
|
|
58
|
+
|
|
59
|
+
const rollupContract = new RollupContract(publicClient, config.l1Contracts.rollupAddress.toString());
|
|
60
|
+
|
|
61
|
+
const l1TxUtils = deps.l1TxUtils ?? new L1TxUtils(publicClient, walletClient, log, config);
|
|
62
|
+
const publisher = deps.publisher ?? new ProverNodePublisher(config, { telemetry, rollupContract, l1TxUtils });
|
|
63
|
+
|
|
64
|
+
const epochCache = await EpochCache.create(config.l1Contracts.rollupAddress, config);
|
|
65
|
+
|
|
66
|
+
// If config.p2pEnabled is true, createProverCoordination will create a p2p client where txs are requested
|
|
67
|
+
// If config.p2pEnabled is false, createProverCoordination request information from the AztecNode
|
|
68
|
+
const proverCoordination = await createProverCoordination(config, {
|
|
69
|
+
aztecNodeTxProvider: deps.aztecNodeTxProvider,
|
|
70
|
+
worldStateSynchronizer,
|
|
71
|
+
archiver,
|
|
72
|
+
epochCache,
|
|
73
|
+
telemetry,
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
const proverNodeConfig: ProverNodeOptions = {
|
|
77
|
+
maxPendingJobs: config.proverNodeMaxPendingJobs,
|
|
78
|
+
pollingIntervalMs: config.proverNodePollingIntervalMs,
|
|
79
|
+
maxParallelBlocksPerEpoch: config.proverNodeMaxParallelBlocksPerEpoch,
|
|
80
|
+
txGatheringMaxParallelRequests: config.txGatheringMaxParallelRequests,
|
|
81
|
+
txGatheringIntervalMs: config.txGatheringIntervalMs,
|
|
82
|
+
txGatheringTimeoutMs: config.txGatheringTimeoutMs,
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
const epochMonitor = await EpochMonitor.create(archiver, proverNodeConfig, telemetry);
|
|
86
|
+
|
|
87
|
+
return new ProverNode(
|
|
88
|
+
prover,
|
|
89
|
+
publisher,
|
|
90
|
+
archiver,
|
|
91
|
+
archiver,
|
|
92
|
+
archiver,
|
|
93
|
+
worldStateSynchronizer,
|
|
94
|
+
proverCoordination,
|
|
95
|
+
epochMonitor,
|
|
96
|
+
proverNodeConfig,
|
|
97
|
+
telemetry,
|
|
98
|
+
);
|
|
99
|
+
}
|
package/src/http.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { ProverNodeApiSchema } from '@aztec/stdlib/interfaces/server';
|
|
2
|
+
import { createTracedJsonRpcServer } from '@aztec/telemetry-client';
|
|
3
|
+
|
|
4
|
+
import type { ProverNode } from './prover-node.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Wrap a ProverNode instance with a JSON RPC HTTP server.
|
|
8
|
+
* @param node - The ProverNode
|
|
9
|
+
* @returns An JSON-RPC HTTP server
|
|
10
|
+
*/
|
|
11
|
+
export function createProverNodeRpcServer(node: ProverNode) {
|
|
12
|
+
return createTracedJsonRpcServer(node, ProverNodeApiSchema);
|
|
13
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { asyncPool } from '@aztec/foundation/async-pool';
|
|
2
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
3
|
+
import { promiseWithResolvers } from '@aztec/foundation/promise';
|
|
4
|
+
import { Timer } from '@aztec/foundation/timer';
|
|
5
|
+
import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
|
|
6
|
+
import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
|
|
7
|
+
import {
|
|
8
|
+
type EpochProver,
|
|
9
|
+
type EpochProvingJobState,
|
|
10
|
+
EpochProvingJobTerminalState,
|
|
11
|
+
type ForkMerkleTreeOperations,
|
|
12
|
+
} from '@aztec/stdlib/interfaces/server';
|
|
13
|
+
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
14
|
+
import type { ProcessedTx, Tx } from '@aztec/stdlib/tx';
|
|
15
|
+
import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
|
|
16
|
+
|
|
17
|
+
import * as crypto from 'node:crypto';
|
|
18
|
+
|
|
19
|
+
import type { ProverNodeMetrics } from '../metrics.js';
|
|
20
|
+
import type { ProverNodePublisher } from '../prover-node-publisher.js';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Job that grabs a range of blocks from the unfinalised chain from L1, gets their txs given their hashes,
|
|
24
|
+
* re-executes their public calls, generates a rollup proof, and submits it to L1. This job will update the
|
|
25
|
+
* world state as part of public call execution via the public processor.
|
|
26
|
+
*/
|
|
27
|
+
export class EpochProvingJob implements Traceable {
|
|
28
|
+
private state: EpochProvingJobState = 'initialized';
|
|
29
|
+
private log = createLogger('prover-node:epoch-proving-job');
|
|
30
|
+
private uuid: string;
|
|
31
|
+
|
|
32
|
+
private runPromise: Promise<void> | undefined;
|
|
33
|
+
private deadlineTimeoutHandler: NodeJS.Timeout | undefined;
|
|
34
|
+
|
|
35
|
+
public readonly tracer: Tracer;
|
|
36
|
+
|
|
37
|
+
constructor(
|
|
38
|
+
private dbProvider: ForkMerkleTreeOperations,
|
|
39
|
+
private epochNumber: bigint,
|
|
40
|
+
private blocks: L2Block[],
|
|
41
|
+
private txs: Tx[],
|
|
42
|
+
private prover: EpochProver,
|
|
43
|
+
private publicProcessorFactory: PublicProcessorFactory,
|
|
44
|
+
private publisher: ProverNodePublisher,
|
|
45
|
+
private l2BlockSource: L2BlockSource,
|
|
46
|
+
private l1ToL2MessageSource: L1ToL2MessageSource,
|
|
47
|
+
private metrics: ProverNodeMetrics,
|
|
48
|
+
private deadline: Date | undefined,
|
|
49
|
+
private config: { parallelBlockLimit: number } = { parallelBlockLimit: 32 },
|
|
50
|
+
private cleanUp: (job: EpochProvingJob) => Promise<void> = () => Promise.resolve(),
|
|
51
|
+
) {
|
|
52
|
+
this.uuid = crypto.randomUUID();
|
|
53
|
+
this.tracer = metrics.client.getTracer('EpochProvingJob');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
public getId(): string {
|
|
57
|
+
return this.uuid;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
public getState(): EpochProvingJobState {
|
|
61
|
+
return this.state;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
public getEpochNumber(): bigint {
|
|
65
|
+
return this.epochNumber;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Proves the given epoch and submits the proof to L1.
|
|
70
|
+
*/
|
|
71
|
+
@trackSpan('EpochProvingJob.run', function () {
|
|
72
|
+
return { [Attributes.EPOCH_NUMBER]: Number(this.epochNumber) };
|
|
73
|
+
})
|
|
74
|
+
public async run() {
|
|
75
|
+
this.scheduleDeadlineStop();
|
|
76
|
+
|
|
77
|
+
const epochNumber = Number(this.epochNumber);
|
|
78
|
+
const epochSizeBlocks = this.blocks.length;
|
|
79
|
+
const epochSizeTxs = this.blocks.reduce((total, current) => total + current.body.txEffects.length, 0);
|
|
80
|
+
const [fromBlock, toBlock] = [this.blocks[0].number, this.blocks.at(-1)!.number];
|
|
81
|
+
this.log.info(`Starting epoch ${epochNumber} proving job with blocks ${fromBlock} to ${toBlock}`, {
|
|
82
|
+
fromBlock,
|
|
83
|
+
toBlock,
|
|
84
|
+
epochSizeBlocks,
|
|
85
|
+
epochNumber,
|
|
86
|
+
uuid: this.uuid,
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
this.progressState('processing');
|
|
90
|
+
const timer = new Timer();
|
|
91
|
+
const { promise, resolve } = promiseWithResolvers<void>();
|
|
92
|
+
this.runPromise = promise;
|
|
93
|
+
|
|
94
|
+
try {
|
|
95
|
+
this.prover.startNewEpoch(epochNumber, fromBlock, epochSizeBlocks);
|
|
96
|
+
await this.prover.startTubeCircuits(this.txs);
|
|
97
|
+
|
|
98
|
+
await asyncPool(this.config.parallelBlockLimit, this.blocks, async block => {
|
|
99
|
+
this.checkState();
|
|
100
|
+
|
|
101
|
+
const globalVariables = block.header.globalVariables;
|
|
102
|
+
const txs = await this.getTxs(block);
|
|
103
|
+
const l1ToL2Messages = await this.getL1ToL2Messages(block);
|
|
104
|
+
const previousHeader = (await this.getBlockHeader(block.number - 1))!;
|
|
105
|
+
|
|
106
|
+
this.log.verbose(`Starting processing block ${block.number}`, {
|
|
107
|
+
number: block.number,
|
|
108
|
+
blockHash: (await block.hash()).toString(),
|
|
109
|
+
lastArchive: block.header.lastArchive.root,
|
|
110
|
+
noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
|
|
111
|
+
nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
|
|
112
|
+
publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
|
|
113
|
+
previousHeader: previousHeader.hash(),
|
|
114
|
+
uuid: this.uuid,
|
|
115
|
+
...globalVariables,
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
// Start block proving
|
|
119
|
+
await this.prover.startNewBlock(globalVariables, l1ToL2Messages, previousHeader);
|
|
120
|
+
|
|
121
|
+
// Process public fns
|
|
122
|
+
const db = await this.dbProvider.fork(block.number - 1);
|
|
123
|
+
const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, true);
|
|
124
|
+
const processed = await this.processTxs(publicProcessor, txs);
|
|
125
|
+
await this.prover.addTxs(processed);
|
|
126
|
+
await db.close();
|
|
127
|
+
this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
|
|
128
|
+
blockNumber: block.number,
|
|
129
|
+
blockHash: (await block.hash()).toString(),
|
|
130
|
+
uuid: this.uuid,
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
// Mark block as completed to pad it
|
|
134
|
+
await this.prover.setBlockCompleted(block.number, block.header);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
const executionTime = timer.ms();
|
|
138
|
+
|
|
139
|
+
this.progressState('awaiting-prover');
|
|
140
|
+
const { publicInputs, proof } = await this.prover.finaliseEpoch();
|
|
141
|
+
this.log.info(`Finalised proof for epoch ${epochNumber}`, { epochNumber, uuid: this.uuid, duration: timer.ms() });
|
|
142
|
+
|
|
143
|
+
this.progressState('publishing-proof');
|
|
144
|
+
const success = await this.publisher.submitEpochProof({ fromBlock, toBlock, epochNumber, publicInputs, proof });
|
|
145
|
+
if (!success) {
|
|
146
|
+
throw new Error('Failed to submit epoch proof to L1');
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
this.log.info(`Submitted proof for epoch ${epochNumber} (blocks ${fromBlock} to ${toBlock})`, {
|
|
150
|
+
epochNumber,
|
|
151
|
+
uuid: this.uuid,
|
|
152
|
+
});
|
|
153
|
+
this.state = 'completed';
|
|
154
|
+
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
|
|
155
|
+
} catch (err: any) {
|
|
156
|
+
if (err && err.name === 'HaltExecutionError') {
|
|
157
|
+
this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, { uuid: this.uuid, epochNumber });
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
this.log.error(`Error running epoch ${epochNumber} prover job`, err, { uuid: this.uuid, epochNumber });
|
|
161
|
+
this.state = 'failed';
|
|
162
|
+
} finally {
|
|
163
|
+
clearTimeout(this.deadlineTimeoutHandler);
|
|
164
|
+
await this.cleanUp(this);
|
|
165
|
+
await this.prover.stop();
|
|
166
|
+
resolve();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
private progressState(state: EpochProvingJobState) {
|
|
171
|
+
this.checkState();
|
|
172
|
+
this.state = state;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
private checkState() {
|
|
176
|
+
if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed') {
|
|
177
|
+
throw new HaltExecutionError(this.state);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
public async stop(state: EpochProvingJobState = 'stopped') {
|
|
182
|
+
this.state = state;
|
|
183
|
+
this.prover.cancel();
|
|
184
|
+
// TODO(palla/prover): Stop the publisher as well
|
|
185
|
+
if (this.runPromise) {
|
|
186
|
+
await this.runPromise;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private scheduleDeadlineStop() {
|
|
191
|
+
const deadline = this.deadline;
|
|
192
|
+
if (deadline) {
|
|
193
|
+
const timeout = deadline.getTime() - Date.now();
|
|
194
|
+
if (timeout <= 0) {
|
|
195
|
+
throw new Error('Cannot start job with deadline in the past');
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
this.deadlineTimeoutHandler = setTimeout(() => {
|
|
199
|
+
if (EpochProvingJobTerminalState.includes(this.state)) {
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
this.log.warn('Stopping job due to deadline hit', { uuid: this.uuid, epochNumber: this.epochNumber });
|
|
203
|
+
this.stop('timed-out').catch(err => {
|
|
204
|
+
this.log.error('Error stopping job', err, { uuid: this.uuid, epochNumber: this.epochNumber });
|
|
205
|
+
});
|
|
206
|
+
}, timeout);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/* Returns the header for the given block number, or the genesis block for block zero. */
|
|
211
|
+
private async getBlockHeader(blockNumber: number) {
|
|
212
|
+
if (blockNumber === 0) {
|
|
213
|
+
return (await this.dbProvider.fork()).getInitialHeader();
|
|
214
|
+
}
|
|
215
|
+
return this.l2BlockSource.getBlockHeader(blockNumber);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
private async getTxs(block: L2Block): Promise<Tx[]> {
|
|
219
|
+
const txHashes = block.body.txEffects.map(tx => tx.txHash.toBigInt());
|
|
220
|
+
const txsAndHashes = await Promise.all(this.txs.map(async tx => ({ tx, hash: await tx.getTxHash() })));
|
|
221
|
+
return txsAndHashes
|
|
222
|
+
.filter(txAndHash => txHashes.includes(txAndHash.hash.toBigInt()))
|
|
223
|
+
.map(txAndHash => txAndHash.tx);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
private getL1ToL2Messages(block: L2Block) {
|
|
227
|
+
return this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(block.number));
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {
|
|
231
|
+
const { deadline } = this;
|
|
232
|
+
const [processedTxs, failedTxs] = await publicProcessor.process(txs, { deadline });
|
|
233
|
+
|
|
234
|
+
if (failedTxs.length) {
|
|
235
|
+
throw new Error(
|
|
236
|
+
`Txs failed processing: ${failedTxs.map(({ tx, error }) => `${tx.getTxHash()} (${error})`).join(', ')}`,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
if (processedTxs.length !== txs.length) {
|
|
241
|
+
throw new Error(`Failed to process all txs: processed ${processedTxs.length} out of ${txs.length}`);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
return processedTxs;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
class HaltExecutionError extends Error {
|
|
249
|
+
constructor(state: EpochProvingJobState) {
|
|
250
|
+
super(`Halted execution due to state ${state}`);
|
|
251
|
+
this.name = 'HaltExecutionError';
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
export { type EpochProvingJobState };
|
package/src/metrics.ts
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import type { L1PublishProofStats, L1PublishStats } from '@aztec/stdlib/stats';
|
|
3
|
+
import {
|
|
4
|
+
Attributes,
|
|
5
|
+
type Gauge,
|
|
6
|
+
type Histogram,
|
|
7
|
+
Metrics,
|
|
8
|
+
type TelemetryClient,
|
|
9
|
+
type UpDownCounter,
|
|
10
|
+
ValueType,
|
|
11
|
+
} from '@aztec/telemetry-client';
|
|
12
|
+
|
|
13
|
+
import { formatEther } from 'viem';
|
|
14
|
+
|
|
15
|
+
export class ProverNodeMetrics {
|
|
16
|
+
proverEpochExecutionDuration: Histogram;
|
|
17
|
+
provingJobDuration: Histogram;
|
|
18
|
+
provingJobBlocks: Gauge;
|
|
19
|
+
provingJobTransactions: Gauge;
|
|
20
|
+
|
|
21
|
+
gasPrice: Histogram;
|
|
22
|
+
txCount: UpDownCounter;
|
|
23
|
+
txDuration: Histogram;
|
|
24
|
+
txGas: Histogram;
|
|
25
|
+
txCalldataSize: Histogram;
|
|
26
|
+
txCalldataGas: Histogram;
|
|
27
|
+
txBlobDataGasUsed: Histogram;
|
|
28
|
+
txBlobDataGasCost: Histogram;
|
|
29
|
+
|
|
30
|
+
private senderBalance: Gauge;
|
|
31
|
+
|
|
32
|
+
constructor(
|
|
33
|
+
public readonly client: TelemetryClient,
|
|
34
|
+
name = 'ProverNode',
|
|
35
|
+
private logger = createLogger('prover-node:publisher:metrics'),
|
|
36
|
+
) {
|
|
37
|
+
const meter = client.getMeter(name);
|
|
38
|
+
this.proverEpochExecutionDuration = meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
|
|
39
|
+
description: 'Duration of execution of an epoch by the prover',
|
|
40
|
+
unit: 'ms',
|
|
41
|
+
valueType: ValueType.INT,
|
|
42
|
+
});
|
|
43
|
+
this.provingJobDuration = meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
|
|
44
|
+
description: 'Duration of proving job',
|
|
45
|
+
unit: 's',
|
|
46
|
+
valueType: ValueType.DOUBLE,
|
|
47
|
+
});
|
|
48
|
+
this.provingJobBlocks = meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
|
|
49
|
+
description: 'Number of blocks in a proven epoch',
|
|
50
|
+
valueType: ValueType.INT,
|
|
51
|
+
});
|
|
52
|
+
this.provingJobTransactions = meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
|
|
53
|
+
description: 'Number of transactions in a proven epoch',
|
|
54
|
+
valueType: ValueType.INT,
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
this.gasPrice = meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
|
|
58
|
+
description: 'The gas price used for transactions',
|
|
59
|
+
unit: 'gwei',
|
|
60
|
+
valueType: ValueType.DOUBLE,
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
this.txCount = meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
|
|
64
|
+
description: 'The number of transactions processed',
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
this.txDuration = meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
|
|
68
|
+
description: 'The duration of transaction processing',
|
|
69
|
+
unit: 'ms',
|
|
70
|
+
valueType: ValueType.INT,
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
this.txGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
|
|
74
|
+
description: 'The gas consumed by transactions',
|
|
75
|
+
unit: 'gas',
|
|
76
|
+
valueType: ValueType.INT,
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
this.txCalldataSize = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
|
|
80
|
+
description: 'The size of the calldata in transactions',
|
|
81
|
+
unit: 'By',
|
|
82
|
+
valueType: ValueType.INT,
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
this.txCalldataGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
|
|
86
|
+
description: 'The gas consumed by the calldata in transactions',
|
|
87
|
+
unit: 'gas',
|
|
88
|
+
valueType: ValueType.INT,
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
this.txBlobDataGasUsed = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
|
|
92
|
+
description: 'The amount of blob gas used in transactions',
|
|
93
|
+
unit: 'gas',
|
|
94
|
+
valueType: ValueType.INT,
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
this.txBlobDataGasCost = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
|
|
98
|
+
description: 'The gas cost of blobs in transactions',
|
|
99
|
+
unit: 'gwei',
|
|
100
|
+
valueType: ValueType.INT,
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
this.senderBalance = meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
|
|
104
|
+
unit: 'eth',
|
|
105
|
+
description: 'The balance of the sender address',
|
|
106
|
+
valueType: ValueType.DOUBLE,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
recordFailedTx() {
|
|
111
|
+
this.txCount.add(1, {
|
|
112
|
+
[Attributes.L1_TX_TYPE]: 'submitProof',
|
|
113
|
+
[Attributes.OK]: false,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
recordSubmitProof(durationMs: number, stats: L1PublishProofStats) {
|
|
118
|
+
this.recordTx(durationMs, stats);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
|
|
122
|
+
this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
|
|
123
|
+
this.provingJobDuration.record(totalTimeMs / 1000);
|
|
124
|
+
this.provingJobBlocks.record(Math.floor(numBlocks));
|
|
125
|
+
this.provingJobTransactions.record(Math.floor(numTxs));
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
public recordSenderBalance(wei: bigint, senderAddress: string) {
|
|
129
|
+
const eth = parseFloat(formatEther(wei, 'wei'));
|
|
130
|
+
this.senderBalance.record(eth, {
|
|
131
|
+
[Attributes.SENDER_ADDRESS]: senderAddress,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
private recordTx(durationMs: number, stats: L1PublishStats) {
|
|
136
|
+
const attributes = {
|
|
137
|
+
[Attributes.L1_TX_TYPE]: 'submitProof',
|
|
138
|
+
[Attributes.L1_SENDER]: stats.sender,
|
|
139
|
+
} as const;
|
|
140
|
+
|
|
141
|
+
this.txCount.add(1, {
|
|
142
|
+
...attributes,
|
|
143
|
+
[Attributes.OK]: true,
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
this.txDuration.record(Math.ceil(durationMs), attributes);
|
|
147
|
+
this.txGas.record(
|
|
148
|
+
// safe to downcast - total block limit is 30M gas which fits in a JS number
|
|
149
|
+
Number(stats.gasUsed),
|
|
150
|
+
attributes,
|
|
151
|
+
);
|
|
152
|
+
this.txCalldataGas.record(stats.calldataGas, attributes);
|
|
153
|
+
this.txCalldataSize.record(stats.calldataSize, attributes);
|
|
154
|
+
|
|
155
|
+
this.txBlobDataGasCost.record(Number(stats.blobDataGas), attributes);
|
|
156
|
+
this.txBlobDataGasUsed.record(Number(stats.blobGasUsed), attributes);
|
|
157
|
+
|
|
158
|
+
try {
|
|
159
|
+
this.gasPrice.record(parseInt(formatEther(stats.gasPrice, 'gwei'), 10));
|
|
160
|
+
} catch (e) {
|
|
161
|
+
// ignore
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
3
|
+
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
4
|
+
import { type L1RollupConstants, getEpochAtSlot } from '@aztec/stdlib/epoch-helpers';
|
|
5
|
+
import {
|
|
6
|
+
type TelemetryClient,
|
|
7
|
+
type Traceable,
|
|
8
|
+
type Tracer,
|
|
9
|
+
getTelemetryClient,
|
|
10
|
+
trackSpan,
|
|
11
|
+
} from '@aztec/telemetry-client';
|
|
12
|
+
|
|
13
|
+
export interface EpochMonitorHandler {
|
|
14
|
+
handleEpochReadyToProve(epochNumber: bigint): Promise<void>;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Fires an event when a new epoch ready to prove is detected.
|
|
19
|
+
*
|
|
20
|
+
* We define an epoch as ready to prove when:
|
|
21
|
+
* - The epoch is complete
|
|
22
|
+
* - Its blocks have not been reorg'd out due to a missing L2 proof
|
|
23
|
+
* - Its first block is the immediate successor of the last proven block
|
|
24
|
+
*
|
|
25
|
+
* This class periodically hits the L2BlockSource.
|
|
26
|
+
* On start it will trigger the event for the last epoch ready to prove.
|
|
27
|
+
*/
|
|
28
|
+
export class EpochMonitor implements Traceable {
|
|
29
|
+
private runningPromise: RunningPromise;
|
|
30
|
+
private log = createLogger('prover-node:epoch-monitor');
|
|
31
|
+
public readonly tracer: Tracer;
|
|
32
|
+
|
|
33
|
+
private handler: EpochMonitorHandler | undefined;
|
|
34
|
+
private latestEpochNumber: bigint | undefined;
|
|
35
|
+
|
|
36
|
+
constructor(
|
|
37
|
+
private readonly l2BlockSource: L2BlockSource,
|
|
38
|
+
private readonly l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
|
|
39
|
+
private options: { pollingIntervalMs: number },
|
|
40
|
+
telemetry: TelemetryClient = getTelemetryClient(),
|
|
41
|
+
) {
|
|
42
|
+
this.tracer = telemetry.getTracer('EpochMonitor');
|
|
43
|
+
this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.options.pollingIntervalMs);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
public static async create(
|
|
47
|
+
l2BlockSource: L2BlockSource,
|
|
48
|
+
options: { pollingIntervalMs: number },
|
|
49
|
+
telemetry: TelemetryClient = getTelemetryClient(),
|
|
50
|
+
): Promise<EpochMonitor> {
|
|
51
|
+
const l1Constants = await l2BlockSource.getL1Constants();
|
|
52
|
+
return new EpochMonitor(l2BlockSource, l1Constants, options, telemetry);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
public start(handler: EpochMonitorHandler) {
|
|
56
|
+
this.handler = handler;
|
|
57
|
+
this.runningPromise.start();
|
|
58
|
+
this.log.info('Started EpochMonitor', this.options);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Exposed for testing */
|
|
62
|
+
public setHandler(handler: EpochMonitorHandler) {
|
|
63
|
+
this.handler = handler;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
public async stop() {
|
|
67
|
+
await this.runningPromise.stop();
|
|
68
|
+
this.log.info('Stopped EpochMonitor');
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
@trackSpan('EpochMonitor.work')
|
|
72
|
+
public async work() {
|
|
73
|
+
const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
|
|
74
|
+
if (epochToProve === undefined) {
|
|
75
|
+
this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
if (this.latestEpochNumber !== undefined && epochToProve <= this.latestEpochNumber) {
|
|
79
|
+
this.log.trace(`Epoch ${epochToProve} already processed`, { epochToProve, blockNumber, slotNumber });
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const isCompleted = await this.l2BlockSource.isEpochComplete(epochToProve);
|
|
84
|
+
if (!isCompleted) {
|
|
85
|
+
this.log.trace(`Epoch ${epochToProve} is not complete`, { epochToProve, blockNumber, slotNumber });
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
|
|
90
|
+
await this.handler?.handleEpochReadyToProve(epochToProve);
|
|
91
|
+
this.latestEpochNumber = epochToProve;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
private async getEpochNumberToProve() {
|
|
95
|
+
const lastBlockProven = await this.l2BlockSource.getProvenBlockNumber();
|
|
96
|
+
const firstBlockToProve = lastBlockProven + 1;
|
|
97
|
+
const firstBlockHeaderToProve = await this.l2BlockSource.getBlockHeader(firstBlockToProve);
|
|
98
|
+
if (!firstBlockHeaderToProve) {
|
|
99
|
+
return { epochToProve: undefined, blockNumber: firstBlockToProve };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const firstSlotOfEpochToProve = firstBlockHeaderToProve.getSlot();
|
|
103
|
+
const epochToProve = getEpochAtSlot(firstSlotOfEpochToProve, this.l1Constants);
|
|
104
|
+
return { epochToProve, blockNumber: firstBlockToProve, slotNumber: firstSlotOfEpochToProve };
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './epoch-monitor.js';
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config';
|
|
2
|
+
|
|
3
|
+
export type ProverCoordinationConfig = {
|
|
4
|
+
proverCoordinationNodeUrl: string | undefined;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
export const proverCoordinationConfigMappings: ConfigMappingsType<ProverCoordinationConfig> = {
|
|
8
|
+
proverCoordinationNodeUrl: {
|
|
9
|
+
env: 'PROVER_COORDINATION_NODE_URL',
|
|
10
|
+
description: 'The URL of the tx provider node',
|
|
11
|
+
parseEnv: (val: string) => val,
|
|
12
|
+
},
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export function getTxProviderConfigFromEnv(): ProverCoordinationConfig {
|
|
16
|
+
return getConfigFromMappings<ProverCoordinationConfig>(proverCoordinationConfigMappings);
|
|
17
|
+
}
|