@aztec/prover-node 0.0.1-commit.d431d1c → 0.0.1-commit.d939eb5aa
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/actions/download-epoch-proving-job.js +1 -1
- package/dest/actions/rerun-epoch-proving-job.d.ts +5 -3
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +4 -4
- package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
- package/dest/bin/run-failed-epoch.js +5 -2
- package/dest/config.d.ts +5 -8
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +16 -19
- package/dest/factory.d.ts +19 -13
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +39 -63
- package/dest/job/epoch-proving-job.d.ts +3 -2
- package/dest/job/epoch-proving-job.d.ts.map +1 -1
- package/dest/job/epoch-proving-job.js +40 -16
- package/dest/metrics.d.ts +11 -1
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +36 -3
- package/dest/monitors/epoch-monitor.d.ts +1 -1
- package/dest/monitors/epoch-monitor.d.ts.map +1 -1
- package/dest/monitors/epoch-monitor.js +5 -7
- package/dest/prover-node-publisher.d.ts +6 -5
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +6 -5
- package/dest/prover-node.d.ts +19 -9
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +17 -12
- package/dest/prover-publisher-factory.d.ts +7 -5
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +7 -5
- package/package.json +23 -22
- package/src/actions/download-epoch-proving-job.ts +1 -1
- package/src/actions/rerun-epoch-proving-job.ts +8 -4
- package/src/actions/upload-epoch-proof-failure.ts +1 -1
- package/src/bin/run-failed-epoch.ts +4 -1
- package/src/config.ts +23 -31
- package/src/factory.ts +67 -103
- package/src/job/epoch-proving-job.ts +62 -24
- package/src/metrics.ts +40 -2
- package/src/monitors/epoch-monitor.ts +3 -4
- package/src/prover-node-publisher.ts +8 -6
- package/src/prover-node.ts +20 -12
- package/src/prover-publisher-factory.ts +16 -10
package/src/factory.ts
CHANGED
|
@@ -1,34 +1,36 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import type { Archiver } from '@aztec/archiver';
|
|
2
|
+
import type { BlobClientInterface } from '@aztec/blob-client/client';
|
|
3
|
+
import { Blob } from '@aztec/blob-lib';
|
|
4
|
+
import type { EpochCacheInterface } from '@aztec/epoch-cache';
|
|
5
5
|
import { createEthereumChain } from '@aztec/ethereum/chain';
|
|
6
|
+
import { makeL1HttpTransport } from '@aztec/ethereum/client';
|
|
6
7
|
import { RollupContract } from '@aztec/ethereum/contracts';
|
|
7
8
|
import { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
|
|
8
9
|
import { PublisherManager } from '@aztec/ethereum/publisher-manager';
|
|
9
10
|
import { pick } from '@aztec/foundation/collection';
|
|
10
11
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
11
12
|
import { DateProvider } from '@aztec/foundation/timer';
|
|
12
|
-
import
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
createForwarderL1TxUtilsFromEthSigner,
|
|
17
|
-
createL1TxUtilsFromEthSignerWithStore,
|
|
18
|
-
} from '@aztec/node-lib/factories';
|
|
19
|
-
import { NodeRpcTxSource, createP2PClient } from '@aztec/p2p';
|
|
20
|
-
import { type ProverClientConfig, createProverClient } from '@aztec/prover-client';
|
|
13
|
+
import { KeystoreManager } from '@aztec/node-keystore';
|
|
14
|
+
import { createForwarderL1TxUtilsFromSigners, createL1TxUtilsFromSigners } from '@aztec/node-lib/factories';
|
|
15
|
+
import { type ProverClientConfig, type ProverClientUserConfig, createProverClient } from '@aztec/prover-client';
|
|
21
16
|
import { createAndStartProvingBroker } from '@aztec/prover-client/broker';
|
|
22
|
-
import
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
17
|
+
import {
|
|
18
|
+
type ProverPublisherConfig,
|
|
19
|
+
type ProverTxSenderConfig,
|
|
20
|
+
getPublisherConfigFromProverConfig,
|
|
21
|
+
} from '@aztec/sequencer-client';
|
|
22
|
+
import type {
|
|
23
|
+
ITxProvider,
|
|
24
|
+
ProverConfig,
|
|
25
|
+
ProvingJobBroker,
|
|
26
|
+
Service,
|
|
27
|
+
WorldStateSynchronizer,
|
|
28
|
+
} from '@aztec/stdlib/interfaces/server';
|
|
26
29
|
import { L1Metrics, type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
27
|
-
import { createWorldStateSynchronizer } from '@aztec/world-state';
|
|
28
30
|
|
|
29
|
-
import { createPublicClient
|
|
31
|
+
import { createPublicClient } from 'viem';
|
|
30
32
|
|
|
31
|
-
import
|
|
33
|
+
import type { SpecificProverNodeConfig } from './config.js';
|
|
32
34
|
import { EpochMonitor } from './monitors/epoch-monitor.js';
|
|
33
35
|
import { ProverNode } from './prover-node.js';
|
|
34
36
|
import { ProverPublisherFactory } from './prover-publisher-factory.js';
|
|
@@ -36,54 +38,42 @@ import { ProverPublisherFactory } from './prover-publisher-factory.js';
|
|
|
36
38
|
export type ProverNodeDeps = {
|
|
37
39
|
telemetry?: TelemetryClient;
|
|
38
40
|
log?: Logger;
|
|
39
|
-
|
|
40
|
-
archiver?: Archiver;
|
|
41
|
+
archiver: Archiver;
|
|
41
42
|
publisherFactory?: ProverPublisherFactory;
|
|
42
43
|
broker?: ProvingJobBroker;
|
|
43
44
|
l1TxUtils?: L1TxUtils;
|
|
44
45
|
dateProvider?: DateProvider;
|
|
46
|
+
worldStateSynchronizer: WorldStateSynchronizer;
|
|
47
|
+
p2pClient: { getTxProvider(): ITxProvider } & Partial<Service>;
|
|
48
|
+
epochCache: EpochCacheInterface;
|
|
49
|
+
blobClient: BlobClientInterface;
|
|
50
|
+
keyStoreManager?: KeystoreManager;
|
|
45
51
|
};
|
|
46
52
|
|
|
47
|
-
/** Creates a new prover node given a config
|
|
53
|
+
/** Creates a new prover node subsystem given a config and dependencies */
|
|
48
54
|
export async function createProverNode(
|
|
49
|
-
userConfig:
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
55
|
+
userConfig: SpecificProverNodeConfig &
|
|
56
|
+
ProverConfig &
|
|
57
|
+
ProverClientUserConfig &
|
|
58
|
+
ProverPublisherConfig &
|
|
59
|
+
ProverTxSenderConfig,
|
|
60
|
+
deps: ProverNodeDeps,
|
|
54
61
|
) {
|
|
55
62
|
const config = { ...userConfig };
|
|
56
63
|
const telemetry = deps.telemetry ?? getTelemetryClient();
|
|
57
64
|
const dateProvider = deps.dateProvider ?? new DateProvider();
|
|
58
|
-
const
|
|
59
|
-
const log = deps.log ?? createLogger('prover-node');
|
|
60
|
-
|
|
61
|
-
// Build a key store from file if given or from environment otherwise
|
|
62
|
-
let keyStoreManager: KeystoreManager | undefined;
|
|
63
|
-
const keyStoreProvided = config.keyStoreDirectory !== undefined && config.keyStoreDirectory.length > 0;
|
|
64
|
-
if (keyStoreProvided) {
|
|
65
|
-
const keyStores = loadKeystores(config.keyStoreDirectory!);
|
|
66
|
-
keyStoreManager = new KeystoreManager(mergeKeystores(keyStores));
|
|
67
|
-
} else {
|
|
68
|
-
const keyStore = createKeyStoreForProver(config);
|
|
69
|
-
if (keyStore) {
|
|
70
|
-
keyStoreManager = new KeystoreManager(keyStore);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
65
|
+
const log = deps.log ?? createLogger('prover');
|
|
73
66
|
|
|
74
|
-
|
|
67
|
+
const { p2pClient, archiver, keyStoreManager, worldStateSynchronizer } = deps;
|
|
75
68
|
|
|
76
69
|
// Extract the prover signers from the key store and verify that we have one.
|
|
70
|
+
await keyStoreManager?.validateSigners();
|
|
77
71
|
const proverSigners = keyStoreManager?.createProverSigners();
|
|
78
72
|
|
|
79
73
|
if (proverSigners === undefined) {
|
|
80
74
|
throw new Error('Failed to create prover key store configuration');
|
|
81
75
|
} else if (proverSigners.signers.length === 0) {
|
|
82
76
|
throw new Error('No prover signers found in the key store');
|
|
83
|
-
} else if (!keyStoreProvided) {
|
|
84
|
-
log.warn(
|
|
85
|
-
'KEY STORE CREATED FROM ENVIRONMENT, IT IS RECOMMENDED TO USE A FILE-BASED KEY STORE IN PRODUCTION ENVIRONMENTS',
|
|
86
|
-
);
|
|
87
77
|
}
|
|
88
78
|
|
|
89
79
|
log.info(`Creating prover with publishers ${proverSigners.signers.map(signer => signer.address.toString()).join()}`);
|
|
@@ -95,28 +85,7 @@ export async function createProverNode(
|
|
|
95
85
|
const proverId = proverSigners.id ?? proverIdInUserConfig ?? proverSigners.signers[0].address;
|
|
96
86
|
|
|
97
87
|
// Now create the prover client configuration from this.
|
|
98
|
-
const proverClientConfig: ProverClientConfig = {
|
|
99
|
-
...config,
|
|
100
|
-
proverId,
|
|
101
|
-
};
|
|
102
|
-
|
|
103
|
-
await trySnapshotSync(config, log);
|
|
104
|
-
|
|
105
|
-
const epochCache = await EpochCache.create(config.l1Contracts.rollupAddress, config);
|
|
106
|
-
|
|
107
|
-
const archiver =
|
|
108
|
-
deps.archiver ??
|
|
109
|
-
(await createArchiver(config, { blobClient, epochCache, telemetry, dateProvider }, { blockUntilSync: true }));
|
|
110
|
-
log.verbose(`Created archiver and synced to block ${await archiver.getBlockNumber()}`);
|
|
111
|
-
|
|
112
|
-
const worldStateConfig = { ...config, worldStateProvenBlocksOnly: false };
|
|
113
|
-
const worldStateSynchronizer = await createWorldStateSynchronizer(
|
|
114
|
-
worldStateConfig,
|
|
115
|
-
archiver,
|
|
116
|
-
options.prefilledPublicData,
|
|
117
|
-
telemetry,
|
|
118
|
-
);
|
|
119
|
-
await worldStateSynchronizer.start();
|
|
88
|
+
const proverClientConfig: ProverClientConfig = { ...config, proverId };
|
|
120
89
|
|
|
121
90
|
const broker = deps.broker ?? (await createAndStartProvingBroker(config, telemetry));
|
|
122
91
|
|
|
@@ -127,7 +96,7 @@ export async function createProverNode(
|
|
|
127
96
|
|
|
128
97
|
const publicClient = createPublicClient({
|
|
129
98
|
chain: chain.chainInfo,
|
|
130
|
-
transport:
|
|
99
|
+
transport: makeL1HttpTransport(config.l1RpcUrls, { timeout: config.l1HttpTimeoutMS }),
|
|
131
100
|
pollingInterval: config.viemPollingIntervalMS,
|
|
132
101
|
});
|
|
133
102
|
|
|
@@ -135,55 +104,45 @@ export async function createProverNode(
|
|
|
135
104
|
|
|
136
105
|
const l1TxUtils = deps.l1TxUtils
|
|
137
106
|
? [deps.l1TxUtils]
|
|
138
|
-
: config.
|
|
139
|
-
? await
|
|
107
|
+
: config.proverPublisherForwarderAddress
|
|
108
|
+
? await createForwarderL1TxUtilsFromSigners(
|
|
140
109
|
publicClient,
|
|
141
110
|
proverSigners.signers,
|
|
142
|
-
config.
|
|
111
|
+
config.proverPublisherForwarderAddress,
|
|
143
112
|
{ ...config, scope: 'prover' },
|
|
144
|
-
{ telemetry, logger: log.createChild('l1-tx-utils'), dateProvider },
|
|
113
|
+
{ telemetry, logger: log.createChild('l1-tx-utils'), dateProvider, kzg: Blob.getViemKzgInstance() },
|
|
145
114
|
)
|
|
146
|
-
: await
|
|
115
|
+
: await createL1TxUtilsFromSigners(
|
|
147
116
|
publicClient,
|
|
148
117
|
proverSigners.signers,
|
|
149
118
|
{ ...config, scope: 'prover' },
|
|
150
119
|
{ telemetry, logger: log.createChild('l1-tx-utils'), dateProvider },
|
|
151
120
|
);
|
|
152
121
|
|
|
122
|
+
// Create a funder L1TxUtils from the keystore funding account (if configured)
|
|
123
|
+
const fundingSigner = keyStoreManager?.createFundingSigner();
|
|
124
|
+
let funderL1TxUtils: L1TxUtils | undefined;
|
|
125
|
+
if (fundingSigner) {
|
|
126
|
+
const [funder] = await createL1TxUtilsFromSigners(
|
|
127
|
+
publicClient,
|
|
128
|
+
[fundingSigner],
|
|
129
|
+
{ ...config, scope: 'prover' },
|
|
130
|
+
{ telemetry, logger: log.createChild('l1-tx-utils:funder'), dateProvider },
|
|
131
|
+
);
|
|
132
|
+
funderL1TxUtils = funder;
|
|
133
|
+
}
|
|
134
|
+
|
|
153
135
|
const publisherFactory =
|
|
154
136
|
deps.publisherFactory ??
|
|
155
137
|
new ProverPublisherFactory(config, {
|
|
156
138
|
rollupContract,
|
|
157
|
-
publisherManager: new PublisherManager(l1TxUtils, config),
|
|
139
|
+
publisherManager: new PublisherManager(l1TxUtils, getPublisherConfigFromProverConfig(config), {
|
|
140
|
+
bindings: log.getBindings(),
|
|
141
|
+
funder: funderL1TxUtils,
|
|
142
|
+
}),
|
|
158
143
|
telemetry,
|
|
159
144
|
});
|
|
160
145
|
|
|
161
|
-
const proofVerifier = new QueuedIVCVerifier(
|
|
162
|
-
config,
|
|
163
|
-
config.realProofs || config.debugForceTxProofVerification
|
|
164
|
-
? await BBCircuitVerifier.new(config)
|
|
165
|
-
: new TestCircuitVerifier(config.proverTestVerificationDelayMs),
|
|
166
|
-
);
|
|
167
|
-
|
|
168
|
-
const p2pClient = await createP2PClient(
|
|
169
|
-
P2PClientType.Prover,
|
|
170
|
-
config,
|
|
171
|
-
archiver,
|
|
172
|
-
proofVerifier,
|
|
173
|
-
worldStateSynchronizer,
|
|
174
|
-
epochCache,
|
|
175
|
-
getPackageVersion() ?? '',
|
|
176
|
-
dateProvider,
|
|
177
|
-
telemetry,
|
|
178
|
-
{
|
|
179
|
-
txCollectionNodeSources: deps.aztecNodeTxProvider
|
|
180
|
-
? [new NodeRpcTxSource(deps.aztecNodeTxProvider, 'TestNode')]
|
|
181
|
-
: [],
|
|
182
|
-
},
|
|
183
|
-
);
|
|
184
|
-
|
|
185
|
-
await p2pClient.start();
|
|
186
|
-
|
|
187
146
|
const proverNodeConfig = {
|
|
188
147
|
...pick(
|
|
189
148
|
config,
|
|
@@ -214,6 +173,9 @@ export async function createProverNode(
|
|
|
214
173
|
l1TxUtils.map(utils => utils.getSenderAddress()),
|
|
215
174
|
);
|
|
216
175
|
|
|
176
|
+
// Extract the shared delayer from the first L1TxUtils instance (all instances share the same delayer)
|
|
177
|
+
const delayer = l1TxUtils[0]?.delayer;
|
|
178
|
+
|
|
217
179
|
return new ProverNode(
|
|
218
180
|
prover,
|
|
219
181
|
publisherFactory,
|
|
@@ -227,5 +189,7 @@ export async function createProverNode(
|
|
|
227
189
|
l1Metrics,
|
|
228
190
|
proverNodeConfig,
|
|
229
191
|
telemetry,
|
|
192
|
+
delayer,
|
|
193
|
+
dateProvider,
|
|
230
194
|
);
|
|
231
195
|
}
|
|
@@ -3,15 +3,16 @@ import { asyncPool } from '@aztec/foundation/async-pool';
|
|
|
3
3
|
import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
|
|
4
4
|
import { padArrayEnd } from '@aztec/foundation/collection';
|
|
5
5
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
6
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
6
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
7
7
|
import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
8
8
|
import { Timer } from '@aztec/foundation/timer';
|
|
9
|
+
import { AVM_MAX_CONCURRENT_SIMULATIONS } from '@aztec/native';
|
|
9
10
|
import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
|
|
10
11
|
import { protocolContractsHash } from '@aztec/protocol-contracts';
|
|
11
12
|
import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
|
|
12
13
|
import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
|
|
13
14
|
import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
|
|
14
|
-
import type {
|
|
15
|
+
import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
|
|
15
16
|
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
16
17
|
import {
|
|
17
18
|
type EpochProver,
|
|
@@ -43,7 +44,7 @@ export type EpochProvingJobOptions = {
|
|
|
43
44
|
*/
|
|
44
45
|
export class EpochProvingJob implements Traceable {
|
|
45
46
|
private state: EpochProvingJobState = 'initialized';
|
|
46
|
-
private log
|
|
47
|
+
private log: Logger;
|
|
47
48
|
private uuid: string;
|
|
48
49
|
|
|
49
50
|
private runPromise: Promise<void> | undefined;
|
|
@@ -62,9 +63,14 @@ export class EpochProvingJob implements Traceable {
|
|
|
62
63
|
private metrics: ProverNodeJobMetrics,
|
|
63
64
|
private deadline: Date | undefined,
|
|
64
65
|
private config: EpochProvingJobOptions,
|
|
66
|
+
bindings?: LoggerBindings,
|
|
65
67
|
) {
|
|
66
68
|
validateEpochProvingJobData(data);
|
|
67
69
|
this.uuid = crypto.randomUUID();
|
|
70
|
+
this.log = createLogger('prover-node:epoch-proving-job', {
|
|
71
|
+
...bindings,
|
|
72
|
+
instanceId: `epoch-${data.epochNumber}`,
|
|
73
|
+
});
|
|
68
74
|
this.tracer = metrics.tracer;
|
|
69
75
|
}
|
|
70
76
|
|
|
@@ -143,19 +149,32 @@ export class EpochProvingJob implements Traceable {
|
|
|
143
149
|
this.runPromise = promise;
|
|
144
150
|
|
|
145
151
|
try {
|
|
152
|
+
const blobTimer = new Timer();
|
|
146
153
|
const blobFieldsPerCheckpoint = this.checkpoints.map(checkpoint => checkpoint.toBlobFields());
|
|
147
154
|
const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
|
|
155
|
+
this.metrics.recordBlobProcessing(blobTimer.ms());
|
|
148
156
|
|
|
149
157
|
this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
|
|
158
|
+
const chonkTimer = new Timer();
|
|
150
159
|
await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
|
|
160
|
+
this.metrics.recordChonkVerifier(chonkTimer.ms());
|
|
151
161
|
|
|
152
162
|
// Everything in the epoch should have the same chainId and version.
|
|
153
163
|
const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
|
|
154
164
|
|
|
155
165
|
const previousBlockHeaders = this.gatherPreviousBlockHeaders();
|
|
156
166
|
|
|
157
|
-
|
|
167
|
+
const allCheckpointsTimer = new Timer();
|
|
168
|
+
|
|
169
|
+
const parallelism = this.config.parallelBlockLimit
|
|
170
|
+
? this.config.parallelBlockLimit
|
|
171
|
+
: AVM_MAX_CONCURRENT_SIMULATIONS > 0
|
|
172
|
+
? AVM_MAX_CONCURRENT_SIMULATIONS
|
|
173
|
+
: this.checkpoints.length;
|
|
174
|
+
|
|
175
|
+
await asyncPool(parallelism, this.checkpoints, async checkpoint => {
|
|
158
176
|
this.checkState();
|
|
177
|
+
const checkpointTimer = new Timer();
|
|
159
178
|
|
|
160
179
|
const checkpointIndex = checkpoint.number - fromCheckpoint;
|
|
161
180
|
const checkpointConstants = CheckpointConstantData.from({
|
|
@@ -188,7 +207,9 @@ export class EpochProvingJob implements Traceable {
|
|
|
188
207
|
previousHeader,
|
|
189
208
|
);
|
|
190
209
|
|
|
191
|
-
for (
|
|
210
|
+
for (let blockIndex = 0; blockIndex < checkpoint.blocks.length; blockIndex++) {
|
|
211
|
+
const blockTimer = new Timer();
|
|
212
|
+
const block = checkpoint.blocks[blockIndex];
|
|
192
213
|
const globalVariables = block.header.globalVariables;
|
|
193
214
|
const txs = this.getTxs(block);
|
|
194
215
|
|
|
@@ -206,8 +227,12 @@ export class EpochProvingJob implements Traceable {
|
|
|
206
227
|
// Start block proving
|
|
207
228
|
await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
|
|
208
229
|
|
|
209
|
-
// Process public fns
|
|
210
|
-
|
|
230
|
+
// Process public fns. L1 to L2 messages are only inserted for the first block of a checkpoint,
|
|
231
|
+
// as the fork for subsequent blocks already includes them from the previous block's synced state.
|
|
232
|
+
const db = await this.createFork(
|
|
233
|
+
BlockNumber(block.number - 1),
|
|
234
|
+
blockIndex === 0 ? l1ToL2Messages : undefined,
|
|
235
|
+
);
|
|
211
236
|
const config = PublicSimulatorConfig.from({
|
|
212
237
|
proverId: this.prover.getProverId().toField(),
|
|
213
238
|
skipFeeEnforcement: false,
|
|
@@ -229,8 +254,11 @@ export class EpochProvingJob implements Traceable {
|
|
|
229
254
|
// Mark block as completed to pad it
|
|
230
255
|
const expectedBlockHeader = block.header;
|
|
231
256
|
await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
|
|
257
|
+
this.metrics.recordBlockProcessing(blockTimer.ms());
|
|
232
258
|
}
|
|
259
|
+
this.metrics.recordCheckpointProcessing(checkpointTimer.ms());
|
|
233
260
|
});
|
|
261
|
+
this.metrics.recordAllCheckpointsProcessing(allCheckpointsTimer.ms());
|
|
234
262
|
|
|
235
263
|
const executionTime = timer.ms();
|
|
236
264
|
|
|
@@ -290,22 +318,29 @@ export class EpochProvingJob implements Traceable {
|
|
|
290
318
|
}
|
|
291
319
|
|
|
292
320
|
/**
|
|
293
|
-
* Create a new db fork for tx processing, inserting
|
|
321
|
+
* Create a new db fork for tx processing, optionally inserting L1 to L2 messages.
|
|
322
|
+
* L1 to L2 messages should only be inserted for the first block in a checkpoint,
|
|
323
|
+
* as subsequent blocks' synced state already includes them.
|
|
294
324
|
* REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
|
|
295
325
|
*/
|
|
296
|
-
private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[]) {
|
|
326
|
+
private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[] | undefined) {
|
|
327
|
+
this.log.verbose(`Creating fork at ${blockNumber}`, { blockNumber });
|
|
297
328
|
const db = await this.dbProvider.fork(blockNumber);
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
329
|
+
|
|
330
|
+
if (l1ToL2Messages !== undefined) {
|
|
331
|
+
this.log.verbose(`Inserting ${l1ToL2Messages.length} L1 to L2 messages in fork`, {
|
|
332
|
+
blockNumber,
|
|
333
|
+
l1ToL2Messages: l1ToL2Messages.map(m => m.toString()),
|
|
334
|
+
});
|
|
335
|
+
const l1ToL2MessagesPadded = padArrayEnd<Fr, number>(
|
|
336
|
+
l1ToL2Messages,
|
|
337
|
+
Fr.ZERO,
|
|
338
|
+
NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
|
|
339
|
+
'Too many L1 to L2 messages',
|
|
340
|
+
);
|
|
341
|
+
await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
|
|
342
|
+
}
|
|
343
|
+
|
|
309
344
|
return db;
|
|
310
345
|
}
|
|
311
346
|
|
|
@@ -362,11 +397,14 @@ export class EpochProvingJob implements Traceable {
|
|
|
362
397
|
const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
|
|
363
398
|
this.epochCheckPromise = new RunningPromise(
|
|
364
399
|
async () => {
|
|
365
|
-
const
|
|
366
|
-
const blockHashes = await Promise.all(
|
|
400
|
+
const blockHeaders = await l2BlockSource.getCheckpointedBlockHeadersForEpoch(this.epochNumber);
|
|
401
|
+
const blockHashes = await Promise.all(blockHeaders.map(header => header.hash()));
|
|
367
402
|
const thisBlocks = this.checkpoints.flatMap(checkpoint => checkpoint.blocks);
|
|
368
403
|
const thisBlockHashes = await Promise.all(thisBlocks.map(block => block.hash()));
|
|
369
|
-
if (
|
|
404
|
+
if (
|
|
405
|
+
blockHeaders.length !== thisBlocks.length ||
|
|
406
|
+
!blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))
|
|
407
|
+
) {
|
|
370
408
|
this.log.warn('Epoch blocks changed underfoot', {
|
|
371
409
|
uuid: this.uuid,
|
|
372
410
|
epochNumber: this.epochNumber,
|
|
@@ -388,7 +426,7 @@ export class EpochProvingJob implements Traceable {
|
|
|
388
426
|
return [this.data.previousBlockHeader, ...lastBlocks.map(block => block.header).slice(0, -1)];
|
|
389
427
|
}
|
|
390
428
|
|
|
391
|
-
private getTxs(block:
|
|
429
|
+
private getTxs(block: L2Block): Tx[] {
|
|
392
430
|
return block.body.txEffects.map(txEffect => this.txs.get(txEffect.txHash.toString())!);
|
|
393
431
|
}
|
|
394
432
|
|
package/src/metrics.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
type TelemetryClient,
|
|
14
14
|
type Tracer,
|
|
15
15
|
type UpDownCounter,
|
|
16
|
+
createUpDownCounterWithDefault,
|
|
16
17
|
} from '@aztec/telemetry-client';
|
|
17
18
|
|
|
18
19
|
import { formatEther, formatUnits } from 'viem';
|
|
@@ -24,6 +25,12 @@ export class ProverNodeJobMetrics {
|
|
|
24
25
|
provingJobBlocks: Gauge;
|
|
25
26
|
provingJobTransactions: Gauge;
|
|
26
27
|
|
|
28
|
+
private blobProcessingDuration: Gauge;
|
|
29
|
+
private chonkVerifierDuration: Gauge;
|
|
30
|
+
private blockProcessingDuration: Histogram;
|
|
31
|
+
private checkpointProcessingDuration: Histogram;
|
|
32
|
+
private allCheckpointsProcessingDuration: Gauge;
|
|
33
|
+
|
|
27
34
|
constructor(
|
|
28
35
|
private meter: Meter,
|
|
29
36
|
public readonly tracer: Tracer,
|
|
@@ -34,6 +41,14 @@ export class ProverNodeJobMetrics {
|
|
|
34
41
|
this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
|
|
35
42
|
this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
|
|
36
43
|
this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS);
|
|
44
|
+
|
|
45
|
+
this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
|
|
46
|
+
this.chonkVerifierDuration = this.meter.createGauge(Metrics.PROVER_NODE_CHONK_VERIFIER_LAST_DURATION);
|
|
47
|
+
this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
|
|
48
|
+
this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
|
|
49
|
+
this.allCheckpointsProcessingDuration = this.meter.createGauge(
|
|
50
|
+
Metrics.PROVER_NODE_ALL_CHECKPOINTS_PROCESSING_LAST_DURATION,
|
|
51
|
+
);
|
|
37
52
|
}
|
|
38
53
|
|
|
39
54
|
public recordProvingJob(
|
|
@@ -49,6 +64,26 @@ export class ProverNodeJobMetrics {
|
|
|
49
64
|
this.provingJobBlocks.record(Math.floor(numBlocks));
|
|
50
65
|
this.provingJobTransactions.record(Math.floor(numTxs));
|
|
51
66
|
}
|
|
67
|
+
|
|
68
|
+
public recordBlobProcessing(durationMs: number) {
|
|
69
|
+
this.blobProcessingDuration.record(Math.ceil(durationMs));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
public recordChonkVerifier(durationMs: number) {
|
|
73
|
+
this.chonkVerifierDuration.record(Math.ceil(durationMs));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
public recordBlockProcessing(durationMs: number) {
|
|
77
|
+
this.blockProcessingDuration.record(Math.ceil(durationMs));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
public recordCheckpointProcessing(durationMs: number) {
|
|
81
|
+
this.checkpointProcessingDuration.record(Math.ceil(durationMs));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
public recordAllCheckpointsProcessing(durationMs: number) {
|
|
85
|
+
this.allCheckpointsProcessingDuration.record(Math.ceil(durationMs));
|
|
86
|
+
}
|
|
52
87
|
}
|
|
53
88
|
|
|
54
89
|
export class ProverNodeRewardsMetrics {
|
|
@@ -65,7 +100,7 @@ export class ProverNodeRewardsMetrics {
|
|
|
65
100
|
) {
|
|
66
101
|
this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH);
|
|
67
102
|
|
|
68
|
-
this.accumulatedRewards = this.meter
|
|
103
|
+
this.accumulatedRewards = createUpDownCounterWithDefault(this.meter, Metrics.PROVER_NODE_REWARDS_TOTAL);
|
|
69
104
|
}
|
|
70
105
|
|
|
71
106
|
public async start() {
|
|
@@ -128,7 +163,10 @@ export class ProverNodePublisherMetrics {
|
|
|
128
163
|
|
|
129
164
|
this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE);
|
|
130
165
|
|
|
131
|
-
this.txCount = this.meter
|
|
166
|
+
this.txCount = createUpDownCounterWithDefault(this.meter, Metrics.L1_PUBLISHER_TX_COUNT, {
|
|
167
|
+
[Attributes.L1_TX_TYPE]: ['submitProof'],
|
|
168
|
+
[Attributes.OK]: [true, false],
|
|
169
|
+
});
|
|
132
170
|
|
|
133
171
|
this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION);
|
|
134
172
|
|
|
@@ -69,9 +69,8 @@ export class EpochMonitor implements Traceable {
|
|
|
69
69
|
|
|
70
70
|
public async work() {
|
|
71
71
|
const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
|
|
72
|
-
this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
|
|
73
72
|
if (epochToProve === undefined) {
|
|
74
|
-
this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
|
|
73
|
+
this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { epochToProve, blockNumber, slotNumber });
|
|
75
74
|
return;
|
|
76
75
|
}
|
|
77
76
|
if (this.latestEpochNumber !== undefined && epochToProve <= this.latestEpochNumber) {
|
|
@@ -86,11 +85,11 @@ export class EpochMonitor implements Traceable {
|
|
|
86
85
|
}
|
|
87
86
|
|
|
88
87
|
if (this.options.provingDelayMs) {
|
|
89
|
-
this.log.
|
|
88
|
+
this.log.warn(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
|
|
90
89
|
await sleep(this.options.provingDelayMs);
|
|
91
90
|
}
|
|
92
91
|
|
|
93
|
-
this.log.
|
|
92
|
+
this.log.verbose(`Epoch ${epochToProve} is ready to be proven`);
|
|
94
93
|
if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
|
|
95
94
|
this.latestEpochNumber = epochToProve;
|
|
96
95
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { BatchedBlob, getEthBlobEvaluationInputs } from '@aztec/blob-lib';
|
|
2
|
-
import {
|
|
2
|
+
import { MAX_CHECKPOINTS_PER_EPOCH } from '@aztec/constants';
|
|
3
3
|
import type { RollupContract, ViemCommitteeAttestation } from '@aztec/ethereum/contracts';
|
|
4
4
|
import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
|
|
5
5
|
import { makeTuple } from '@aztec/foundation/array';
|
|
@@ -7,7 +7,7 @@ import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
|
|
|
7
7
|
import { areArraysEqual } from '@aztec/foundation/collection';
|
|
8
8
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
9
9
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
10
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
10
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
11
11
|
import type { Tuple } from '@aztec/foundation/serialize';
|
|
12
12
|
import { Timer } from '@aztec/foundation/timer';
|
|
13
13
|
import { RollupAbi } from '@aztec/l1-artifacts';
|
|
@@ -31,7 +31,7 @@ export type L1SubmitEpochProofArgs = {
|
|
|
31
31
|
endTimestamp: Fr;
|
|
32
32
|
outHash: Fr;
|
|
33
33
|
proverId: Fr;
|
|
34
|
-
fees: Tuple<FeeRecipient, typeof
|
|
34
|
+
fees: Tuple<FeeRecipient, typeof MAX_CHECKPOINTS_PER_EPOCH>;
|
|
35
35
|
proof: Proof;
|
|
36
36
|
};
|
|
37
37
|
|
|
@@ -39,7 +39,7 @@ export class ProverNodePublisher {
|
|
|
39
39
|
private interrupted = false;
|
|
40
40
|
private metrics: ProverNodePublisherMetrics;
|
|
41
41
|
|
|
42
|
-
protected log
|
|
42
|
+
protected log: Logger;
|
|
43
43
|
|
|
44
44
|
protected rollupContract: RollupContract;
|
|
45
45
|
|
|
@@ -52,10 +52,12 @@ export class ProverNodePublisher {
|
|
|
52
52
|
l1TxUtils: L1TxUtils;
|
|
53
53
|
telemetry?: TelemetryClient;
|
|
54
54
|
},
|
|
55
|
+
bindings?: LoggerBindings,
|
|
55
56
|
) {
|
|
56
57
|
const telemetry = deps.telemetry ?? getTelemetryClient();
|
|
57
58
|
|
|
58
59
|
this.metrics = new ProverNodePublisherMetrics(telemetry, 'ProverNode');
|
|
60
|
+
this.log = createLogger('prover-node:l1-tx-publisher', bindings);
|
|
59
61
|
|
|
60
62
|
this.rollupContract = deps.rollupContract;
|
|
61
63
|
this.l1TxUtils = deps.l1TxUtils;
|
|
@@ -166,7 +168,7 @@ export class ProverNodePublisher {
|
|
|
166
168
|
// toCheckpoint can't be greater than pending
|
|
167
169
|
if (toCheckpoint > pending) {
|
|
168
170
|
throw new Error(
|
|
169
|
-
`Cannot submit epoch proof for ${fromCheckpoint}-${toCheckpoint} as
|
|
171
|
+
`Cannot submit epoch proof for ${fromCheckpoint}-${toCheckpoint} as proposed checkpoint is ${pending}`,
|
|
170
172
|
);
|
|
171
173
|
}
|
|
172
174
|
|
|
@@ -269,7 +271,7 @@ export class ProverNodePublisher {
|
|
|
269
271
|
outHash: args.publicInputs.outHash.toString(),
|
|
270
272
|
proverId: EthAddress.fromField(args.publicInputs.constants.proverId).toString(),
|
|
271
273
|
} /*_args*/,
|
|
272
|
-
makeTuple(
|
|
274
|
+
makeTuple(MAX_CHECKPOINTS_PER_EPOCH * 2, i =>
|
|
273
275
|
i % 2 === 0
|
|
274
276
|
? args.publicInputs.fees[i / 2].recipient.toField().toString()
|
|
275
277
|
: args.publicInputs.fees[(i - 1) / 2].value.toString(),
|