@aztec/prover-client 3.0.3 → 3.9.9-nightly.20260312
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/light/index.d.ts +2 -0
- package/dest/light/index.d.ts.map +1 -0
- package/dest/light/index.js +1 -0
- package/dest/light/lightweight_checkpoint_builder.d.ts +34 -14
- package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -1
- package/dest/light/lightweight_checkpoint_builder.js +124 -27
- package/dest/mocks/fixtures.d.ts +1 -1
- package/dest/mocks/fixtures.d.ts.map +1 -1
- package/dest/mocks/fixtures.js +2 -1
- package/dest/mocks/test_context.d.ts +4 -2
- package/dest/mocks/test_context.d.ts.map +1 -1
- package/dest/mocks/test_context.js +22 -5
- package/dest/orchestrator/block-building-helpers.d.ts +5 -5
- package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
- package/dest/orchestrator/block-building-helpers.js +4 -4
- package/dest/orchestrator/block-proving-state.d.ts +4 -1
- package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/block-proving-state.js +7 -0
- package/dest/orchestrator/checkpoint-proving-state.d.ts +17 -4
- package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/checkpoint-proving-state.js +37 -4
- package/dest/orchestrator/epoch-proving-state.d.ts +7 -6
- package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/epoch-proving-state.js +37 -1
- package/dest/orchestrator/orchestrator.d.ts +21 -4
- package/dest/orchestrator/orchestrator.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator.js +559 -154
- package/dest/orchestrator/orchestrator_metrics.d.ts +1 -3
- package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator_metrics.js +2 -15
- package/dest/orchestrator/tx-proving-state.d.ts +5 -4
- package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/tx-proving-state.js +6 -6
- package/dest/prover-client/factory.d.ts +3 -3
- package/dest/prover-client/factory.d.ts.map +1 -1
- package/dest/prover-client/prover-client.d.ts +6 -6
- package/dest/prover-client/prover-client.d.ts.map +1 -1
- package/dest/prover-client/prover-client.js +15 -10
- package/dest/proving_broker/broker_prover_facade.d.ts +7 -5
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
- package/dest/proving_broker/broker_prover_facade.js +8 -15
- package/dest/proving_broker/config.d.ts +13 -1
- package/dest/proving_broker/config.d.ts.map +1 -1
- package/dest/proving_broker/config.js +19 -2
- package/dest/proving_broker/proof_store/factory.d.ts +2 -5
- package/dest/proving_broker/proof_store/factory.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/factory.js +7 -30
- package/dest/proving_broker/proof_store/file_store_proof_store.d.ts +18 -0
- package/dest/proving_broker/proof_store/file_store_proof_store.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/file_store_proof_store.js +60 -0
- package/dest/proving_broker/proof_store/index.d.ts +2 -2
- package/dest/proving_broker/proof_store/index.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/index.js +1 -1
- package/dest/proving_broker/proving_agent.d.ts +5 -9
- package/dest/proving_broker/proving_agent.d.ts.map +1 -1
- package/dest/proving_broker/proving_agent.js +4 -19
- package/dest/proving_broker/proving_broker.d.ts +7 -4
- package/dest/proving_broker/proving_broker.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.js +40 -14
- package/dest/proving_broker/proving_broker_database/persisted.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.js +389 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts +3 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.js +22 -35
- package/dest/proving_broker/proving_job_controller.d.ts +4 -3
- package/dest/proving_broker/proving_job_controller.d.ts.map +1 -1
- package/dest/proving_broker/proving_job_controller.js +7 -5
- package/dest/proving_broker/rpc.d.ts +4 -2
- package/dest/proving_broker/rpc.d.ts.map +1 -1
- package/dest/proving_broker/rpc.js +8 -0
- package/dest/test/mock_proof_store.d.ts +3 -3
- package/dest/test/mock_proof_store.d.ts.map +1 -1
- package/dest/test/mock_prover.d.ts +5 -5
- package/dest/test/mock_prover.d.ts.map +1 -1
- package/dest/test/mock_prover.js +4 -4
- package/package.json +17 -18
- package/src/light/index.ts +1 -0
- package/src/light/lightweight_checkpoint_builder.ts +188 -34
- package/src/mocks/fixtures.ts +2 -1
- package/src/mocks/test_context.ts +17 -4
- package/src/orchestrator/block-building-helpers.ts +4 -4
- package/src/orchestrator/block-proving-state.ts +9 -0
- package/src/orchestrator/checkpoint-proving-state.ts +51 -5
- package/src/orchestrator/epoch-proving-state.ts +59 -9
- package/src/orchestrator/orchestrator.ts +160 -138
- package/src/orchestrator/orchestrator_metrics.ts +2 -25
- package/src/orchestrator/tx-proving-state.ts +8 -11
- package/src/prover-client/factory.ts +6 -2
- package/src/prover-client/prover-client.ts +33 -24
- package/src/proving_broker/broker_prover_facade.ts +18 -20
- package/src/proving_broker/config.ts +22 -0
- package/src/proving_broker/proof_store/factory.ts +10 -32
- package/src/proving_broker/proof_store/file_store_proof_store.ts +78 -0
- package/src/proving_broker/proof_store/index.ts +1 -1
- package/src/proving_broker/proving_agent.ts +6 -19
- package/src/proving_broker/proving_broker.ts +41 -11
- package/src/proving_broker/proving_broker_database/persisted.ts +15 -1
- package/src/proving_broker/proving_broker_instrumentation.ts +23 -35
- package/src/proving_broker/proving_job_controller.ts +10 -5
- package/src/proving_broker/rpc.ts +14 -0
- package/src/test/mock_prover.ts +2 -14
- package/dest/block-factory/index.d.ts +0 -2
- package/dest/block-factory/index.d.ts.map +0 -1
- package/dest/block-factory/index.js +0 -1
- package/dest/block-factory/light.d.ts +0 -38
- package/dest/block-factory/light.d.ts.map +0 -1
- package/dest/block-factory/light.js +0 -108
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +0 -14
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +0 -1
- package/dest/proving_broker/proof_store/gcs_proof_store.js +0 -52
- package/dest/proving_broker/proving_agent_instrumentation.d.ts +0 -8
- package/dest/proving_broker/proving_agent_instrumentation.d.ts.map +0 -1
- package/dest/proving_broker/proving_agent_instrumentation.js +0 -16
- package/src/block-factory/index.ts +0 -1
- package/src/block-factory/light.ts +0 -137
- package/src/proving_broker/proof_store/gcs_proof_store.ts +0 -76
- package/src/proving_broker/proving_agent_instrumentation.ts +0 -21
|
@@ -1,41 +1,18 @@
|
|
|
1
|
-
import {
|
|
2
|
-
type Histogram,
|
|
3
|
-
Metrics,
|
|
4
|
-
type TelemetryClient,
|
|
5
|
-
type Tracer,
|
|
6
|
-
type UpDownCounter,
|
|
7
|
-
ValueType,
|
|
8
|
-
} from '@aztec/telemetry-client';
|
|
1
|
+
import { type Histogram, Metrics, type TelemetryClient, type Tracer } from '@aztec/telemetry-client';
|
|
9
2
|
|
|
10
3
|
export class ProvingOrchestratorMetrics {
|
|
11
4
|
public readonly tracer: Tracer;
|
|
12
5
|
|
|
13
6
|
private baseRollupInputsDuration: Histogram;
|
|
14
|
-
private avmFallbackCount: UpDownCounter;
|
|
15
7
|
|
|
16
8
|
constructor(client: TelemetryClient, name = 'ProvingOrchestrator') {
|
|
17
9
|
this.tracer = client.getTracer(name);
|
|
18
10
|
const meter = client.getMeter(name);
|
|
19
11
|
|
|
20
|
-
this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION
|
|
21
|
-
unit: 'ms',
|
|
22
|
-
description: 'Duration to build base rollup inputs',
|
|
23
|
-
valueType: ValueType.INT,
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
this.avmFallbackCount = meter.createUpDownCounter(Metrics.PROVING_ORCHESTRATOR_AVM_FALLBACK_COUNT, {
|
|
27
|
-
description: 'How many times the AVM fallback was used',
|
|
28
|
-
valueType: ValueType.INT,
|
|
29
|
-
});
|
|
30
|
-
|
|
31
|
-
this.avmFallbackCount.add(0);
|
|
12
|
+
this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION);
|
|
32
13
|
}
|
|
33
14
|
|
|
34
15
|
recordBaseRollupInputs(durationMs: number) {
|
|
35
16
|
this.baseRollupInputsDuration.record(Math.ceil(durationMs));
|
|
36
17
|
}
|
|
37
|
-
|
|
38
|
-
incAvmFallback() {
|
|
39
|
-
this.avmFallbackCount.add(1);
|
|
40
|
-
}
|
|
41
18
|
}
|
|
@@ -2,8 +2,8 @@ import { AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED, NESTED_RECURSIVE_ROLLUP_HONK_PROO
|
|
|
2
2
|
import type { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
3
|
import { getVkData } from '@aztec/noir-protocol-circuits-types/server/vks';
|
|
4
4
|
import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
|
|
5
|
-
import type {
|
|
6
|
-
import { ProofData, ProofDataForFixedVk } from '@aztec/stdlib/proofs';
|
|
5
|
+
import type { PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
|
|
6
|
+
import { ProofData, ProofDataForFixedVk, RecursiveProof } from '@aztec/stdlib/proofs';
|
|
7
7
|
import {
|
|
8
8
|
type BaseRollupHints,
|
|
9
9
|
PrivateBaseRollupHints,
|
|
@@ -32,7 +32,7 @@ export class TxProvingState {
|
|
|
32
32
|
PublicChonkVerifierPublicInputs,
|
|
33
33
|
typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH
|
|
34
34
|
>;
|
|
35
|
-
private
|
|
35
|
+
private avmProof?: RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>;
|
|
36
36
|
|
|
37
37
|
constructor(
|
|
38
38
|
public readonly processedTx: ProcessedTx,
|
|
@@ -46,7 +46,7 @@ export class TxProvingState {
|
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
public ready() {
|
|
49
|
-
return !this.requireAvmProof || (!!this.
|
|
49
|
+
return !this.requireAvmProof || (!!this.avmProof && !!this.publicChonkVerifier);
|
|
50
50
|
}
|
|
51
51
|
|
|
52
52
|
public getAvmInputs(): AvmCircuitInputs {
|
|
@@ -80,8 +80,8 @@ export class TxProvingState {
|
|
|
80
80
|
this.publicChonkVerifier = publicChonkVerifierProofAndVk;
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
public setAvmProof(
|
|
84
|
-
this.
|
|
83
|
+
public setAvmProof(avmProof: RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>) {
|
|
84
|
+
this.avmProof = avmProof;
|
|
85
85
|
}
|
|
86
86
|
|
|
87
87
|
#getPrivateBaseInputs() {
|
|
@@ -105,7 +105,7 @@ export class TxProvingState {
|
|
|
105
105
|
if (!this.publicChonkVerifier) {
|
|
106
106
|
throw new Error('Tx not ready for proving base rollup: public chonk verifier proof undefined');
|
|
107
107
|
}
|
|
108
|
-
if (!this.
|
|
108
|
+
if (!this.avmProof) {
|
|
109
109
|
throw new Error('Tx not ready for proving base rollup: avm proof undefined');
|
|
110
110
|
}
|
|
111
111
|
if (!(this.baseRollupHints instanceof PublicBaseRollupHints)) {
|
|
@@ -114,10 +114,7 @@ export class TxProvingState {
|
|
|
114
114
|
|
|
115
115
|
const publicChonkVerifierProofData = toProofData(this.publicChonkVerifier);
|
|
116
116
|
|
|
117
|
-
const avmProofData = new ProofDataForFixedVk(
|
|
118
|
-
this.processedTx.avmProvingRequest.inputs.publicInputs,
|
|
119
|
-
this.avm.proof,
|
|
120
|
-
);
|
|
117
|
+
const avmProofData = new ProofDataForFixedVk(this.processedTx.avmProvingRequest.inputs.publicInputs, this.avmProof);
|
|
121
118
|
|
|
122
119
|
return new PublicTxBaseRollupPrivateInputs(publicChonkVerifierProofData, avmProofData, this.baseRollupHints);
|
|
123
120
|
}
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
ForkMerkleTreeOperations,
|
|
3
|
+
ProvingJobBroker,
|
|
4
|
+
ReadonlyWorldStateAccess,
|
|
5
|
+
} from '@aztec/stdlib/interfaces/server';
|
|
2
6
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
3
7
|
|
|
4
8
|
import type { ProverClientConfig } from '../config.js';
|
|
@@ -6,7 +10,7 @@ import { ProverClient } from './prover-client.js';
|
|
|
6
10
|
|
|
7
11
|
export function createProverClient(
|
|
8
12
|
config: ProverClientConfig,
|
|
9
|
-
worldState: ForkMerkleTreeOperations,
|
|
13
|
+
worldState: ForkMerkleTreeOperations & ReadonlyWorldStateAccess,
|
|
10
14
|
broker: ProvingJobBroker,
|
|
11
15
|
telemetry: TelemetryClient = getTelemetryClient(),
|
|
12
16
|
) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type ACVMConfig, type BBConfig, BBNativeRollupProver, TestCircuitProver } from '@aztec/bb-prover';
|
|
2
2
|
import { times } from '@aztec/foundation/collection';
|
|
3
3
|
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
4
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
4
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
5
5
|
import { NativeACVMSimulator } from '@aztec/simulator/server';
|
|
6
6
|
import {
|
|
7
7
|
type ActualProverConfig,
|
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
type ProvingJobBroker,
|
|
12
12
|
type ProvingJobConsumer,
|
|
13
13
|
type ProvingJobProducer,
|
|
14
|
+
type ReadonlyWorldStateAccess,
|
|
14
15
|
type ServerCircuitProver,
|
|
15
16
|
tryStop,
|
|
16
17
|
} from '@aztec/stdlib/interfaces/server';
|
|
@@ -28,24 +29,35 @@ export class ProverClient implements EpochProverManager {
|
|
|
28
29
|
private running = false;
|
|
29
30
|
private agents: ProvingAgent[] = [];
|
|
30
31
|
|
|
31
|
-
private proofStore: ProofStore;
|
|
32
|
-
private failedProofStore: ProofStore | undefined;
|
|
33
|
-
|
|
34
32
|
private constructor(
|
|
35
33
|
private config: ProverClientConfig,
|
|
36
|
-
private worldState: ForkMerkleTreeOperations,
|
|
34
|
+
private worldState: ForkMerkleTreeOperations & ReadonlyWorldStateAccess,
|
|
37
35
|
private orchestratorClient: ProvingJobProducer,
|
|
36
|
+
private proofStore: ProofStore,
|
|
37
|
+
private failedProofStore: ProofStore | undefined,
|
|
38
38
|
private agentClient?: ProvingJobConsumer,
|
|
39
39
|
private telemetry: TelemetryClient = getTelemetryClient(),
|
|
40
|
-
private log = createLogger('prover-client:tx-prover'),
|
|
41
|
-
) {
|
|
42
|
-
this.proofStore = new InlineProofStore();
|
|
43
|
-
this.failedProofStore = this.config.failedProofStore ? createProofStore(this.config.failedProofStore) : undefined;
|
|
44
|
-
}
|
|
40
|
+
private log: Logger = createLogger('prover-client:tx-prover'),
|
|
41
|
+
) {}
|
|
45
42
|
|
|
46
43
|
public createEpochProver(): EpochProver {
|
|
47
|
-
const
|
|
48
|
-
const
|
|
44
|
+
const bindings = this.log.getBindings();
|
|
45
|
+
const facade = new BrokerCircuitProverFacade(
|
|
46
|
+
this.orchestratorClient,
|
|
47
|
+
this.proofStore,
|
|
48
|
+
this.failedProofStore,
|
|
49
|
+
undefined,
|
|
50
|
+
bindings,
|
|
51
|
+
);
|
|
52
|
+
const orchestrator = new ProvingOrchestrator(
|
|
53
|
+
this.worldState,
|
|
54
|
+
facade,
|
|
55
|
+
this.config.proverId,
|
|
56
|
+
this.config.cancelJobsOnStop,
|
|
57
|
+
this.config.enqueueConcurrency,
|
|
58
|
+
this.telemetry,
|
|
59
|
+
bindings,
|
|
60
|
+
);
|
|
49
61
|
return new ServerEpochProver(facade, orchestrator);
|
|
50
62
|
}
|
|
51
63
|
|
|
@@ -99,11 +111,13 @@ export class ProverClient implements EpochProverManager {
|
|
|
99
111
|
*/
|
|
100
112
|
public static async new(
|
|
101
113
|
config: ProverClientConfig,
|
|
102
|
-
worldState: ForkMerkleTreeOperations,
|
|
114
|
+
worldState: ForkMerkleTreeOperations & ReadonlyWorldStateAccess,
|
|
103
115
|
broker: ProvingJobBroker,
|
|
104
116
|
telemetry: TelemetryClient = getTelemetryClient(),
|
|
105
117
|
) {
|
|
106
|
-
const
|
|
118
|
+
const proofStore = await createProofStore(config.proofStore);
|
|
119
|
+
const failedProofStore = config.failedProofStore ? await createProofStore(config.failedProofStore) : undefined;
|
|
120
|
+
const prover = new ProverClient(config, worldState, broker, proofStore, failedProofStore, broker, telemetry);
|
|
107
121
|
await prover.start();
|
|
108
122
|
return prover;
|
|
109
123
|
}
|
|
@@ -127,17 +141,11 @@ export class ProverClient implements EpochProverManager {
|
|
|
127
141
|
|
|
128
142
|
const proofStore = new InlineProofStore();
|
|
129
143
|
const prover = await buildServerCircuitProver(this.config, this.telemetry);
|
|
144
|
+
const bindings = this.log.getBindings();
|
|
130
145
|
this.agents = times(
|
|
131
146
|
this.config.proverAgentCount,
|
|
132
147
|
() =>
|
|
133
|
-
new ProvingAgent(
|
|
134
|
-
this.agentClient!,
|
|
135
|
-
proofStore,
|
|
136
|
-
prover,
|
|
137
|
-
[],
|
|
138
|
-
this.config.proverAgentPollIntervalMs,
|
|
139
|
-
this.telemetry,
|
|
140
|
-
),
|
|
148
|
+
new ProvingAgent(this.agentClient!, proofStore, prover, [], this.config.proverAgentPollIntervalMs, bindings),
|
|
141
149
|
);
|
|
142
150
|
|
|
143
151
|
await Promise.all(this.agents.map(agent => agent.start()));
|
|
@@ -149,15 +157,16 @@ export class ProverClient implements EpochProverManager {
|
|
|
149
157
|
}
|
|
150
158
|
|
|
151
159
|
export function buildServerCircuitProver(
|
|
152
|
-
config: ActualProverConfig & ACVMConfig & BBConfig,
|
|
160
|
+
config: Omit<ActualProverConfig, 'enqueueConcurrency'> & ACVMConfig & BBConfig,
|
|
153
161
|
telemetry: TelemetryClient,
|
|
154
162
|
): Promise<ServerCircuitProver> {
|
|
155
163
|
if (config.realProofs) {
|
|
156
164
|
return BBNativeRollupProver.new(config, telemetry);
|
|
157
165
|
}
|
|
158
166
|
|
|
167
|
+
const logger = createLogger('prover-client:acvm-native');
|
|
159
168
|
const simulator = config.acvmBinaryPath
|
|
160
|
-
? new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath)
|
|
169
|
+
? new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath, undefined, logger)
|
|
161
170
|
: undefined;
|
|
162
171
|
|
|
163
172
|
return Promise.resolve(new TestCircuitProver(simulator, config, telemetry));
|
|
@@ -5,14 +5,11 @@ import type {
|
|
|
5
5
|
RECURSIVE_PROOF_LENGTH,
|
|
6
6
|
} from '@aztec/constants';
|
|
7
7
|
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
8
|
-
import {
|
|
9
|
-
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
10
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
8
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
11
9
|
import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
12
10
|
import { truncate } from '@aztec/foundation/string';
|
|
13
11
|
import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
|
|
14
12
|
import {
|
|
15
|
-
type ProofAndVerificationKey,
|
|
16
13
|
type ProofUri,
|
|
17
14
|
type ProvingJobId,
|
|
18
15
|
type ProvingJobInputsMap,
|
|
@@ -24,7 +21,7 @@ import {
|
|
|
24
21
|
makeProvingJobId,
|
|
25
22
|
} from '@aztec/stdlib/interfaces/server';
|
|
26
23
|
import type { ParityBasePrivateInputs, ParityPublicInputs, ParityRootPrivateInputs } from '@aztec/stdlib/parity';
|
|
27
|
-
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
24
|
+
import { ProvingRequestType, RecursiveProof } from '@aztec/stdlib/proofs';
|
|
28
25
|
import type {
|
|
29
26
|
BlockMergeRollupPrivateInputs,
|
|
30
27
|
BlockRollupPublicInputs,
|
|
@@ -48,6 +45,8 @@ import type {
|
|
|
48
45
|
TxRollupPublicInputs,
|
|
49
46
|
} from '@aztec/stdlib/rollup';
|
|
50
47
|
|
|
48
|
+
import { createHash } from 'node:crypto';
|
|
49
|
+
|
|
51
50
|
import { InlineProofStore, type ProofStore } from './proof_store/index.js';
|
|
52
51
|
|
|
53
52
|
// Perform a snapshot sync every 30 seconds
|
|
@@ -70,14 +69,17 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
|
|
|
70
69
|
private runningPromise?: RunningPromise;
|
|
71
70
|
private timeOfLastSnapshotSync = Date.now();
|
|
72
71
|
private jobsToRetrieve: Set<ProvingJobId> = new Set();
|
|
72
|
+
private log: Logger;
|
|
73
73
|
|
|
74
74
|
constructor(
|
|
75
75
|
private broker: ProvingJobProducer,
|
|
76
76
|
private proofStore: ProofStore = new InlineProofStore(),
|
|
77
77
|
private failedProofStore?: ProofStore,
|
|
78
78
|
private pollIntervalMs = 1000,
|
|
79
|
-
|
|
80
|
-
) {
|
|
79
|
+
bindings?: LoggerBindings,
|
|
80
|
+
) {
|
|
81
|
+
this.log = createLogger('prover-client:broker-circuit-prover-facade', bindings);
|
|
82
|
+
}
|
|
81
83
|
|
|
82
84
|
/**
|
|
83
85
|
* This is a critical section. This function can not be async since it writes
|
|
@@ -398,24 +400,16 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
|
|
|
398
400
|
|
|
399
401
|
getAvmProof(
|
|
400
402
|
inputs: AvmCircuitInputs,
|
|
401
|
-
skipPublicInputsValidation?: boolean, // TODO(#14234)[Unconditional PIs validation]: remove this argument
|
|
402
403
|
signal?: AbortSignal,
|
|
403
404
|
epochNumber?: EpochNumber,
|
|
404
|
-
): Promise<
|
|
405
|
-
this.log.info(`getAvmProof() called with skipPublicInputsValidation: ${skipPublicInputsValidation}`);
|
|
406
|
-
|
|
405
|
+
): Promise<RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>> {
|
|
407
406
|
return this.enqueueJob(
|
|
408
407
|
this.generateId(ProvingRequestType.PUBLIC_VM, inputs, epochNumber),
|
|
409
408
|
ProvingRequestType.PUBLIC_VM,
|
|
410
409
|
inputs,
|
|
411
410
|
epochNumber,
|
|
412
411
|
signal,
|
|
413
|
-
)
|
|
414
|
-
// TODO(#14234)[Unconditional PIs validation]: Remove ".then()".
|
|
415
|
-
// Override the default value of skipPublicInputsValidation potentially set in BBNativeRollupProver.getAvmProof().
|
|
416
|
-
result.proof.proof[0] = skipPublicInputsValidation ? new Fr(1) : new Fr(0);
|
|
417
|
-
return result;
|
|
418
|
-
});
|
|
412
|
+
);
|
|
419
413
|
}
|
|
420
414
|
|
|
421
415
|
getBaseParityProof(
|
|
@@ -666,8 +660,12 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
|
|
|
666
660
|
);
|
|
667
661
|
}
|
|
668
662
|
|
|
669
|
-
private generateId(
|
|
670
|
-
|
|
671
|
-
|
|
663
|
+
private generateId(
|
|
664
|
+
type: ProvingRequestType,
|
|
665
|
+
inputs: { toBuffer(): Buffer },
|
|
666
|
+
epochNumber = EpochNumber.ZERO,
|
|
667
|
+
): ProvingJobId {
|
|
668
|
+
const inputsHash = createHash('sha256').update(inputs.toBuffer()).digest('hex');
|
|
669
|
+
return makeProvingJobId(epochNumber, type, inputsHash);
|
|
672
670
|
}
|
|
673
671
|
}
|
|
@@ -31,6 +31,8 @@ export const ProverBrokerConfig = z.object({
|
|
|
31
31
|
proverBrokerBatchIntervalMs: z.number().int().nonnegative(),
|
|
32
32
|
/** The maximum number of epochs to keep results for */
|
|
33
33
|
proverBrokerMaxEpochsToKeepResultsFor: z.number().int().nonnegative(),
|
|
34
|
+
/** Enable debug replay mode for replaying proving jobs from stored inputs */
|
|
35
|
+
proverBrokerDebugReplayEnabled: z.boolean(),
|
|
34
36
|
});
|
|
35
37
|
|
|
36
38
|
export type ProverBrokerConfig = z.infer<typeof ProverBrokerConfig> &
|
|
@@ -74,6 +76,11 @@ export const proverBrokerConfigMappings: ConfigMappingsType<ProverBrokerConfig>
|
|
|
74
76
|
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
75
77
|
description: "The size of the prover broker's database. Will override the dataStoreMapSizeKb if set.",
|
|
76
78
|
},
|
|
79
|
+
proverBrokerDebugReplayEnabled: {
|
|
80
|
+
env: 'PROVER_BROKER_DEBUG_REPLAY_ENABLED',
|
|
81
|
+
description: 'Enable debug replay mode for replaying proving jobs from stored inputs',
|
|
82
|
+
...booleanConfigHelper(false),
|
|
83
|
+
},
|
|
77
84
|
...dataConfigMappings,
|
|
78
85
|
...l1ReaderConfigMappings,
|
|
79
86
|
...pickConfigMappings(chainConfigMappings, ['rollupVersion']),
|
|
@@ -100,6 +107,10 @@ export const ProverAgentConfig = z.object({
|
|
|
100
107
|
proverTestDelayFactor: z.number(),
|
|
101
108
|
/** The delay (ms) to inject during fake proof verification */
|
|
102
109
|
proverTestVerificationDelayMs: z.number().optional(),
|
|
110
|
+
/** Whether to abort pending proving jobs when the orchestrator is cancelled */
|
|
111
|
+
cancelJobsOnStop: z.boolean(),
|
|
112
|
+
/** Where to store proving results. Must be accessible to both prover node and agents. If not set will inline-encode the parameters */
|
|
113
|
+
proofStore: z.string().optional(),
|
|
103
114
|
});
|
|
104
115
|
|
|
105
116
|
export type ProverAgentConfig = z.infer<typeof ProverAgentConfig>;
|
|
@@ -153,4 +164,15 @@ export const proverAgentConfigMappings: ConfigMappingsType<ProverAgentConfig> =
|
|
|
153
164
|
description: 'The delay (ms) to inject during fake proof verification',
|
|
154
165
|
...numberConfigHelper(10),
|
|
155
166
|
},
|
|
167
|
+
cancelJobsOnStop: {
|
|
168
|
+
env: 'PROVER_CANCEL_JOBS_ON_STOP',
|
|
169
|
+
description:
|
|
170
|
+
'Whether to abort pending proving jobs when the orchestrator is cancelled. ' +
|
|
171
|
+
'When false (default), jobs remain in the broker queue and can be reused on restart/reorg.',
|
|
172
|
+
...booleanConfigHelper(false),
|
|
173
|
+
},
|
|
174
|
+
proofStore: {
|
|
175
|
+
env: 'PROVER_PROOF_STORE',
|
|
176
|
+
description: 'Optional proof input store for the prover',
|
|
177
|
+
},
|
|
156
178
|
};
|
|
@@ -1,42 +1,20 @@
|
|
|
1
1
|
import { createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import { createFileStore } from '@aztec/stdlib/file-store';
|
|
2
3
|
|
|
3
|
-
import {
|
|
4
|
+
import { FileStoreProofStore } from './file_store_proof_store.js';
|
|
4
5
|
import { InlineProofStore } from './inline_proof_store.js';
|
|
5
6
|
import type { ProofStore } from './proof_store.js';
|
|
6
7
|
|
|
7
|
-
export function createProofStore(
|
|
8
|
-
|
|
8
|
+
export async function createProofStore(
|
|
9
|
+
config: string | undefined,
|
|
10
|
+
logger = createLogger('prover-client:proof-store'),
|
|
11
|
+
): Promise<ProofStore> {
|
|
12
|
+
if (!config) {
|
|
9
13
|
logger.info('Creating inline proof store');
|
|
10
14
|
return new InlineProofStore();
|
|
11
|
-
} else if (config.startsWith('gs://')) {
|
|
12
|
-
try {
|
|
13
|
-
const url = new URL(config);
|
|
14
|
-
const bucket = url.host;
|
|
15
|
-
const path = url.pathname.replace(/^\/+/, '');
|
|
16
|
-
logger.info(`Creating google cloud proof store at ${bucket}`, { bucket, path });
|
|
17
|
-
return new GoogleCloudStorageProofStore(bucket, path);
|
|
18
|
-
} catch {
|
|
19
|
-
throw new Error(
|
|
20
|
-
`Invalid google cloud proof store definition: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`,
|
|
21
|
-
);
|
|
22
|
-
}
|
|
23
|
-
} else {
|
|
24
|
-
throw new Error(`Unknown proof store config: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`);
|
|
25
15
|
}
|
|
26
|
-
}
|
|
27
16
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
): Pick<ProofStore, 'getProofInput' | 'getProofOutput'> {
|
|
32
|
-
if (uri.startsWith('data://')) {
|
|
33
|
-
return createProofStore(undefined, logger);
|
|
34
|
-
} else if (uri.startsWith('gs://')) {
|
|
35
|
-
const url = new URL(uri);
|
|
36
|
-
const basePath = url.pathname.replace(/^\/+/, '').split('/').slice(0, -3);
|
|
37
|
-
url.pathname = basePath.join('/');
|
|
38
|
-
return createProofStore(uri, logger);
|
|
39
|
-
} else {
|
|
40
|
-
throw new Error(`Unknown proof store config: '${uri}'. Supported protocols are 'data://' and 'gs://'.`);
|
|
41
|
-
}
|
|
17
|
+
const fileStore = await createFileStore(config, logger);
|
|
18
|
+
logger.info(`Creating file store proof store at ${config}`);
|
|
19
|
+
return new FileStoreProofStore(fileStore);
|
|
42
20
|
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
|
|
2
|
+
import type { FileStore } from '@aztec/stdlib/file-store';
|
|
3
|
+
import {
|
|
4
|
+
type ProofUri,
|
|
5
|
+
type ProvingJobId,
|
|
6
|
+
type ProvingJobInputs,
|
|
7
|
+
type ProvingJobInputsMap,
|
|
8
|
+
ProvingJobResult,
|
|
9
|
+
type ProvingJobResultsMap,
|
|
10
|
+
getProvingJobInputClassFor,
|
|
11
|
+
} from '@aztec/stdlib/interfaces/server';
|
|
12
|
+
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
13
|
+
|
|
14
|
+
import type { ProofStore } from './proof_store.js';
|
|
15
|
+
|
|
16
|
+
const INPUTS_PATH = 'inputs';
|
|
17
|
+
const OUTPUTS_PATH = 'outputs';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* A proof store implementation backed by a generic FileStore.
|
|
21
|
+
* Supports any storage backend (GCS, S3, local filesystem) via the FileStore abstraction.
|
|
22
|
+
*/
|
|
23
|
+
export class FileStoreProofStore implements ProofStore {
|
|
24
|
+
constructor(private readonly fileStore: FileStore) {}
|
|
25
|
+
|
|
26
|
+
async saveProofInput<T extends ProvingRequestType>(
|
|
27
|
+
id: ProvingJobId,
|
|
28
|
+
type: T,
|
|
29
|
+
inputs: ProvingJobInputsMap[T],
|
|
30
|
+
): Promise<ProofUri> {
|
|
31
|
+
const path = `${INPUTS_PATH}/${ProvingRequestType[type]}/${id}`;
|
|
32
|
+
const uri = await this.fileStore.save(path, inputs.toBuffer());
|
|
33
|
+
return uri as ProofUri;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async saveProofOutput<T extends ProvingRequestType>(
|
|
37
|
+
id: ProvingJobId,
|
|
38
|
+
type: T,
|
|
39
|
+
result: ProvingJobResultsMap[T],
|
|
40
|
+
): Promise<ProofUri> {
|
|
41
|
+
const jobResult = { type, result } as ProvingJobResult;
|
|
42
|
+
const json = jsonStringify(jobResult);
|
|
43
|
+
const path = `${OUTPUTS_PATH}/${ProvingRequestType[type]}/${id}.json`;
|
|
44
|
+
const uri = await this.fileStore.save(path, Buffer.from(json, 'utf-8'));
|
|
45
|
+
return uri as ProofUri;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async getProofInput(uri: ProofUri): Promise<ProvingJobInputs> {
|
|
49
|
+
try {
|
|
50
|
+
const buffer = await this.fileStore.read(uri);
|
|
51
|
+
const type = this.extractTypeFromUri(uri);
|
|
52
|
+
const inputs = getProvingJobInputClassFor(type).fromBuffer(buffer);
|
|
53
|
+
return { inputs, type } as ProvingJobInputs;
|
|
54
|
+
} catch (err) {
|
|
55
|
+
throw new Error(`Error getting proof input at ${uri}: ${err}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async getProofOutput(uri: ProofUri): Promise<ProvingJobResult> {
|
|
60
|
+
try {
|
|
61
|
+
const buffer = await this.fileStore.read(uri);
|
|
62
|
+
return jsonParseWithSchema(buffer.toString('utf-8'), ProvingJobResult);
|
|
63
|
+
} catch (err) {
|
|
64
|
+
throw new Error(`Error getting proof output at ${uri}: ${err}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
private extractTypeFromUri(uri: string): ProvingRequestType {
|
|
69
|
+
const url = new URL(uri);
|
|
70
|
+
const pathParts = url.pathname.split('/').filter(Boolean);
|
|
71
|
+
const typeString = pathParts.at(-2);
|
|
72
|
+
const type = typeString ? ProvingRequestType[typeString as keyof typeof ProvingRequestType] : undefined;
|
|
73
|
+
if (type === undefined) {
|
|
74
|
+
throw new Error(`Unrecognized proof type ${typeString} in URI ${uri}`);
|
|
75
|
+
}
|
|
76
|
+
return type;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AbortError } from '@aztec/foundation/error';
|
|
2
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
3
3
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
4
4
|
import { truncate } from '@aztec/foundation/string';
|
|
5
5
|
import { ProvingError } from '@aztec/stdlib/errors';
|
|
@@ -13,27 +13,17 @@ import type {
|
|
|
13
13
|
ServerCircuitProver,
|
|
14
14
|
} from '@aztec/stdlib/interfaces/server';
|
|
15
15
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
16
|
-
import {
|
|
17
|
-
type TelemetryClient,
|
|
18
|
-
type Traceable,
|
|
19
|
-
type Tracer,
|
|
20
|
-
getTelemetryClient,
|
|
21
|
-
trackSpan,
|
|
22
|
-
} from '@aztec/telemetry-client';
|
|
23
16
|
|
|
24
17
|
import type { ProofStore } from './proof_store/index.js';
|
|
25
|
-
import { ProvingAgentInstrumentation } from './proving_agent_instrumentation.js';
|
|
26
18
|
import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_controller.js';
|
|
27
19
|
|
|
28
20
|
/**
|
|
29
21
|
* A helper class that encapsulates a circuit prover and connects it to a job source.
|
|
30
22
|
*/
|
|
31
|
-
export class ProvingAgent
|
|
23
|
+
export class ProvingAgent {
|
|
32
24
|
private currentJobController?: ProvingJobController;
|
|
33
25
|
private runningPromise: RunningPromise;
|
|
34
|
-
private
|
|
35
|
-
|
|
36
|
-
public readonly tracer: Tracer;
|
|
26
|
+
private log: Logger;
|
|
37
27
|
|
|
38
28
|
constructor(
|
|
39
29
|
/** The source of proving jobs */
|
|
@@ -46,12 +36,9 @@ export class ProvingAgent implements Traceable {
|
|
|
46
36
|
private proofAllowList: Array<ProvingRequestType> = [],
|
|
47
37
|
/** How long to wait between jobs */
|
|
48
38
|
private pollIntervalMs = 1000,
|
|
49
|
-
|
|
50
|
-
client: TelemetryClient = getTelemetryClient(),
|
|
51
|
-
private log = createLogger('prover-client:proving-agent'),
|
|
39
|
+
bindings?: LoggerBindings,
|
|
52
40
|
) {
|
|
53
|
-
this.
|
|
54
|
-
this.instrumentation = new ProvingAgentInstrumentation(client);
|
|
41
|
+
this.log = createLogger('prover-client:proving-agent', bindings);
|
|
55
42
|
this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollIntervalMs);
|
|
56
43
|
}
|
|
57
44
|
|
|
@@ -85,7 +72,6 @@ export class ProvingAgent implements Traceable {
|
|
|
85
72
|
return this.runningPromise.isRunning() ? { status: 'running' } : { status: 'stopped' };
|
|
86
73
|
}
|
|
87
74
|
|
|
88
|
-
@trackSpan('ProvingAgent.safeWork')
|
|
89
75
|
private async work() {
|
|
90
76
|
// every tick we need to take one of the following actions:
|
|
91
77
|
// 1. send a hearbeat to the broker that we're working on some job
|
|
@@ -175,6 +161,7 @@ export class ProvingAgent implements Traceable {
|
|
|
175
161
|
// no need to await this here. The controller will stay alive (in DONE state) until the result is send to the broker
|
|
176
162
|
void this.runningPromise.trigger();
|
|
177
163
|
},
|
|
164
|
+
this.log.getBindings(),
|
|
178
165
|
);
|
|
179
166
|
|
|
180
167
|
if (abortedProofJobId) {
|