@aztec/prover-client 0.70.0 → 0.71.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/bin/get-proof-inputs.d.ts +2 -0
- package/dest/bin/get-proof-inputs.d.ts.map +1 -0
- package/dest/bin/get-proof-inputs.js +50 -0
- package/dest/block_builder/light.d.ts +3 -4
- package/dest/block_builder/light.d.ts.map +1 -1
- package/dest/block_builder/light.js +8 -11
- package/dest/mocks/test_context.d.ts.map +1 -1
- package/dest/mocks/test_context.js +9 -10
- package/dest/orchestrator/block-building-helpers.d.ts +8 -7
- package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
- package/dest/orchestrator/block-building-helpers.js +20 -16
- package/dest/orchestrator/orchestrator.d.ts +1 -1
- package/dest/orchestrator/orchestrator.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator.js +5 -11
- package/dest/prover-agent/memory-proving-queue.d.ts +1 -1
- package/dest/prover-agent/memory-proving-queue.d.ts.map +1 -1
- package/dest/prover-agent/memory-proving-queue.js +2 -2
- package/dest/prover-agent/prover-agent.d.ts +0 -2
- package/dest/prover-agent/prover-agent.d.ts.map +1 -1
- package/dest/prover-agent/prover-agent.js +4 -6
- package/dest/prover-client/factory.d.ts.map +1 -1
- package/dest/prover-client/factory.js +3 -3
- package/dest/prover-client/prover-client.d.ts +4 -2
- package/dest/prover-client/prover-client.d.ts.map +1 -1
- package/dest/prover-client/prover-client.js +13 -15
- package/dest/proving_broker/broker_prover_facade.d.ts +4 -3
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
- package/dest/proving_broker/broker_prover_facade.js +25 -6
- package/dest/proving_broker/factory.d.ts.map +1 -1
- package/dest/proving_broker/factory.js +3 -3
- package/dest/proving_broker/index.d.ts +1 -1
- package/dest/proving_broker/index.d.ts.map +1 -1
- package/dest/proving_broker/index.js +2 -2
- package/dest/proving_broker/proof_store/factory.d.ts +6 -0
- package/dest/proving_broker/proof_store/factory.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/factory.js +39 -0
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +13 -0
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/gcs_proof_store.js +46 -0
- package/dest/proving_broker/proof_store/index.d.ts +4 -0
- package/dest/proving_broker/proof_store/index.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/index.js +4 -0
- package/dest/proving_broker/proof_store/inline_proof_store.d.ts +14 -0
- package/dest/proving_broker/proof_store/inline_proof_store.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/inline_proof_store.js +37 -0
- package/dest/proving_broker/{proof_store.d.ts → proof_store/proof_store.d.ts} +1 -12
- package/dest/proving_broker/proof_store/proof_store.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/proof_store.js +2 -0
- package/dest/proving_broker/proving_agent.d.ts +4 -4
- package/dest/proving_broker/proving_agent.d.ts.map +1 -1
- package/dest/proving_broker/proving_agent.js +5 -5
- package/dest/proving_broker/proving_broker.d.ts +1 -1
- package/dest/proving_broker/proving_broker.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.js +3 -3
- package/dest/proving_broker/proving_broker_database/persisted.d.ts +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.js +4 -4
- package/dest/test/mock_prover.d.ts +1 -1
- package/dest/test/mock_prover.d.ts.map +1 -1
- package/dest/test/mock_prover.js +4 -5
- package/package.json +14 -12
- package/src/bin/get-proof-inputs.ts +60 -0
- package/src/block_builder/light.ts +6 -11
- package/src/mocks/test_context.ts +7 -9
- package/src/orchestrator/block-building-helpers.ts +358 -326
- package/src/orchestrator/orchestrator.ts +11 -15
- package/src/prover-agent/memory-proving-queue.ts +1 -1
- package/src/prover-agent/prover-agent.ts +10 -4
- package/src/prover-client/factory.ts +2 -3
- package/src/prover-client/prover-client.ts +14 -15
- package/src/proving_broker/broker_prover_facade.ts +28 -5
- package/src/proving_broker/factory.ts +9 -5
- package/src/proving_broker/index.ts +1 -1
- package/src/proving_broker/proof_store/factory.ts +42 -0
- package/src/proving_broker/proof_store/gcs_proof_store.ts +72 -0
- package/src/proving_broker/proof_store/index.ts +3 -0
- package/src/proving_broker/{proof_store.ts → proof_store/inline_proof_store.ts} +1 -44
- package/src/proving_broker/proof_store/proof_store.ts +54 -0
- package/src/proving_broker/proving_agent.ts +11 -5
- package/src/proving_broker/proving_broker.ts +8 -2
- package/src/proving_broker/proving_broker_database/persisted.ts +3 -3
- package/src/test/mock_prover.ts +3 -4
- package/dest/proving_broker/proof_store.d.ts.map +0 -1
- package/dest/proving_broker/proof_store.js +0 -37
|
@@ -49,7 +49,14 @@ import { pushTestData } from '@aztec/foundation/testing';
|
|
|
49
49
|
import { elapsed } from '@aztec/foundation/timer';
|
|
50
50
|
import { type TreeNodeLocation } from '@aztec/foundation/trees';
|
|
51
51
|
import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vks';
|
|
52
|
-
import {
|
|
52
|
+
import {
|
|
53
|
+
Attributes,
|
|
54
|
+
type TelemetryClient,
|
|
55
|
+
type Tracer,
|
|
56
|
+
getTelemetryClient,
|
|
57
|
+
trackSpan,
|
|
58
|
+
wrapCallbackInSpan,
|
|
59
|
+
} from '@aztec/telemetry-client';
|
|
53
60
|
|
|
54
61
|
import { inspect } from 'util';
|
|
55
62
|
|
|
@@ -94,8 +101,8 @@ export class ProvingOrchestrator implements EpochProver {
|
|
|
94
101
|
constructor(
|
|
95
102
|
private dbProvider: ForkMerkleTreeOperations,
|
|
96
103
|
private prover: ServerCircuitProver,
|
|
97
|
-
telemetryClient: TelemetryClient,
|
|
98
104
|
private readonly proverId: Fr = Fr.ZERO,
|
|
105
|
+
telemetryClient: TelemetryClient = getTelemetryClient(),
|
|
99
106
|
) {
|
|
100
107
|
this.metrics = new ProvingOrchestratorMetrics(telemetryClient, 'ProvingOrchestrator');
|
|
101
108
|
}
|
|
@@ -263,11 +270,6 @@ export class ProvingOrchestrator implements EpochProver {
|
|
|
263
270
|
|
|
264
271
|
logger.info(`Received transaction: ${tx.hash}`);
|
|
265
272
|
|
|
266
|
-
if (tx.isEmpty) {
|
|
267
|
-
logger.warn(`Ignoring empty transaction ${tx.hash} - it will not be added to this block`);
|
|
268
|
-
continue;
|
|
269
|
-
}
|
|
270
|
-
|
|
271
273
|
const [hints, treeSnapshots] = await this.prepareTransaction(tx, provingState);
|
|
272
274
|
const txProvingState = new TxProvingState(tx, hints, treeSnapshots);
|
|
273
275
|
const txIndex = provingState.addNewTx(txProvingState);
|
|
@@ -521,9 +523,7 @@ export class ProvingOrchestrator implements EpochProver {
|
|
|
521
523
|
buildBaseRollupHints(tx, provingState.globalVariables, db, provingState.spongeBlobState),
|
|
522
524
|
);
|
|
523
525
|
|
|
524
|
-
|
|
525
|
-
this.metrics.recordBaseRollupInputs(ms);
|
|
526
|
-
}
|
|
526
|
+
this.metrics.recordBaseRollupInputs(ms);
|
|
527
527
|
|
|
528
528
|
const promises = [MerkleTreeId.NOTE_HASH_TREE, MerkleTreeId.NULLIFIER_TREE, MerkleTreeId.PUBLIC_DATA_TREE].map(
|
|
529
529
|
async (id: MerkleTreeId) => {
|
|
@@ -551,11 +551,7 @@ export class ProvingOrchestrator implements EpochProver {
|
|
|
551
551
|
const { processedTx } = txProvingState;
|
|
552
552
|
const { rollupType, inputs } = txProvingState.getBaseRollupTypeAndInputs();
|
|
553
553
|
|
|
554
|
-
logger.debug(
|
|
555
|
-
`Enqueuing deferred proving base rollup${
|
|
556
|
-
processedTx.isEmpty ? ' with padding tx' : ''
|
|
557
|
-
} for ${processedTx.hash.toString()}`,
|
|
558
|
-
);
|
|
554
|
+
logger.debug(`Enqueuing deferred proving base rollup for ${processedTx.hash.toString()}`);
|
|
559
555
|
|
|
560
556
|
this.deferredProving(
|
|
561
557
|
provingState,
|
|
@@ -40,7 +40,7 @@ import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from
|
|
|
40
40
|
import { PriorityMemoryQueue } from '@aztec/foundation/queue';
|
|
41
41
|
import { type TelemetryClient, type Tracer, trackSpan } from '@aztec/telemetry-client';
|
|
42
42
|
|
|
43
|
-
import { InlineProofStore, type ProofStore } from '../proving_broker/proof_store.js';
|
|
43
|
+
import { InlineProofStore, type ProofStore } from '../proving_broker/proof_store/index.js';
|
|
44
44
|
import { ProvingQueueMetrics } from './queue_metrics.js';
|
|
45
45
|
|
|
46
46
|
type ProvingJobWithResolvers<T extends ProvingRequestType = ProvingRequestType> = ProvingJob &
|
|
@@ -11,10 +11,16 @@ import {
|
|
|
11
11
|
import { createLogger } from '@aztec/foundation/log';
|
|
12
12
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
13
13
|
import { elapsed } from '@aztec/foundation/timer';
|
|
14
|
-
import {
|
|
15
|
-
|
|
14
|
+
import {
|
|
15
|
+
Attributes,
|
|
16
|
+
type TelemetryClient,
|
|
17
|
+
type Traceable,
|
|
18
|
+
type Tracer,
|
|
19
|
+
getTelemetryClient,
|
|
20
|
+
trackSpan,
|
|
21
|
+
} from '@aztec/telemetry-client';
|
|
16
22
|
|
|
17
|
-
import { InlineProofStore } from '../proving_broker/proof_store.js';
|
|
23
|
+
import { InlineProofStore } from '../proving_broker/proof_store/index.js';
|
|
18
24
|
|
|
19
25
|
const PRINT_THRESHOLD_NS = 6e10; // 60 seconds
|
|
20
26
|
|
|
@@ -42,7 +48,7 @@ export class ProverAgent implements ProverAgentApi, Traceable {
|
|
|
42
48
|
/** How long to wait between jobs */
|
|
43
49
|
private pollIntervalMs = 100,
|
|
44
50
|
/** Telemetry client */
|
|
45
|
-
|
|
51
|
+
telemetry: TelemetryClient = getTelemetryClient(),
|
|
46
52
|
/** Logger */
|
|
47
53
|
private log = createLogger('prover-client:prover-agent'),
|
|
48
54
|
) {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { type ForkMerkleTreeOperations, type ProvingJobBroker } from '@aztec/circuit-types';
|
|
2
|
-
import { type TelemetryClient } from '@aztec/telemetry-client';
|
|
3
|
-
import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
|
|
2
|
+
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
4
3
|
|
|
5
4
|
import { type ProverClientConfig } from '../config.js';
|
|
6
5
|
import { ProverClient } from './prover-client.js';
|
|
@@ -9,7 +8,7 @@ export function createProverClient(
|
|
|
9
8
|
config: ProverClientConfig,
|
|
10
9
|
worldState: ForkMerkleTreeOperations,
|
|
11
10
|
broker: ProvingJobBroker,
|
|
12
|
-
telemetry: TelemetryClient =
|
|
11
|
+
telemetry: TelemetryClient = getTelemetryClient(),
|
|
13
12
|
) {
|
|
14
13
|
return ProverClient.new(config, worldState, broker, telemetry);
|
|
15
14
|
}
|
|
@@ -13,12 +13,12 @@ import { Fr } from '@aztec/circuits.js';
|
|
|
13
13
|
import { times } from '@aztec/foundation/collection';
|
|
14
14
|
import { createLogger } from '@aztec/foundation/log';
|
|
15
15
|
import { NativeACVMSimulator } from '@aztec/simulator/server';
|
|
16
|
-
import { type TelemetryClient } from '@aztec/telemetry-client';
|
|
16
|
+
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
17
17
|
|
|
18
18
|
import { type ProverClientConfig } from '../config.js';
|
|
19
19
|
import { ProvingOrchestrator } from '../orchestrator/orchestrator.js';
|
|
20
20
|
import { BrokerCircuitProverFacade } from '../proving_broker/broker_prover_facade.js';
|
|
21
|
-
import { InlineProofStore } from '../proving_broker/proof_store.js';
|
|
21
|
+
import { InlineProofStore, type ProofStore, createProofStore } from '../proving_broker/proof_store/index.js';
|
|
22
22
|
import { ProvingAgent } from '../proving_broker/proving_agent.js';
|
|
23
23
|
import { ServerEpochProver } from './server-epoch-prover.js';
|
|
24
24
|
|
|
@@ -27,21 +27,24 @@ export class ProverClient implements EpochProverManager {
|
|
|
27
27
|
private running = false;
|
|
28
28
|
private agents: ProvingAgent[] = [];
|
|
29
29
|
|
|
30
|
+
private proofStore: ProofStore;
|
|
31
|
+
private failedProofStore: ProofStore | undefined;
|
|
32
|
+
|
|
30
33
|
private constructor(
|
|
31
34
|
private config: ProverClientConfig,
|
|
32
35
|
private worldState: ForkMerkleTreeOperations,
|
|
33
|
-
private telemetry: TelemetryClient,
|
|
34
36
|
private orchestratorClient: ProvingJobProducer,
|
|
35
37
|
private agentClient?: ProvingJobConsumer,
|
|
38
|
+
private telemetry: TelemetryClient = getTelemetryClient(),
|
|
36
39
|
private log = createLogger('prover-client:tx-prover'),
|
|
37
40
|
) {
|
|
38
|
-
|
|
39
|
-
|
|
41
|
+
this.proofStore = new InlineProofStore();
|
|
42
|
+
this.failedProofStore = this.config.failedProofStore ? createProofStore(this.config.failedProofStore) : undefined;
|
|
40
43
|
}
|
|
41
44
|
|
|
42
45
|
public createEpochProver(): EpochProver {
|
|
43
|
-
const facade = new BrokerCircuitProverFacade(this.orchestratorClient);
|
|
44
|
-
const orchestrator = new ProvingOrchestrator(this.worldState, facade, this.
|
|
46
|
+
const facade = new BrokerCircuitProverFacade(this.orchestratorClient, this.proofStore, this.failedProofStore);
|
|
47
|
+
const orchestrator = new ProvingOrchestrator(this.worldState, facade, this.config.proverId, this.telemetry);
|
|
45
48
|
return new ServerEpochProver(facade, orchestrator);
|
|
46
49
|
}
|
|
47
50
|
|
|
@@ -60,10 +63,6 @@ export class ProverClient implements EpochProverManager {
|
|
|
60
63
|
await this.createAndStartAgents();
|
|
61
64
|
}
|
|
62
65
|
|
|
63
|
-
if (!this.config.realProofs && newConfig.realProofs) {
|
|
64
|
-
// TODO(palla/prover-node): Reset padding tx here once we cache it at this class
|
|
65
|
-
}
|
|
66
|
-
|
|
67
66
|
this.config = newConfig;
|
|
68
67
|
}
|
|
69
68
|
|
|
@@ -100,9 +99,9 @@ export class ProverClient implements EpochProverManager {
|
|
|
100
99
|
config: ProverClientConfig,
|
|
101
100
|
worldState: ForkMerkleTreeOperations,
|
|
102
101
|
broker: ProvingJobBroker,
|
|
103
|
-
telemetry: TelemetryClient,
|
|
102
|
+
telemetry: TelemetryClient = getTelemetryClient(),
|
|
104
103
|
) {
|
|
105
|
-
const prover = new ProverClient(config, worldState,
|
|
104
|
+
const prover = new ProverClient(config, worldState, broker, broker, telemetry);
|
|
106
105
|
await prover.start();
|
|
107
106
|
return prover;
|
|
108
107
|
}
|
|
@@ -133,9 +132,9 @@ export class ProverClient implements EpochProverManager {
|
|
|
133
132
|
this.agentClient!,
|
|
134
133
|
proofStore,
|
|
135
134
|
prover,
|
|
136
|
-
this.telemetry,
|
|
137
135
|
[],
|
|
138
136
|
this.config.proverAgentPollIntervalMs,
|
|
137
|
+
this.telemetry,
|
|
139
138
|
),
|
|
140
139
|
);
|
|
141
140
|
|
|
@@ -159,5 +158,5 @@ export function buildServerCircuitProver(
|
|
|
159
158
|
? new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath)
|
|
160
159
|
: undefined;
|
|
161
160
|
|
|
162
|
-
return Promise.resolve(new TestCircuitProver(
|
|
161
|
+
return Promise.resolve(new TestCircuitProver(simulationProvider, config, telemetry));
|
|
163
162
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
type ProofAndVerificationKey,
|
|
3
|
+
type ProofUri,
|
|
3
4
|
type ProvingJobId,
|
|
4
5
|
type ProvingJobInputsMap,
|
|
5
6
|
type ProvingJobProducer,
|
|
@@ -41,10 +42,7 @@ import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise'
|
|
|
41
42
|
import { SerialQueue } from '@aztec/foundation/queue';
|
|
42
43
|
import { truncate } from '@aztec/foundation/string';
|
|
43
44
|
|
|
44
|
-
import { InlineProofStore, type ProofStore } from './proof_store.js';
|
|
45
|
-
|
|
46
|
-
// 20 minutes, roughly the length of an Aztec epoch. If a proof isn't ready in this amount of time then we've failed to prove the whole epoch
|
|
47
|
-
const MAX_WAIT_MS = 1_200_000;
|
|
45
|
+
import { InlineProofStore, type ProofStore } from './proof_store/index.js';
|
|
48
46
|
|
|
49
47
|
// Perform a snapshot sync every 30 seconds
|
|
50
48
|
const SNAPSHOT_SYNC_INTERVAL_MS = 30_000;
|
|
@@ -55,6 +53,7 @@ const SNAPSHOT_SYNC_CHECK_MAX_REQUEST_SIZE = 1000;
|
|
|
55
53
|
type ProvingJob = {
|
|
56
54
|
id: ProvingJobId;
|
|
57
55
|
type: ProvingRequestType;
|
|
56
|
+
inputsUri: ProofUri;
|
|
58
57
|
promise: PromiseWithResolvers<any>;
|
|
59
58
|
abortFn?: () => Promise<void>;
|
|
60
59
|
signal?: AbortSignal;
|
|
@@ -70,7 +69,7 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
|
|
|
70
69
|
constructor(
|
|
71
70
|
private broker: ProvingJobProducer,
|
|
72
71
|
private proofStore: ProofStore = new InlineProofStore(),
|
|
73
|
-
private
|
|
72
|
+
private failedProofStore?: ProofStore,
|
|
74
73
|
private pollIntervalMs = 1000,
|
|
75
74
|
private log = createLogger('prover-client:broker-circuit-prover-facade'),
|
|
76
75
|
) {}
|
|
@@ -125,6 +124,7 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
|
|
|
125
124
|
const job: ProvingJob = {
|
|
126
125
|
id,
|
|
127
126
|
type,
|
|
127
|
+
inputsUri,
|
|
128
128
|
promise,
|
|
129
129
|
abortFn,
|
|
130
130
|
signal,
|
|
@@ -335,6 +335,9 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
|
|
|
335
335
|
`Resolving proving job with error id=${job.id} type=${ProvingRequestType[job.type]}`,
|
|
336
336
|
result.reason,
|
|
337
337
|
);
|
|
338
|
+
if (result.reason !== 'Aborted') {
|
|
339
|
+
void this.backupFailedProofInputs(job);
|
|
340
|
+
}
|
|
338
341
|
job.promise.reject(new Error(result.reason));
|
|
339
342
|
}
|
|
340
343
|
|
|
@@ -365,6 +368,26 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
|
|
|
365
368
|
}
|
|
366
369
|
}
|
|
367
370
|
|
|
371
|
+
private async backupFailedProofInputs(job: ProvingJob) {
|
|
372
|
+
try {
|
|
373
|
+
if (!this.failedProofStore) {
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
const inputs = await this.proofStore.getProofInput(job.inputsUri);
|
|
377
|
+
const uri = await this.failedProofStore.saveProofInput(job.id, inputs.type, inputs.inputs);
|
|
378
|
+
this.log.info(`Stored proof inputs for failed job id=${job.id} type=${ProvingRequestType[job.type]} at ${uri}`, {
|
|
379
|
+
id: job.id,
|
|
380
|
+
type: job.type,
|
|
381
|
+
uri,
|
|
382
|
+
});
|
|
383
|
+
} catch (err) {
|
|
384
|
+
this.log.error(
|
|
385
|
+
`Error backing up proof inputs for failed job id=${job.id} type=${ProvingRequestType[job.type]}`,
|
|
386
|
+
err,
|
|
387
|
+
);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
368
391
|
private async monitorForCompletedJobs() {
|
|
369
392
|
// Monitoring for completed jobs involves 2 stages.
|
|
370
393
|
|
|
@@ -11,11 +11,15 @@ export async function createAndStartProvingBroker(
|
|
|
11
11
|
): Promise<ProvingBroker> {
|
|
12
12
|
const database = config.dataDirectory ? await KVBrokerDatabase.new(config, client) : new InMemoryBrokerDatabase();
|
|
13
13
|
|
|
14
|
-
const broker = new ProvingBroker(
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
const broker = new ProvingBroker(
|
|
15
|
+
database,
|
|
16
|
+
{
|
|
17
|
+
jobTimeoutMs: config.proverBrokerJobTimeoutMs,
|
|
18
|
+
maxRetries: config.proverBrokerJobMaxRetries,
|
|
19
|
+
timeoutIntervalMs: config.proverBrokerPollIntervalMs,
|
|
20
|
+
},
|
|
21
|
+
client,
|
|
22
|
+
);
|
|
19
23
|
|
|
20
24
|
await broker.start();
|
|
21
25
|
return broker;
|
|
@@ -4,6 +4,6 @@ export * from './rpc.js';
|
|
|
4
4
|
export * from './proving_broker_database.js';
|
|
5
5
|
export * from './proving_broker_database/memory.js';
|
|
6
6
|
export * from './proving_broker_database/persisted.js';
|
|
7
|
-
export * from './proof_store.js';
|
|
7
|
+
export * from './proof_store/index.js';
|
|
8
8
|
export * from './factory.js';
|
|
9
9
|
export * from './config.js';
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
2
|
+
|
|
3
|
+
import { GoogleCloudStorageProofStore } from './gcs_proof_store.js';
|
|
4
|
+
import { InlineProofStore } from './inline_proof_store.js';
|
|
5
|
+
import { type ProofStore } from './proof_store.js';
|
|
6
|
+
|
|
7
|
+
export function createProofStore(config: string | undefined, logger = createLogger('prover-client:proof-store')) {
|
|
8
|
+
if (config === undefined) {
|
|
9
|
+
logger.info('Creating inline proof store');
|
|
10
|
+
return new InlineProofStore();
|
|
11
|
+
} else if (config.startsWith('gs://')) {
|
|
12
|
+
try {
|
|
13
|
+
const url = new URL(config);
|
|
14
|
+
const bucket = url.host;
|
|
15
|
+
const path = url.pathname.replace(/^\/+/, '');
|
|
16
|
+
logger.info(`Creating google cloud proof store at ${bucket}`, { bucket, path });
|
|
17
|
+
return new GoogleCloudStorageProofStore(bucket, path);
|
|
18
|
+
} catch (err) {
|
|
19
|
+
throw new Error(
|
|
20
|
+
`Invalid google cloud proof store definition: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`,
|
|
21
|
+
);
|
|
22
|
+
}
|
|
23
|
+
} else {
|
|
24
|
+
throw new Error(`Unknown proof store config: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function createProofStoreForUri(
|
|
29
|
+
uri: string,
|
|
30
|
+
logger = createLogger('prover-client:proof-store'),
|
|
31
|
+
): Pick<ProofStore, 'getProofInput' | 'getProofOutput'> {
|
|
32
|
+
if (uri.startsWith('data://')) {
|
|
33
|
+
return createProofStore(undefined, logger);
|
|
34
|
+
} else if (uri.startsWith('gs://')) {
|
|
35
|
+
const url = new URL(uri);
|
|
36
|
+
const basePath = url.pathname.replace(/^\/+/, '').split('/').slice(0, -3);
|
|
37
|
+
url.pathname = basePath.join('/');
|
|
38
|
+
return createProofStore(uri, logger);
|
|
39
|
+
} else {
|
|
40
|
+
throw new Error(`Unknown proof store config: '${uri}'. Supported protocols are 'data://' and 'gs://'.`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type ProofUri,
|
|
3
|
+
type ProvingJobId,
|
|
4
|
+
type ProvingJobInputs,
|
|
5
|
+
type ProvingJobInputsMap,
|
|
6
|
+
type ProvingJobResult,
|
|
7
|
+
type ProvingJobResultsMap,
|
|
8
|
+
ProvingRequestType,
|
|
9
|
+
getProvingJobInputClassFor,
|
|
10
|
+
} from '@aztec/circuit-types';
|
|
11
|
+
|
|
12
|
+
import { Storage } from '@google-cloud/storage';
|
|
13
|
+
import { join } from 'path';
|
|
14
|
+
|
|
15
|
+
import { type ProofStore } from './proof_store.js';
|
|
16
|
+
|
|
17
|
+
const INPUTS_PATH = 'inputs';
|
|
18
|
+
|
|
19
|
+
export class GoogleCloudStorageProofStore implements ProofStore {
|
|
20
|
+
private readonly storage: Storage;
|
|
21
|
+
|
|
22
|
+
constructor(private readonly bucketName: string, private readonly path: string) {
|
|
23
|
+
this.storage = new Storage();
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
public async saveProofInput<T extends ProvingRequestType>(
|
|
27
|
+
id: ProvingJobId,
|
|
28
|
+
type: T,
|
|
29
|
+
inputs: ProvingJobInputsMap[T],
|
|
30
|
+
): Promise<ProofUri> {
|
|
31
|
+
const path = join(this.path, INPUTS_PATH, ProvingRequestType[type], id);
|
|
32
|
+
const file = this.storage.bucket(this.bucketName).file(path);
|
|
33
|
+
await file.save(inputs.toBuffer());
|
|
34
|
+
return file.cloudStorageURI.toString() as ProofUri;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
saveProofOutput<T extends ProvingRequestType>(
|
|
38
|
+
_id: ProvingJobId,
|
|
39
|
+
_type: T,
|
|
40
|
+
_result: ProvingJobResultsMap[T],
|
|
41
|
+
): Promise<ProofUri> {
|
|
42
|
+
throw new Error('Not implemented');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
public async getProofInput(uri: ProofUri): Promise<ProvingJobInputs> {
|
|
46
|
+
try {
|
|
47
|
+
const url = new URL(uri);
|
|
48
|
+
const bucket = this.storage.bucket(url.host);
|
|
49
|
+
const path = url.pathname.replace(/^\/+/, '');
|
|
50
|
+
const file = bucket.file(path);
|
|
51
|
+
if (!(await file.exists())) {
|
|
52
|
+
throw new Error(`File at ${uri} does not exist`);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const typeString = path.split('/').at(-2);
|
|
56
|
+
const type = typeString ? ProvingRequestType[typeString as keyof typeof ProvingRequestType] : undefined;
|
|
57
|
+
if (type === undefined) {
|
|
58
|
+
throw new Error(`Unrecognized proof type ${type} in path ${path}`);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const contents = await file.download();
|
|
62
|
+
const inputs = getProvingJobInputClassFor(type).fromBuffer(contents[0]);
|
|
63
|
+
return { inputs, type } as ProvingJobInputs;
|
|
64
|
+
} catch (err) {
|
|
65
|
+
throw new Error(`Error getting proof input at ${uri}: ${err}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
getProofOutput(_uri: ProofUri): Promise<ProvingJobResult> {
|
|
70
|
+
throw new Error('Not implemented');
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -10,50 +10,7 @@ import {
|
|
|
10
10
|
import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
|
|
11
11
|
import { type ZodFor } from '@aztec/foundation/schemas';
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
* A database for storing proof inputs and outputs.
|
|
15
|
-
*/
|
|
16
|
-
export interface ProofStore {
|
|
17
|
-
/**
|
|
18
|
-
* Save a proof input to the database.
|
|
19
|
-
* @param jobId - The ID of the job the proof input is associated with.
|
|
20
|
-
* @param type - The type of the proving request.
|
|
21
|
-
* @param inputs - The proof input to save.
|
|
22
|
-
* @returns The URI of the saved proof input.
|
|
23
|
-
*/
|
|
24
|
-
saveProofInput<T extends ProvingRequestType>(
|
|
25
|
-
jobId: ProvingJobId,
|
|
26
|
-
type: T,
|
|
27
|
-
inputs: ProvingJobInputsMap[T],
|
|
28
|
-
): Promise<ProofUri>;
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Save a proof output to the database.
|
|
32
|
-
* @param jobId - The ID of the job the proof input is associated with.
|
|
33
|
-
* @param type - The type of the proving request.
|
|
34
|
-
* @param result - The proof output to save.
|
|
35
|
-
* @returns The URI of the saved proof output.
|
|
36
|
-
*/
|
|
37
|
-
saveProofOutput<T extends ProvingRequestType>(
|
|
38
|
-
id: ProvingJobId,
|
|
39
|
-
type: T,
|
|
40
|
-
result: ProvingJobResultsMap[T],
|
|
41
|
-
): Promise<ProofUri>;
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Retrieve a proof input from the database.
|
|
45
|
-
* @param uri - The URI of the proof input to retrieve.
|
|
46
|
-
* @returns The proof input.
|
|
47
|
-
*/
|
|
48
|
-
getProofInput(uri: ProofUri): Promise<ProvingJobInputs>;
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Retrieve a proof output from the database.
|
|
52
|
-
* @param uri - The URI of the proof output to retrieve.
|
|
53
|
-
* @returns The proof output.
|
|
54
|
-
*/
|
|
55
|
-
getProofOutput(uri: ProofUri): Promise<ProvingJobResult>;
|
|
56
|
-
}
|
|
13
|
+
import { type ProofStore } from './proof_store.js';
|
|
57
14
|
|
|
58
15
|
// use an ASCII encoded data uri https://datatracker.ietf.org/doc/html/rfc2397#section-2
|
|
59
16
|
// we do this to avoid double encoding to base64 (since the inputs already serialize to a base64 string)
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type ProofUri,
|
|
3
|
+
type ProvingJobId,
|
|
4
|
+
type ProvingJobInputs,
|
|
5
|
+
type ProvingJobInputsMap,
|
|
6
|
+
type ProvingJobResult,
|
|
7
|
+
type ProvingJobResultsMap,
|
|
8
|
+
type ProvingRequestType,
|
|
9
|
+
} from '@aztec/circuit-types';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* A database for storing proof inputs and outputs.
|
|
13
|
+
*/
|
|
14
|
+
export interface ProofStore {
|
|
15
|
+
/**
|
|
16
|
+
* Save a proof input to the database.
|
|
17
|
+
* @param jobId - The ID of the job the proof input is associated with.
|
|
18
|
+
* @param type - The type of the proving request.
|
|
19
|
+
* @param inputs - The proof input to save.
|
|
20
|
+
* @returns The URI of the saved proof input.
|
|
21
|
+
*/
|
|
22
|
+
saveProofInput<T extends ProvingRequestType>(
|
|
23
|
+
jobId: ProvingJobId,
|
|
24
|
+
type: T,
|
|
25
|
+
inputs: ProvingJobInputsMap[T],
|
|
26
|
+
): Promise<ProofUri>;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Save a proof output to the database.
|
|
30
|
+
* @param jobId - The ID of the job the proof input is associated with.
|
|
31
|
+
* @param type - The type of the proving request.
|
|
32
|
+
* @param result - The proof output to save.
|
|
33
|
+
* @returns The URI of the saved proof output.
|
|
34
|
+
*/
|
|
35
|
+
saveProofOutput<T extends ProvingRequestType>(
|
|
36
|
+
id: ProvingJobId,
|
|
37
|
+
type: T,
|
|
38
|
+
result: ProvingJobResultsMap[T],
|
|
39
|
+
): Promise<ProofUri>;
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Retrieve a proof input from the database.
|
|
43
|
+
* @param uri - The URI of the proof input to retrieve.
|
|
44
|
+
* @returns The proof input.
|
|
45
|
+
*/
|
|
46
|
+
getProofInput(uri: ProofUri): Promise<ProvingJobInputs>;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Retrieve a proof output from the database.
|
|
50
|
+
* @param uri - The URI of the proof output to retrieve.
|
|
51
|
+
* @returns The proof output.
|
|
52
|
+
*/
|
|
53
|
+
getProofOutput(uri: ProofUri): Promise<ProvingJobResult>;
|
|
54
|
+
}
|
|
@@ -12,9 +12,15 @@ import { createLogger } from '@aztec/foundation/log';
|
|
|
12
12
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
13
13
|
import { truncate } from '@aztec/foundation/string';
|
|
14
14
|
import { Timer } from '@aztec/foundation/timer';
|
|
15
|
-
import {
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
import {
|
|
16
|
+
type TelemetryClient,
|
|
17
|
+
type Traceable,
|
|
18
|
+
type Tracer,
|
|
19
|
+
getTelemetryClient,
|
|
20
|
+
trackSpan,
|
|
21
|
+
} from '@aztec/telemetry-client';
|
|
22
|
+
|
|
23
|
+
import { type ProofStore } from './proof_store/index.js';
|
|
18
24
|
import { ProvingAgentInstrumentation } from './proving_agent_instrumentation.js';
|
|
19
25
|
import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_controller.js';
|
|
20
26
|
|
|
@@ -36,12 +42,12 @@ export class ProvingAgent implements Traceable {
|
|
|
36
42
|
private proofStore: ProofStore,
|
|
37
43
|
/** The prover implementation to defer jobs to */
|
|
38
44
|
private circuitProver: ServerCircuitProver,
|
|
39
|
-
/** A telemetry client through which to emit metrics */
|
|
40
|
-
client: TelemetryClient,
|
|
41
45
|
/** Optional list of allowed proof types to build */
|
|
42
46
|
private proofAllowList: Array<ProvingRequestType> = [],
|
|
43
47
|
/** How long to wait between jobs */
|
|
44
48
|
private pollIntervalMs = 1000,
|
|
49
|
+
/** A telemetry client through which to emit metrics */
|
|
50
|
+
client: TelemetryClient = getTelemetryClient(),
|
|
45
51
|
private log = createLogger('prover-client:proving-agent'),
|
|
46
52
|
) {
|
|
47
53
|
this.tracer = client.getTracer('ProvingAgent');
|
|
@@ -13,7 +13,13 @@ import { createLogger } from '@aztec/foundation/log';
|
|
|
13
13
|
import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
14
14
|
import { PriorityMemoryQueue, SerialQueue } from '@aztec/foundation/queue';
|
|
15
15
|
import { Timer } from '@aztec/foundation/timer';
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
type TelemetryClient,
|
|
18
|
+
type Traceable,
|
|
19
|
+
type Tracer,
|
|
20
|
+
getTelemetryClient,
|
|
21
|
+
trackSpan,
|
|
22
|
+
} from '@aztec/telemetry-client';
|
|
17
23
|
|
|
18
24
|
import assert from 'assert';
|
|
19
25
|
|
|
@@ -108,13 +114,13 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
108
114
|
|
|
109
115
|
public constructor(
|
|
110
116
|
private database: ProvingBrokerDatabase,
|
|
111
|
-
client: TelemetryClient,
|
|
112
117
|
{
|
|
113
118
|
jobTimeoutMs = 30_000,
|
|
114
119
|
timeoutIntervalMs = 10_000,
|
|
115
120
|
maxRetries = 3,
|
|
116
121
|
maxEpochsToKeepResultsFor = 1,
|
|
117
122
|
}: ProofRequestBrokerConfig = {},
|
|
123
|
+
client: TelemetryClient = getTelemetryClient(),
|
|
118
124
|
private logger = createLogger('prover-client:proving-broker'),
|
|
119
125
|
) {
|
|
120
126
|
this.tracer = client.getTracer('ProvingBroker');
|
|
@@ -9,7 +9,7 @@ import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
|
|
|
9
9
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
10
10
|
import { type AztecMap } from '@aztec/kv-store';
|
|
11
11
|
import { AztecLmdbStore } from '@aztec/kv-store/lmdb';
|
|
12
|
-
import { Attributes, LmdbMetrics, type TelemetryClient } from '@aztec/telemetry-client';
|
|
12
|
+
import { Attributes, LmdbMetrics, type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
13
13
|
|
|
14
14
|
import { mkdir, readdir } from 'fs/promises';
|
|
15
15
|
import { join } from 'path';
|
|
@@ -68,7 +68,7 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
68
68
|
private constructor(
|
|
69
69
|
private epochs: Map<number, SingleEpochDatabase>,
|
|
70
70
|
private config: ProverBrokerConfig,
|
|
71
|
-
client: TelemetryClient,
|
|
71
|
+
client: TelemetryClient = getTelemetryClient(),
|
|
72
72
|
private logger: Logger,
|
|
73
73
|
) {
|
|
74
74
|
this.metrics = new LmdbMetrics(
|
|
@@ -91,7 +91,7 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
91
91
|
|
|
92
92
|
public static async new(
|
|
93
93
|
config: ProverBrokerConfig,
|
|
94
|
-
client: TelemetryClient,
|
|
94
|
+
client: TelemetryClient = getTelemetryClient(),
|
|
95
95
|
logger = createLogger('prover-client:proving-broker-database'),
|
|
96
96
|
) {
|
|
97
97
|
const epochs: Map<number, SingleEpochDatabase> = new Map<number, SingleEpochDatabase>();
|
package/src/test/mock_prover.ts
CHANGED
|
@@ -43,15 +43,14 @@ import {
|
|
|
43
43
|
makeRootRollupPublicInputs,
|
|
44
44
|
} from '@aztec/circuits.js/testing';
|
|
45
45
|
import { times } from '@aztec/foundation/collection';
|
|
46
|
-
import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
|
|
47
46
|
|
|
48
|
-
import { InlineProofStore, type ProofStore } from '../proving_broker/proof_store.js';
|
|
47
|
+
import { InlineProofStore, type ProofStore } from '../proving_broker/proof_store/index.js';
|
|
49
48
|
import { ProvingAgent } from '../proving_broker/proving_agent.js';
|
|
50
49
|
import { ProvingBroker } from '../proving_broker/proving_broker.js';
|
|
51
50
|
import { InMemoryBrokerDatabase } from '../proving_broker/proving_broker_database/memory.js';
|
|
52
51
|
|
|
53
52
|
export class TestBroker implements ProvingJobProducer {
|
|
54
|
-
private broker = new ProvingBroker(new InMemoryBrokerDatabase()
|
|
53
|
+
private broker = new ProvingBroker(new InMemoryBrokerDatabase());
|
|
55
54
|
private agents: ProvingAgent[];
|
|
56
55
|
|
|
57
56
|
constructor(
|
|
@@ -62,7 +61,7 @@ export class TestBroker implements ProvingJobProducer {
|
|
|
62
61
|
) {
|
|
63
62
|
this.agents = times(
|
|
64
63
|
agentCount,
|
|
65
|
-
() => new ProvingAgent(this.broker, proofStore, prover,
|
|
64
|
+
() => new ProvingAgent(this.broker, proofStore, prover, undefined, agentPollInterval),
|
|
66
65
|
);
|
|
67
66
|
}
|
|
68
67
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"proof_store.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proof_store.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,gBAAgB,EAChB,KAAK,mBAAmB,EACxB,gBAAgB,EAChB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACxB,MAAM,sBAAsB,CAAC;AAI9B;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;;;OAMG;IACH,cAAc,CAAC,CAAC,SAAS,kBAAkB,EACzC,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,CAAC,EACP,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAC7B,OAAO,CAAC,QAAQ,CAAC,CAAC;IAErB;;;;;;OAMG;IACH,eAAe,CAAC,CAAC,SAAS,kBAAkB,EAC1C,EAAE,EAAE,YAAY,EAChB,IAAI,EAAE,CAAC,EACP,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC9B,OAAO,CAAC,QAAQ,CAAC,CAAC;IAErB;;;;OAIG;IACH,aAAa,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAExD;;;;OAIG;IACH,cAAc,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC1D;AAOD;;GAEG;AACH,qBAAa,gBAAiB,YAAW,UAAU;IACjD,cAAc,CAAC,CAAC,SAAS,kBAAkB,EACzC,GAAG,EAAE,YAAY,EACjB,IAAI,EAAE,CAAC,EACP,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAC7B,OAAO,CAAC,QAAQ,CAAC;IAKpB,eAAe,CAAC,CAAC,SAAS,kBAAkB,EAC1C,GAAG,EAAE,YAAY,EACjB,IAAI,EAAE,CAAC,EACP,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC9B,OAAO,CAAC,QAAQ,CAAC;IAKpB,aAAa,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIvD,cAAc,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIxD,OAAO,CAAC,MAAM;IAKd,OAAO,CAAC,MAAM;CAQf"}
|