@aztec/prover-client 0.0.0-test.1 → 0.0.1-commit.b655e406
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/block-factory/index.d.ts +2 -0
- package/dest/block-factory/index.d.ts.map +1 -0
- package/dest/block-factory/light.d.ts +38 -0
- package/dest/block-factory/light.d.ts.map +1 -0
- package/dest/block-factory/light.js +94 -0
- package/dest/config.d.ts +6 -6
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +11 -1
- package/dest/mocks/fixtures.d.ts +7 -4
- package/dest/mocks/fixtures.d.ts.map +1 -1
- package/dest/mocks/fixtures.js +32 -4
- package/dest/mocks/test_context.d.ts +43 -15
- package/dest/mocks/test_context.d.ts.map +1 -1
- package/dest/mocks/test_context.js +110 -48
- package/dest/orchestrator/block-building-helpers.d.ts +37 -28
- package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
- package/dest/orchestrator/block-building-helpers.js +156 -150
- package/dest/orchestrator/block-proving-state.d.ts +62 -46
- package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/block-proving-state.js +223 -179
- package/dest/orchestrator/checkpoint-proving-state.d.ts +63 -0
- package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -0
- package/dest/orchestrator/checkpoint-proving-state.js +211 -0
- package/dest/orchestrator/epoch-proving-state.d.ts +37 -24
- package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/epoch-proving-state.js +143 -73
- package/dest/orchestrator/orchestrator.d.ts +34 -31
- package/dest/orchestrator/orchestrator.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator.js +392 -234
- package/dest/orchestrator/orchestrator_metrics.d.ts +2 -0
- package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator_metrics.js +9 -0
- package/dest/orchestrator/tx-proving-state.d.ts +12 -10
- package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/tx-proving-state.js +30 -38
- package/dest/prover-client/prover-client.d.ts +3 -3
- package/dest/prover-client/prover-client.d.ts.map +1 -1
- package/dest/prover-client/prover-client.js +5 -4
- package/dest/prover-client/server-epoch-prover.d.ts +13 -10
- package/dest/prover-client/server-epoch-prover.d.ts.map +1 -1
- package/dest/prover-client/server-epoch-prover.js +11 -11
- package/dest/proving_broker/broker_prover_facade.d.ts +22 -15
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
- package/dest/proving_broker/broker_prover_facade.js +64 -39
- package/dest/proving_broker/config.d.ts +9 -4
- package/dest/proving_broker/config.d.ts.map +1 -1
- package/dest/proving_broker/config.js +15 -4
- package/dest/proving_broker/factory.d.ts +1 -1
- package/dest/proving_broker/factory.d.ts.map +1 -1
- package/dest/proving_broker/factory.js +5 -1
- package/dest/proving_broker/fixtures.js +1 -1
- package/dest/proving_broker/proof_store/factory.js +1 -1
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/gcs_proof_store.js +1 -0
- package/dest/proving_broker/proof_store/index.d.ts +1 -0
- package/dest/proving_broker/proof_store/index.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/index.js +1 -0
- package/dest/proving_broker/proving_agent.d.ts +3 -3
- package/dest/proving_broker/proving_agent.d.ts.map +1 -1
- package/dest/proving_broker/proving_agent.js +83 -47
- package/dest/proving_broker/proving_broker.d.ts +11 -2
- package/dest/proving_broker/proving_broker.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.js +34 -22
- package/dest/proving_broker/proving_broker_database/memory.js +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.js +9 -8
- package/dest/proving_broker/proving_job_controller.d.ts +7 -8
- package/dest/proving_broker/proving_job_controller.d.ts.map +1 -1
- package/dest/proving_broker/proving_job_controller.js +89 -61
- package/dest/proving_broker/rpc.d.ts +3 -5
- package/dest/proving_broker/rpc.d.ts.map +1 -1
- package/dest/proving_broker/rpc.js +1 -4
- package/dest/test/mock_proof_store.d.ts +9 -0
- package/dest/test/mock_proof_store.d.ts.map +1 -0
- package/dest/test/mock_proof_store.js +10 -0
- package/dest/test/mock_prover.d.ts +23 -16
- package/dest/test/mock_prover.d.ts.map +1 -1
- package/dest/test/mock_prover.js +38 -20
- package/package.json +29 -29
- package/src/block-factory/index.ts +1 -0
- package/src/block-factory/light.ts +140 -0
- package/src/config.ts +24 -8
- package/src/mocks/fixtures.ts +43 -15
- package/src/mocks/test_context.ts +201 -75
- package/src/orchestrator/block-building-helpers.ts +247 -243
- package/src/orchestrator/block-proving-state.ts +247 -231
- package/src/orchestrator/checkpoint-proving-state.ts +299 -0
- package/src/orchestrator/epoch-proving-state.ts +187 -111
- package/src/orchestrator/orchestrator.ts +590 -289
- package/src/orchestrator/orchestrator_metrics.ts +20 -1
- package/src/orchestrator/tx-proving-state.ts +60 -61
- package/src/prover-client/prover-client.ts +16 -14
- package/src/prover-client/server-epoch-prover.ts +40 -21
- package/src/proving_broker/broker_prover_facade.ts +200 -113
- package/src/proving_broker/config.ts +17 -6
- package/src/proving_broker/factory.ts +2 -1
- package/src/proving_broker/fixtures.ts +1 -1
- package/src/proving_broker/proof_store/factory.ts +1 -1
- package/src/proving_broker/proof_store/gcs_proof_store.ts +5 -1
- package/src/proving_broker/proof_store/index.ts +1 -0
- package/src/proving_broker/proof_store/inline_proof_store.ts +1 -1
- package/src/proving_broker/proving_agent.ts +89 -47
- package/src/proving_broker/proving_broker.ts +51 -32
- package/src/proving_broker/proving_broker_database/memory.ts +1 -1
- package/src/proving_broker/proving_broker_database/persisted.ts +9 -8
- package/src/proving_broker/proving_job_controller.ts +92 -81
- package/src/proving_broker/rpc.ts +1 -6
- package/src/test/mock_proof_store.ts +14 -0
- package/src/test/mock_prover.ts +164 -60
- package/dest/bin/get-proof-inputs.d.ts +0 -2
- package/dest/bin/get-proof-inputs.d.ts.map +0 -1
- package/dest/bin/get-proof-inputs.js +0 -51
- package/dest/block_builder/index.d.ts +0 -6
- package/dest/block_builder/index.d.ts.map +0 -1
- package/dest/block_builder/light.d.ts +0 -33
- package/dest/block_builder/light.d.ts.map +0 -1
- package/dest/block_builder/light.js +0 -82
- package/src/bin/get-proof-inputs.ts +0 -59
- package/src/block_builder/index.ts +0 -6
- package/src/block_builder/light.ts +0 -101
- /package/dest/{block_builder → block-factory}/index.js +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { sha256 } from '@aztec/foundation/crypto';
|
|
2
|
+
import { Fr } from '@aztec/foundation/fields';
|
|
2
3
|
import { createLogger } from '@aztec/foundation/log';
|
|
3
4
|
import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
4
5
|
import { truncate } from '@aztec/foundation/string';
|
|
@@ -49,7 +50,13 @@ export class BrokerCircuitProverFacade {
|
|
|
49
50
|
}
|
|
50
51
|
// Create a promise for this job id, regardless of whether it was enqueued at the broker
|
|
51
52
|
// The running promise will monitor for the job to be completed and resolve it either way
|
|
53
|
+
// We install an error handler to prevent unhandled rejections in the process before the
|
|
54
|
+
// job promise is awaited by the caller (see #13166)
|
|
52
55
|
const promise = promiseWithResolvers();
|
|
56
|
+
promise.promise.catch((err)=>this.log.error(`Job errored with '${err.message ?? err}' id=${id} type=${ProvingRequestType[type]}`, {
|
|
57
|
+
provingJobId: id,
|
|
58
|
+
provingJobType: ProvingRequestType[type]
|
|
59
|
+
}));
|
|
53
60
|
const abortFn = ()=>{
|
|
54
61
|
signal?.removeEventListener('abort', abortFn);
|
|
55
62
|
void this.broker.cancelProvingJob(id).catch((err)=>this.log.warn(`Error cancelling job id=${id}`, err));
|
|
@@ -75,23 +82,28 @@ export class BrokerCircuitProverFacade {
|
|
|
75
82
|
try {
|
|
76
83
|
const inputsUri = await this.proofStore.saveProofInput(id, type, inputs);
|
|
77
84
|
job.inputsUri = inputsUri;
|
|
85
|
+
// Send the job to the broker
|
|
78
86
|
const jobStatus = await this.broker.enqueueProvingJob({
|
|
79
87
|
id,
|
|
80
88
|
type,
|
|
81
89
|
inputsUri,
|
|
82
90
|
epochNumber
|
|
83
91
|
});
|
|
92
|
+
const jobLogText = `id=${id} type=${ProvingRequestType[type]} epochNumber=${epochNumber}`;
|
|
93
|
+
const jobLogData = {
|
|
94
|
+
provingJobId: id,
|
|
95
|
+
provingJobType: ProvingRequestType[type],
|
|
96
|
+
epochNumber,
|
|
97
|
+
inputsUri: truncate(inputsUri),
|
|
98
|
+
status: jobStatus.status,
|
|
99
|
+
numOutstandingJobs: this.jobs.size
|
|
100
|
+
};
|
|
84
101
|
// If we are here then the job was successfully accepted by the broker
|
|
85
102
|
// the returned status is for before any action was performed
|
|
86
103
|
if (jobStatus.status === 'fulfilled' || jobStatus.status === 'rejected') {
|
|
87
104
|
// Job was already completed by the broker
|
|
88
105
|
// No need to notify the broker on aborted job
|
|
89
|
-
this.log.verbose(`Job already completed when sent to broker
|
|
90
|
-
provingJobId: id,
|
|
91
|
-
provingJobType: ProvingRequestType[type],
|
|
92
|
-
epochNumber,
|
|
93
|
-
inputsUri: truncate(inputsUri)
|
|
94
|
-
});
|
|
106
|
+
this.log.verbose(`Job already completed when sent to broker ${jobLogText}`, jobLogData);
|
|
95
107
|
// Job was not enqueued. It must be completed already, add to our set of already completed jobs
|
|
96
108
|
this.jobsToRetrieve.add(id);
|
|
97
109
|
} else {
|
|
@@ -99,21 +111,10 @@ export class BrokerCircuitProverFacade {
|
|
|
99
111
|
signal?.addEventListener('abort', job.abortFn);
|
|
100
112
|
// Job added for the first time
|
|
101
113
|
if (jobStatus.status === 'not-found') {
|
|
102
|
-
this.log.verbose(`Job enqueued with broker
|
|
103
|
-
provingJobId: id,
|
|
104
|
-
provingJobType: ProvingRequestType[type],
|
|
105
|
-
epochNumber,
|
|
106
|
-
inputsUri: truncate(inputsUri),
|
|
107
|
-
numOutstandingJobs: this.jobs.size
|
|
108
|
-
});
|
|
114
|
+
this.log.verbose(`Job enqueued with broker ${jobLogText}`, jobLogData);
|
|
109
115
|
} else {
|
|
110
116
|
// Job was previously sent to the broker but is not completed
|
|
111
|
-
this.log.verbose(`Job already in queue or in progress when sent to broker
|
|
112
|
-
provingJobId: id,
|
|
113
|
-
provingJobType: ProvingRequestType[type],
|
|
114
|
-
epochNumber,
|
|
115
|
-
inputsUri: truncate(inputsUri)
|
|
116
|
-
});
|
|
117
|
+
this.log.verbose(`Job already in queue or in progress when sent to broker ${jobLogText}`, jobLogData);
|
|
117
118
|
}
|
|
118
119
|
}
|
|
119
120
|
} catch (err) {
|
|
@@ -301,42 +302,66 @@ export class BrokerCircuitProverFacade {
|
|
|
301
302
|
// We have a list of jobs that we have been told are ready, so we go ahead and ask for their results
|
|
302
303
|
await this.retrieveJobsThatShouldBeReady();
|
|
303
304
|
}
|
|
304
|
-
getAvmProof(inputs, signal, epochNumber) {
|
|
305
|
-
|
|
305
|
+
getAvmProof(inputs, skipPublicInputsValidation, signal, epochNumber) {
|
|
306
|
+
this.log.info(`getAvmProof() called with skipPublicInputsValidation: ${skipPublicInputsValidation}`);
|
|
307
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.PUBLIC_VM, inputs, epochNumber), ProvingRequestType.PUBLIC_VM, inputs, epochNumber, signal).then((result)=>{
|
|
308
|
+
// TODO(#14234)[Unconditional PIs validation]: Remove ".then()".
|
|
309
|
+
// Override the default value of skipPublicInputsValidation potentially set in BBNativeRollupProver.getAvmProof().
|
|
310
|
+
result.proof.proof[0] = skipPublicInputsValidation ? new Fr(1) : new Fr(0);
|
|
311
|
+
return result;
|
|
312
|
+
});
|
|
306
313
|
}
|
|
307
314
|
getBaseParityProof(inputs, signal, epochNumber) {
|
|
308
|
-
return this.enqueueJob(this.generateId(ProvingRequestType.
|
|
315
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.PARITY_BASE, inputs, epochNumber), ProvingRequestType.PARITY_BASE, inputs, epochNumber, signal);
|
|
309
316
|
}
|
|
310
|
-
|
|
311
|
-
return this.enqueueJob(this.generateId(ProvingRequestType.
|
|
317
|
+
getTxMergeRollupProof(input, signal, epochNumber) {
|
|
318
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.TX_MERGE_ROLLUP, input, epochNumber), ProvingRequestType.TX_MERGE_ROLLUP, input, epochNumber, signal);
|
|
319
|
+
}
|
|
320
|
+
getPublicChonkVerifierProof(inputs, signal, epochNumber) {
|
|
321
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.PUBLIC_CHONK_VERIFIER, inputs, epochNumber), ProvingRequestType.PUBLIC_CHONK_VERIFIER, inputs, epochNumber, signal);
|
|
322
|
+
}
|
|
323
|
+
getPrivateTxBaseRollupProof(baseRollupInput, signal, epochNumber) {
|
|
324
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.PRIVATE_TX_BASE_ROLLUP, baseRollupInput, epochNumber), ProvingRequestType.PRIVATE_TX_BASE_ROLLUP, baseRollupInput, epochNumber, signal);
|
|
325
|
+
}
|
|
326
|
+
getPublicTxBaseRollupProof(inputs, signal, epochNumber) {
|
|
327
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.PUBLIC_TX_BASE_ROLLUP, inputs, epochNumber), ProvingRequestType.PUBLIC_TX_BASE_ROLLUP, inputs, epochNumber, signal);
|
|
328
|
+
}
|
|
329
|
+
getRootParityProof(inputs, signal, epochNumber) {
|
|
330
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.PARITY_ROOT, inputs, epochNumber), ProvingRequestType.PARITY_ROOT, inputs, epochNumber, signal);
|
|
331
|
+
}
|
|
332
|
+
getBlockRootFirstRollupProof(input, signal, epochNumber) {
|
|
333
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.BLOCK_ROOT_FIRST_ROLLUP, input, epochNumber), ProvingRequestType.BLOCK_ROOT_FIRST_ROLLUP, input, epochNumber, signal);
|
|
334
|
+
}
|
|
335
|
+
getBlockRootSingleTxFirstRollupProof(input, signal, epochNumber) {
|
|
336
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.BLOCK_ROOT_SINGLE_TX_FIRST_ROLLUP, input, epochNumber), ProvingRequestType.BLOCK_ROOT_SINGLE_TX_FIRST_ROLLUP, input, epochNumber, signal);
|
|
337
|
+
}
|
|
338
|
+
getBlockRootEmptyTxFirstRollupProof(input, signal, epochNumber) {
|
|
339
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.BLOCK_ROOT_EMPTY_TX_FIRST_ROLLUP, input, epochNumber), ProvingRequestType.BLOCK_ROOT_EMPTY_TX_FIRST_ROLLUP, input, epochNumber, signal);
|
|
312
340
|
}
|
|
313
341
|
getBlockRootRollupProof(input, signal, epochNumber) {
|
|
314
342
|
return this.enqueueJob(this.generateId(ProvingRequestType.BLOCK_ROOT_ROLLUP, input, epochNumber), ProvingRequestType.BLOCK_ROOT_ROLLUP, input, epochNumber, signal);
|
|
315
343
|
}
|
|
316
|
-
|
|
317
|
-
return this.enqueueJob(this.generateId(ProvingRequestType.
|
|
344
|
+
getBlockRootSingleTxRollupProof(input, signal, epochNumber) {
|
|
345
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.BLOCK_ROOT_SINGLE_TX_ROLLUP, input, epochNumber), ProvingRequestType.BLOCK_ROOT_SINGLE_TX_ROLLUP, input, epochNumber, signal);
|
|
318
346
|
}
|
|
319
|
-
|
|
320
|
-
return this.enqueueJob(this.generateId(ProvingRequestType.
|
|
347
|
+
getBlockMergeRollupProof(input, signal, epochNumber) {
|
|
348
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.BLOCK_MERGE_ROLLUP, input, epochNumber), ProvingRequestType.BLOCK_MERGE_ROLLUP, input, epochNumber, signal);
|
|
321
349
|
}
|
|
322
|
-
|
|
323
|
-
return this.enqueueJob(this.generateId(ProvingRequestType.
|
|
350
|
+
getCheckpointRootRollupProof(input, signal, epochNumber) {
|
|
351
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.CHECKPOINT_ROOT_ROLLUP, input, epochNumber), ProvingRequestType.CHECKPOINT_ROOT_ROLLUP, input, epochNumber, signal);
|
|
324
352
|
}
|
|
325
|
-
|
|
326
|
-
return this.enqueueJob(this.generateId(ProvingRequestType.
|
|
353
|
+
getCheckpointRootSingleBlockRollupProof(input, signal, epochNumber) {
|
|
354
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP, input, epochNumber), ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP, input, epochNumber, signal);
|
|
327
355
|
}
|
|
328
|
-
|
|
329
|
-
return this.enqueueJob(this.generateId(ProvingRequestType.
|
|
356
|
+
getCheckpointPaddingRollupProof(input, signal, epochNumber) {
|
|
357
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.CHECKPOINT_PADDING_ROLLUP, input, epochNumber), ProvingRequestType.CHECKPOINT_PADDING_ROLLUP, input, epochNumber, signal);
|
|
330
358
|
}
|
|
331
|
-
|
|
332
|
-
return this.enqueueJob(this.generateId(ProvingRequestType.
|
|
359
|
+
getCheckpointMergeRollupProof(input, signal, epochNumber) {
|
|
360
|
+
return this.enqueueJob(this.generateId(ProvingRequestType.CHECKPOINT_MERGE_ROLLUP, input, epochNumber), ProvingRequestType.CHECKPOINT_MERGE_ROLLUP, input, epochNumber, signal);
|
|
333
361
|
}
|
|
334
362
|
getRootRollupProof(input, signal, epochNumber) {
|
|
335
363
|
return this.enqueueJob(this.generateId(ProvingRequestType.ROOT_ROLLUP, input, epochNumber), ProvingRequestType.ROOT_ROLLUP, input, epochNumber, signal);
|
|
336
364
|
}
|
|
337
|
-
getTubeProof(tubeInput, signal, epochNumber) {
|
|
338
|
-
return this.enqueueJob(this.generateId(ProvingRequestType.TUBE_PROOF, tubeInput, epochNumber), ProvingRequestType.TUBE_PROOF, tubeInput, epochNumber, signal);
|
|
339
|
-
}
|
|
340
365
|
generateId(type, inputs, epochNumber = 0) {
|
|
341
366
|
const inputsHash = sha256(inputs.toBuffer());
|
|
342
367
|
return makeProvingJobId(epochNumber, type, inputsHash.toString('hex'));
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type L1ReaderConfig } from '@aztec/ethereum';
|
|
2
2
|
import { type ConfigMappingsType } from '@aztec/foundation/config';
|
|
3
3
|
import { type DataStoreConfig } from '@aztec/kv-store/config';
|
|
4
|
+
import { type ChainConfig } from '@aztec/stdlib/config';
|
|
4
5
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
5
6
|
import { z } from 'zod';
|
|
6
7
|
export declare const ProverBrokerConfig: z.ZodObject<{
|
|
@@ -13,7 +14,9 @@ export declare const ProverBrokerConfig: z.ZodObject<{
|
|
|
13
14
|
/** If starting a prover broker locally, the directory to store broker data */
|
|
14
15
|
dataDirectory: z.ZodOptional<z.ZodString>;
|
|
15
16
|
/** The size of the data store map */
|
|
16
|
-
|
|
17
|
+
dataStoreMapSizeKb: z.ZodNumber;
|
|
18
|
+
/** The size of the prover broker's database. Will override the dataStoreMapSizeKb if set. */
|
|
19
|
+
proverBrokerStoreMapSizeKb: z.ZodOptional<z.ZodNumber>;
|
|
17
20
|
/** The prover broker may batch jobs together before writing to the database */
|
|
18
21
|
proverBrokerBatchSize: z.ZodNumber;
|
|
19
22
|
/** How often the job batches get flushed */
|
|
@@ -24,22 +27,24 @@ export declare const ProverBrokerConfig: z.ZodObject<{
|
|
|
24
27
|
proverBrokerJobMaxRetries: number;
|
|
25
28
|
proverBrokerJobTimeoutMs: number;
|
|
26
29
|
proverBrokerPollIntervalMs: number;
|
|
27
|
-
|
|
30
|
+
dataStoreMapSizeKb: number;
|
|
28
31
|
proverBrokerBatchSize: number;
|
|
29
32
|
proverBrokerBatchIntervalMs: number;
|
|
30
33
|
proverBrokerMaxEpochsToKeepResultsFor: number;
|
|
31
34
|
dataDirectory?: string | undefined;
|
|
35
|
+
proverBrokerStoreMapSizeKb?: number | undefined;
|
|
32
36
|
}, {
|
|
33
37
|
proverBrokerJobMaxRetries: number;
|
|
34
38
|
proverBrokerJobTimeoutMs: number;
|
|
35
39
|
proverBrokerPollIntervalMs: number;
|
|
36
|
-
|
|
40
|
+
dataStoreMapSizeKb: number;
|
|
37
41
|
proverBrokerBatchSize: number;
|
|
38
42
|
proverBrokerBatchIntervalMs: number;
|
|
39
43
|
proverBrokerMaxEpochsToKeepResultsFor: number;
|
|
40
44
|
dataDirectory?: string | undefined;
|
|
45
|
+
proverBrokerStoreMapSizeKb?: number | undefined;
|
|
41
46
|
}>;
|
|
42
|
-
export type ProverBrokerConfig = z.infer<typeof ProverBrokerConfig> & Pick<DataStoreConfig, '
|
|
47
|
+
export type ProverBrokerConfig = z.infer<typeof ProverBrokerConfig> & Pick<DataStoreConfig, 'dataStoreMapSizeKb' | 'dataDirectory'> & L1ReaderConfig & Pick<ChainConfig, 'rollupVersion'>;
|
|
43
48
|
export declare const proverBrokerConfigMappings: ConfigMappingsType<ProverBrokerConfig>;
|
|
44
49
|
export declare const defaultProverBrokerConfig: ProverBrokerConfig;
|
|
45
50
|
export declare const ProverAgentConfig: z.ZodObject<{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/proving_broker/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,cAAc,EAA0B,MAAM,iBAAiB,CAAC;AAC9E,OAAO,EACL,KAAK,kBAAkB,EAIxB,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/proving_broker/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,cAAc,EAA0B,MAAM,iBAAiB,CAAC;AAC9E,OAAO,EACL,KAAK,kBAAkB,EAIxB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,sBAAsB,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,kBAAkB;IAC7B,qFAAqF;;IAErF,uHAAuH;;IAEvH,6FAA6F;;IAE7F,8EAA8E;;IAE9E,qCAAqC;;IAErC,6FAA6F;;IAE7F,+EAA+E;;IAE/E,4CAA4C;;IAE5C,uDAAuD;;;;;;;;;;;;;;;;;;;;;;EAEvD,CAAC;AAEH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,GACjE,IAAI,CAAC,eAAe,EAAE,oBAAoB,GAAG,eAAe,CAAC,GAC7D,cAAc,GACd,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;AAErC,eAAO,MAAM,0BAA0B,EAAE,kBAAkB,CAAC,kBAAkB,CAuC7E,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,kBAAiE,CAAC;AAE1G,eAAO,MAAM,iBAAiB;IAC5B,2CAA2C;;IAE3C,wDAAwD;;IAExD,gDAAgD;;IAEhD,+CAA+C;;IAE/C,uCAAuC;;IAEvC,gDAAgD;;IAEhD,2DAA2D;;IAE3D,8EAA8E;;;;;;;;;;;;;;;;;;;;EAE9E,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAElE,eAAO,MAAM,yBAAyB,EAAE,kBAAkB,CAAC,iBAAiB,CA4C3E,CAAC"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { l1ReaderConfigMappings } from '@aztec/ethereum';
|
|
2
2
|
import { booleanConfigHelper, getDefaultConfig, numberConfigHelper } from '@aztec/foundation/config';
|
|
3
|
+
import { pickConfigMappings } from '@aztec/foundation/config';
|
|
3
4
|
import { dataConfigMappings } from '@aztec/kv-store/config';
|
|
5
|
+
import { chainConfigMappings } from '@aztec/stdlib/config';
|
|
4
6
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
5
7
|
import { z } from 'zod';
|
|
6
8
|
export const ProverBrokerConfig = z.object({
|
|
@@ -8,7 +10,8 @@ export const ProverBrokerConfig = z.object({
|
|
|
8
10
|
/** If starting a prover broker locally, the time after which a job times out and gets assigned to a different agent */ proverBrokerJobTimeoutMs: z.number().int().nonnegative(),
|
|
9
11
|
/** If starting a prover broker locally, the interval the broker checks for timed out jobs */ proverBrokerPollIntervalMs: z.number().int().nonnegative(),
|
|
10
12
|
/** If starting a prover broker locally, the directory to store broker data */ dataDirectory: z.string().optional(),
|
|
11
|
-
/** The size of the data store map */
|
|
13
|
+
/** The size of the data store map */ dataStoreMapSizeKb: z.number().int().nonnegative(),
|
|
14
|
+
/** The size of the prover broker's database. Will override the dataStoreMapSizeKb if set. */ proverBrokerStoreMapSizeKb: z.number().int().nonnegative().optional(),
|
|
12
15
|
/** The prover broker may batch jobs together before writing to the database */ proverBrokerBatchSize: z.number().int().nonnegative(),
|
|
13
16
|
/** How often the job batches get flushed */ proverBrokerBatchIntervalMs: z.number().int().nonnegative(),
|
|
14
17
|
/** The maximum number of epochs to keep results for */ proverBrokerMaxEpochsToKeepResultsFor: z.number().int().nonnegative()
|
|
@@ -44,8 +47,16 @@ export const proverBrokerConfigMappings = {
|
|
|
44
47
|
description: 'The maximum number of epochs to keep results for',
|
|
45
48
|
...numberConfigHelper(1)
|
|
46
49
|
},
|
|
50
|
+
proverBrokerStoreMapSizeKb: {
|
|
51
|
+
env: 'PROVER_BROKER_STORE_MAP_SIZE_KB',
|
|
52
|
+
parseEnv: (val)=>val ? +val : undefined,
|
|
53
|
+
description: "The size of the prover broker's database. Will override the dataStoreMapSizeKb if set."
|
|
54
|
+
},
|
|
55
|
+
...dataConfigMappings,
|
|
47
56
|
...l1ReaderConfigMappings,
|
|
48
|
-
...
|
|
57
|
+
...pickConfigMappings(chainConfigMappings, [
|
|
58
|
+
'rollupVersion'
|
|
59
|
+
])
|
|
49
60
|
};
|
|
50
61
|
export const defaultProverBrokerConfig = getDefaultConfig(proverBrokerConfigMappings);
|
|
51
62
|
export const ProverAgentConfig = z.object({
|
|
@@ -70,7 +81,7 @@ export const proverAgentConfigMappings = {
|
|
|
70
81
|
proverAgentPollIntervalMs: {
|
|
71
82
|
env: 'PROVER_AGENT_POLL_INTERVAL_MS',
|
|
72
83
|
description: 'The interval agents poll for jobs at',
|
|
73
|
-
...numberConfigHelper(
|
|
84
|
+
...numberConfigHelper(1000)
|
|
74
85
|
},
|
|
75
86
|
proverAgentProofTypes: {
|
|
76
87
|
env: 'PROVER_AGENT_PROOF_TYPES',
|
|
@@ -84,7 +95,7 @@ export const proverAgentConfigMappings = {
|
|
|
84
95
|
realProofs: {
|
|
85
96
|
env: 'PROVER_REAL_PROOFS',
|
|
86
97
|
description: 'Whether to construct real proofs',
|
|
87
|
-
...booleanConfigHelper(
|
|
98
|
+
...booleanConfigHelper(true)
|
|
88
99
|
},
|
|
89
100
|
proverTestDelayType: {
|
|
90
101
|
env: 'PROVER_TEST_DELAY_TYPE',
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { TelemetryClient } from '@aztec/telemetry-client';
|
|
2
2
|
import type { ProverBrokerConfig } from './config.js';
|
|
3
3
|
import { ProvingBroker } from './proving_broker.js';
|
|
4
|
-
export declare function createAndStartProvingBroker(
|
|
4
|
+
export declare function createAndStartProvingBroker(_config: ProverBrokerConfig, client: TelemetryClient): Promise<ProvingBroker>;
|
|
5
5
|
//# sourceMappingURL=factory.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/proving_broker/factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIpD,wBAAsB,2BAA2B,CAC/C,
|
|
1
|
+
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/proving_broker/factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIpD,wBAAsB,2BAA2B,CAC/C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,aAAa,CAAC,CAQxB"}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { ProvingBroker } from './proving_broker.js';
|
|
2
2
|
import { InMemoryBrokerDatabase } from './proving_broker_database/memory.js';
|
|
3
3
|
import { KVBrokerDatabase } from './proving_broker_database/persisted.js';
|
|
4
|
-
export async function createAndStartProvingBroker(
|
|
4
|
+
export async function createAndStartProvingBroker(_config, client) {
|
|
5
|
+
const config = {
|
|
6
|
+
..._config,
|
|
7
|
+
dataStoreMapSizeKb: _config.proverBrokerStoreMapSizeKb ?? _config.dataStoreMapSizeKb
|
|
8
|
+
};
|
|
5
9
|
const database = config.dataDirectory ? await KVBrokerDatabase.new(config, client) : new InMemoryBrokerDatabase();
|
|
6
10
|
const broker = new ProvingBroker(database, config, client);
|
|
7
11
|
await broker.start();
|
|
@@ -2,7 +2,7 @@ import { randomBytes } from '@aztec/foundation/crypto';
|
|
|
2
2
|
import { makeProvingJobId } from '@aztec/stdlib/interfaces/server';
|
|
3
3
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
4
4
|
export function makeRandomProvingJobId(epochNumber) {
|
|
5
|
-
return makeProvingJobId(epochNumber ?? 1, ProvingRequestType.
|
|
5
|
+
return makeProvingJobId(epochNumber ?? 1, ProvingRequestType.PARITY_BASE, randomBytes(8).toString('hex'));
|
|
6
6
|
}
|
|
7
7
|
export function makeInputsUri() {
|
|
8
8
|
return randomBytes(8).toString('hex');
|
|
@@ -15,7 +15,7 @@ export function createProofStore(config, logger = createLogger('prover-client:pr
|
|
|
15
15
|
path
|
|
16
16
|
});
|
|
17
17
|
return new GoogleCloudStorageProofStore(bucket, path);
|
|
18
|
-
} catch
|
|
18
|
+
} catch {
|
|
19
19
|
throw new Error(`Invalid google cloud proof store definition: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`);
|
|
20
20
|
}
|
|
21
21
|
} else {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gcs_proof_store.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proof_store/gcs_proof_store.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACrB,KAAK,oBAAoB,EAE1B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAK1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"gcs_proof_store.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proof_store/gcs_proof_store.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACrB,KAAK,oBAAoB,EAE1B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAK1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAKnD,qBAAa,4BAA6B,YAAW,UAAU;IAI3D,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAJvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;gBAGf,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM;IAKlB,cAAc,CAAC,CAAC,SAAS,kBAAkB,EACtD,EAAE,EAAE,YAAY,EAChB,IAAI,EAAE,CAAC,EACP,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAC7B,OAAO,CAAC,QAAQ,CAAC;IAOpB,eAAe,CAAC,CAAC,SAAS,kBAAkB,EAC1C,GAAG,EAAE,YAAY,EACjB,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,QAAQ,CAAC;IAIP,aAAa,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwBpE,cAAc,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAG1D"}
|
|
@@ -3,6 +3,7 @@ import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
|
3
3
|
import { Storage } from '@google-cloud/storage';
|
|
4
4
|
import { join } from 'path';
|
|
5
5
|
const INPUTS_PATH = 'inputs';
|
|
6
|
+
// REFACTOR(#13067): Use the stdlib/file-store instead of referencing google-cloud-storage directly.
|
|
6
7
|
export class GoogleCloudStorageProofStore {
|
|
7
8
|
bucketName;
|
|
8
9
|
path;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proof_store/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proof_store/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ProverAgentStatus, ProvingJobConsumer, ServerCircuitProver } from '@aztec/stdlib/interfaces/server';
|
|
2
2
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
3
3
|
import { type TelemetryClient, type Traceable, type Tracer } from '@aztec/telemetry-client';
|
|
4
4
|
import type { ProofStore } from './proof_store/index.js';
|
|
@@ -20,7 +20,6 @@ export declare class ProvingAgent implements Traceable {
|
|
|
20
20
|
private currentJobController?;
|
|
21
21
|
private runningPromise;
|
|
22
22
|
private instrumentation;
|
|
23
|
-
private idleTimer;
|
|
24
23
|
readonly tracer: Tracer;
|
|
25
24
|
constructor(
|
|
26
25
|
/** The source of proving jobs */
|
|
@@ -39,8 +38,9 @@ export declare class ProvingAgent implements Traceable {
|
|
|
39
38
|
isRunning(): boolean;
|
|
40
39
|
start(): void;
|
|
41
40
|
stop(): Promise<void>;
|
|
41
|
+
getStatus(): ProverAgentStatus;
|
|
42
42
|
private work;
|
|
43
43
|
private startJob;
|
|
44
|
-
|
|
44
|
+
private reportResult;
|
|
45
45
|
}
|
|
46
46
|
//# sourceMappingURL=proving_agent.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proving_agent.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_agent.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAEV,
|
|
1
|
+
{"version":3,"file":"proving_agent.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_agent.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAEV,iBAAiB,EACjB,kBAAkB,EAIlB,mBAAmB,EACpB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAIzD;;GAEG;AACH,qBAAa,YAAa,YAAW,SAAS;IAQ1C,iCAAiC;IACjC,OAAO,CAAC,MAAM;IACd,gDAAgD;IAChD,OAAO,CAAC,UAAU;IAClB,iDAAiD;IACjD,OAAO,CAAC,aAAa;IACrB,oDAAoD;IACpD,OAAO,CAAC,cAAc;IACtB,oCAAoC;IACpC,OAAO,CAAC,cAAc;IAGtB,OAAO,CAAC,GAAG;IAnBb,OAAO,CAAC,oBAAoB,CAAC,CAAuB;IACpD,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,eAAe,CAA8B;IAErD,SAAgB,MAAM,EAAE,MAAM,CAAC;;IAG7B,iCAAiC;IACzB,MAAM,EAAE,kBAAkB;IAClC,gDAAgD;IACxC,UAAU,EAAE,UAAU;IAC9B,iDAAiD;IACzC,aAAa,EAAE,mBAAmB;IAC1C,oDAAoD;IAC5C,cAAc,GAAE,KAAK,CAAC,kBAAkB,CAAM;IACtD,oCAAoC;IAC5B,cAAc,SAAO;IAC7B,uDAAuD;IACvD,MAAM,GAAE,eAAsC,EACtC,GAAG,yCAA8C;IAOpD,gBAAgB,CAAC,aAAa,EAAE,mBAAmB,GAAG,IAAI;IAI1D,SAAS,IAAI,OAAO;IAIpB,KAAK,IAAI,IAAI;IAIP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAK3B,SAAS,IAAI,iBAAiB;YAcvB,IAAI;YAqDJ,QAAQ;YAuDR,YAAY;CA0B3B"}
|
|
@@ -4,10 +4,10 @@ function _ts_decorate(decorators, target, key, desc) {
|
|
|
4
4
|
else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
5
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
6
|
}
|
|
7
|
+
import { AbortError } from '@aztec/foundation/error';
|
|
7
8
|
import { createLogger } from '@aztec/foundation/log';
|
|
8
9
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
9
10
|
import { truncate } from '@aztec/foundation/string';
|
|
10
|
-
import { Timer } from '@aztec/foundation/timer';
|
|
11
11
|
import { ProvingError } from '@aztec/stdlib/errors';
|
|
12
12
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
13
13
|
import { getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
@@ -25,7 +25,6 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
|
|
|
25
25
|
currentJobController;
|
|
26
26
|
runningPromise;
|
|
27
27
|
instrumentation;
|
|
28
|
-
idleTimer;
|
|
29
28
|
tracer;
|
|
30
29
|
constructor(/** The source of proving jobs */ broker, /** Database holding proof inputs and outputs */ proofStore, /** The prover implementation to defer jobs to */ circuitProver, /** Optional list of allowed proof types to build */ proofAllowList = [], /** How long to wait between jobs */ pollIntervalMs = 1000, /** A telemetry client through which to emit metrics */ client = getTelemetryClient(), log = createLogger('prover-client:proving-agent')){
|
|
31
30
|
this.broker = broker;
|
|
@@ -34,28 +33,6 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
|
|
|
34
33
|
this.proofAllowList = proofAllowList;
|
|
35
34
|
this.pollIntervalMs = pollIntervalMs;
|
|
36
35
|
this.log = log;
|
|
37
|
-
this.handleJobResult = async (jobId, type, err, result)=>{
|
|
38
|
-
let maybeJob;
|
|
39
|
-
if (err) {
|
|
40
|
-
const retry = err.name === ProvingError.NAME ? err.retry : false;
|
|
41
|
-
this.log.error(`Job id=${jobId} type=${ProvingRequestType[type]} failed err=${err.message} retry=${retry}`, err);
|
|
42
|
-
maybeJob = await this.broker.reportProvingJobError(jobId, err.message, retry, {
|
|
43
|
-
allowList: this.proofAllowList
|
|
44
|
-
});
|
|
45
|
-
} else if (result) {
|
|
46
|
-
const outputUri = await this.proofStore.saveProofOutput(jobId, type, result);
|
|
47
|
-
this.log.info(`Job id=${jobId} type=${ProvingRequestType[type]} completed outputUri=${truncate(outputUri)}`);
|
|
48
|
-
maybeJob = await this.broker.reportProvingJobSuccess(jobId, outputUri, {
|
|
49
|
-
allowList: this.proofAllowList
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
if (maybeJob) {
|
|
53
|
-
const { job, time } = maybeJob;
|
|
54
|
-
await this.startJob(job, time);
|
|
55
|
-
} else {
|
|
56
|
-
this.idleTimer = new Timer();
|
|
57
|
-
}
|
|
58
|
-
};
|
|
59
36
|
this.tracer = client.getTracer('ProvingAgent');
|
|
60
37
|
this.instrumentation = new ProvingAgentInstrumentation(client);
|
|
61
38
|
this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollIntervalMs);
|
|
@@ -67,42 +44,78 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
|
|
|
67
44
|
return this.runningPromise?.isRunning() ?? false;
|
|
68
45
|
}
|
|
69
46
|
start() {
|
|
70
|
-
this.idleTimer = new Timer();
|
|
71
47
|
this.runningPromise.start();
|
|
72
48
|
}
|
|
73
49
|
async stop() {
|
|
74
50
|
this.currentJobController?.abort();
|
|
75
51
|
await this.runningPromise.stop();
|
|
76
52
|
}
|
|
53
|
+
getStatus() {
|
|
54
|
+
if (this.currentJobController) {
|
|
55
|
+
return {
|
|
56
|
+
status: 'proving',
|
|
57
|
+
jobId: this.currentJobController.getJobId(),
|
|
58
|
+
proofType: this.currentJobController.getProofType(),
|
|
59
|
+
startedAtISO: new Date(this.currentJobController.getStartedAt()).toISOString()
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
return this.runningPromise.isRunning() ? {
|
|
63
|
+
status: 'running'
|
|
64
|
+
} : {
|
|
65
|
+
status: 'stopped'
|
|
66
|
+
};
|
|
67
|
+
}
|
|
77
68
|
async work() {
|
|
78
|
-
// every tick we need to
|
|
79
|
-
//
|
|
80
|
-
//
|
|
81
|
-
//
|
|
69
|
+
// every tick we need to take one of the following actions:
|
|
70
|
+
// 1. send a hearbeat to the broker that we're working on some job
|
|
71
|
+
// 2. if the job is complete, send its result to the broker
|
|
72
|
+
// 3. get a job from the broker
|
|
73
|
+
// Any one of these actions could give us a new job to work on. If that happens we abort the current job.
|
|
74
|
+
//
|
|
75
|
+
// This loop gets triggered in one of two ways:
|
|
76
|
+
// - either on a timer (see pollIntervalMs)
|
|
77
|
+
// - or when a proof completes
|
|
82
78
|
let maybeJob;
|
|
83
|
-
if (this.currentJobController
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
79
|
+
if (this.currentJobController) {
|
|
80
|
+
const status = this.currentJobController.getStatus();
|
|
81
|
+
const jobId = this.currentJobController.getJobId();
|
|
82
|
+
const proofType = this.currentJobController.getProofType();
|
|
83
|
+
const startedAt = this.currentJobController.getStartedAt();
|
|
84
|
+
const result = this.currentJobController.getResult();
|
|
85
|
+
if (status === ProvingJobControllerStatus.RUNNING) {
|
|
86
|
+
maybeJob = await this.broker.reportProvingJobProgress(jobId, startedAt, {
|
|
87
|
+
allowList: this.proofAllowList
|
|
88
|
+
});
|
|
89
|
+
} else if (status === ProvingJobControllerStatus.DONE) {
|
|
90
|
+
if (result) {
|
|
91
|
+
maybeJob = await this.reportResult(jobId, proofType, result);
|
|
92
|
+
} else {
|
|
93
|
+
this.log.warn(`Job controller for job ${this.currentJobController.getJobId()} is done but doesn't have a result`, {
|
|
94
|
+
jobId
|
|
95
|
+
});
|
|
96
|
+
maybeJob = await this.reportResult(jobId, proofType, new ProvingError('No result found after proving', undefined, /* retry */ true));
|
|
97
|
+
}
|
|
98
|
+
this.currentJobController = undefined;
|
|
99
|
+
} else {
|
|
100
|
+
// IDLE status should not be seen because a job is started as soon as it is created
|
|
101
|
+
this.log.warn(`Idle job controller for job: ${this.currentJobController.getJobId()}. Skipping main loop work`, {
|
|
102
|
+
jobId: this.currentJobController.getJobId()
|
|
103
|
+
});
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
87
106
|
} else {
|
|
88
107
|
maybeJob = await this.broker.getProvingJob({
|
|
89
108
|
allowList: this.proofAllowList
|
|
90
109
|
});
|
|
91
110
|
}
|
|
92
|
-
if (
|
|
93
|
-
|
|
94
|
-
}
|
|
95
|
-
if (this.idleTimer) {
|
|
96
|
-
this.instrumentation.recordIdleTime(this.idleTimer);
|
|
111
|
+
if (maybeJob) {
|
|
112
|
+
await this.startJob(maybeJob);
|
|
97
113
|
}
|
|
98
|
-
this.idleTimer = undefined;
|
|
99
|
-
const { job, time } = maybeJob;
|
|
100
|
-
await this.startJob(job, time);
|
|
101
114
|
}
|
|
102
|
-
async startJob(job, startedAt) {
|
|
115
|
+
async startJob({ job, time: startedAt }) {
|
|
103
116
|
let abortedProofJobId;
|
|
104
117
|
let abortedProofName;
|
|
105
|
-
if (this.currentJobController?.getStatus() === ProvingJobControllerStatus.
|
|
118
|
+
if (this.currentJobController?.getStatus() === ProvingJobControllerStatus.RUNNING) {
|
|
106
119
|
abortedProofJobId = this.currentJobController.getJobId();
|
|
107
120
|
abortedProofName = this.currentJobController.getProofTypeName();
|
|
108
121
|
this.currentJobController?.abort();
|
|
@@ -110,16 +123,20 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
|
|
|
110
123
|
let inputs;
|
|
111
124
|
try {
|
|
112
125
|
inputs = await this.proofStore.getProofInput(job.inputsUri);
|
|
113
|
-
} catch
|
|
126
|
+
} catch {
|
|
114
127
|
const maybeJob = await this.broker.reportProvingJobError(job.id, 'Failed to load proof inputs', true, {
|
|
115
128
|
allowList: this.proofAllowList
|
|
116
129
|
});
|
|
117
130
|
if (maybeJob) {
|
|
118
|
-
return this.startJob(maybeJob
|
|
131
|
+
return this.startJob(maybeJob);
|
|
119
132
|
}
|
|
120
133
|
return;
|
|
121
134
|
}
|
|
122
|
-
this.currentJobController = new ProvingJobController(job.id, inputs, job.epochNumber, startedAt, this.circuitProver,
|
|
135
|
+
this.currentJobController = new ProvingJobController(job.id, inputs, job.epochNumber, startedAt, this.circuitProver, ()=>{
|
|
136
|
+
// trigger a run of the main work loop when proving completes
|
|
137
|
+
// no need to await this here. The controller will stay alive (in DONE state) until the result is send to the broker
|
|
138
|
+
void this.runningPromise.trigger();
|
|
139
|
+
});
|
|
123
140
|
if (abortedProofJobId) {
|
|
124
141
|
this.log.info(`Aborting job id=${abortedProofJobId} type=${abortedProofName} to start new job id=${this.currentJobController.getJobId()} type=${this.currentJobController.getProofTypeName()} inputsUri=${truncate(job.inputsUri)}`);
|
|
125
142
|
} else {
|
|
@@ -127,7 +144,26 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
|
|
|
127
144
|
}
|
|
128
145
|
this.currentJobController.start();
|
|
129
146
|
}
|
|
130
|
-
|
|
147
|
+
async reportResult(jobId, type, result) {
|
|
148
|
+
let maybeJob;
|
|
149
|
+
if (result instanceof AbortError) {
|
|
150
|
+
// no-op
|
|
151
|
+
this.log.warn(`Job id=${jobId} was aborted. Not reporting result back to broker`, result);
|
|
152
|
+
} else if (result instanceof Error) {
|
|
153
|
+
const retry = result.name === ProvingError.NAME ? result.retry : false;
|
|
154
|
+
this.log.error(`Job id=${jobId} type=${ProvingRequestType[type]} failed err=${result.message} retry=${retry}`, result);
|
|
155
|
+
maybeJob = await this.broker.reportProvingJobError(jobId, result.message, retry, {
|
|
156
|
+
allowList: this.proofAllowList
|
|
157
|
+
});
|
|
158
|
+
} else {
|
|
159
|
+
const outputUri = await this.proofStore.saveProofOutput(jobId, type, result);
|
|
160
|
+
this.log.info(`Job id=${jobId} type=${ProvingRequestType[type]} completed outputUri=${truncate(outputUri)}`);
|
|
161
|
+
maybeJob = await this.broker.reportProvingJobSuccess(jobId, outputUri, {
|
|
162
|
+
allowList: this.proofAllowList
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
return maybeJob;
|
|
166
|
+
}
|
|
131
167
|
}
|
|
132
168
|
_ts_decorate([
|
|
133
169
|
trackSpan('ProvingAgent.safeWork')
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type GetProvingJobResponse, type ProofUri, type ProvingJob, type ProvingJobConsumer, type ProvingJobFilter, type ProvingJobId, type ProvingJobProducer, type ProvingJobStatus } from '@aztec/stdlib/interfaces/server';
|
|
2
|
+
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
2
3
|
import { type TelemetryClient, type Traceable, type Tracer } from '@aztec/telemetry-client';
|
|
3
4
|
import { type ProverBrokerConfig } from './config.js';
|
|
4
5
|
import type { ProvingBrokerDatabase } from './proving_broker_database.js';
|
|
@@ -27,7 +28,7 @@ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobCons
|
|
|
27
28
|
/**
|
|
28
29
|
* The broker keeps track of the highest epoch its seen.
|
|
29
30
|
* This information is used for garbage collection: once it reaches the next epoch, it can start pruning the database of old state.
|
|
30
|
-
* It is important that this value is
|
|
31
|
+
* It is important that this value is initialized to zero. This ensures that we don't delete any old jobs until the current
|
|
31
32
|
* process instance receives a job request informing it of the actual current highest epoch
|
|
32
33
|
* Example:
|
|
33
34
|
* proving epoch 11 - the broker will wipe all jobs for epochs 9 and lower
|
|
@@ -61,4 +62,12 @@ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobCons
|
|
|
61
62
|
private isJobStale;
|
|
62
63
|
private oldestEpochToKeep;
|
|
63
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* Relative priority of each proof type. Proofs higher up on the list are more important and should be prioritized
|
|
67
|
+
* over proofs lower on the list.
|
|
68
|
+
*
|
|
69
|
+
* The aim is that this will speed up block proving as the closer we get to a block's root proof the more likely it
|
|
70
|
+
* is to get picked up by agents
|
|
71
|
+
*/
|
|
72
|
+
export declare const PROOF_TYPES_IN_PRIORITY_ORDER: ProvingRequestType[];
|
|
64
73
|
//# sourceMappingURL=proving_broker.d.ts.map
|