@aztec/prover-client 0.0.0-test.0 → 0.0.1-commit.21caa21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/block-factory/index.d.ts +2 -0
- package/dest/block-factory/index.d.ts.map +1 -0
- package/dest/block-factory/light.d.ts +38 -0
- package/dest/block-factory/light.d.ts.map +1 -0
- package/dest/block-factory/light.js +108 -0
- package/dest/config.d.ts +7 -7
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +11 -1
- package/dest/index.d.ts +1 -1
- package/dest/light/lightweight_checkpoint_builder.d.ts +28 -0
- package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -0
- package/dest/light/lightweight_checkpoint_builder.js +107 -0
- package/dest/mocks/fixtures.d.ts +8 -8
- package/dest/mocks/fixtures.d.ts.map +1 -1
- package/dest/mocks/fixtures.js +32 -14
- package/dest/mocks/test_context.d.ts +40 -31
- package/dest/mocks/test_context.d.ts.map +1 -1
- package/dest/mocks/test_context.js +134 -86
- package/dest/orchestrator/block-building-helpers.d.ts +36 -29
- package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
- package/dest/orchestrator/block-building-helpers.js +168 -188
- package/dest/orchestrator/block-proving-state.d.ts +68 -47
- package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/block-proving-state.js +281 -176
- package/dest/orchestrator/checkpoint-proving-state.d.ts +62 -0
- package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -0
- package/dest/orchestrator/checkpoint-proving-state.js +208 -0
- package/dest/orchestrator/epoch-proving-state.d.ts +40 -26
- package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/epoch-proving-state.js +143 -73
- package/dest/orchestrator/index.d.ts +1 -1
- package/dest/orchestrator/orchestrator.d.ts +35 -32
- package/dest/orchestrator/orchestrator.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator.js +389 -239
- package/dest/orchestrator/orchestrator_metrics.d.ts +3 -1
- package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator_metrics.js +9 -0
- package/dest/orchestrator/tx-proving-state.d.ts +13 -11
- package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/tx-proving-state.js +23 -40
- package/dest/prover-client/factory.d.ts +1 -1
- package/dest/prover-client/index.d.ts +1 -1
- package/dest/prover-client/prover-client.d.ts +4 -4
- package/dest/prover-client/prover-client.d.ts.map +1 -1
- package/dest/prover-client/prover-client.js +5 -4
- package/dest/prover-client/server-epoch-prover.d.ts +15 -11
- package/dest/prover-client/server-epoch-prover.d.ts.map +1 -1
- package/dest/prover-client/server-epoch-prover.js +11 -11
- package/dest/proving_broker/broker_prover_facade.d.ts +23 -16
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
- package/dest/proving_broker/broker_prover_facade.js +67 -41
- package/dest/proving_broker/config.d.ts +18 -9
- package/dest/proving_broker/config.d.ts.map +1 -1
- package/dest/proving_broker/config.js +22 -5
- package/dest/proving_broker/factory.d.ts +2 -2
- package/dest/proving_broker/factory.d.ts.map +1 -1
- package/dest/proving_broker/factory.js +5 -1
- package/dest/proving_broker/fixtures.d.ts +3 -2
- package/dest/proving_broker/fixtures.d.ts.map +1 -1
- package/dest/proving_broker/fixtures.js +2 -1
- package/dest/proving_broker/index.d.ts +1 -1
- package/dest/proving_broker/proof_store/factory.d.ts +2 -2
- package/dest/proving_broker/proof_store/factory.js +1 -1
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +1 -1
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/gcs_proof_store.js +1 -0
- package/dest/proving_broker/proof_store/index.d.ts +2 -1
- package/dest/proving_broker/proof_store/index.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/index.js +1 -0
- package/dest/proving_broker/proof_store/inline_proof_store.d.ts +1 -1
- package/dest/proving_broker/proof_store/inline_proof_store.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/proof_store.d.ts +1 -1
- package/dest/proving_broker/proving_agent.d.ts +4 -4
- package/dest/proving_broker/proving_agent.d.ts.map +1 -1
- package/dest/proving_broker/proving_agent.js +83 -47
- package/dest/proving_broker/proving_agent_instrumentation.d.ts +1 -1
- package/dest/proving_broker/proving_agent_instrumentation.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.d.ts +13 -4
- package/dest/proving_broker/proving_broker.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.js +36 -23
- package/dest/proving_broker/proving_broker_database/memory.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/memory.js +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.js +12 -10
- package/dest/proving_broker/proving_broker_database.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
- package/dest/proving_broker/proving_job_controller.d.ts +9 -9
- package/dest/proving_broker/proving_job_controller.d.ts.map +1 -1
- package/dest/proving_broker/proving_job_controller.js +89 -61
- package/dest/proving_broker/rpc.d.ts +4 -6
- package/dest/proving_broker/rpc.d.ts.map +1 -1
- package/dest/proving_broker/rpc.js +1 -4
- package/dest/test/mock_proof_store.d.ts +9 -0
- package/dest/test/mock_proof_store.d.ts.map +1 -0
- package/dest/test/mock_proof_store.js +10 -0
- package/dest/test/mock_prover.d.ts +23 -17
- package/dest/test/mock_prover.d.ts.map +1 -1
- package/dest/test/mock_prover.js +38 -20
- package/package.json +32 -31
- package/src/block-factory/index.ts +1 -0
- package/src/block-factory/light.ts +137 -0
- package/src/config.ts +24 -8
- package/src/light/lightweight_checkpoint_builder.ts +142 -0
- package/src/mocks/fixtures.ts +42 -37
- package/src/mocks/test_context.ts +207 -115
- package/src/orchestrator/block-building-helpers.ts +256 -333
- package/src/orchestrator/block-proving-state.ts +323 -230
- package/src/orchestrator/checkpoint-proving-state.ts +301 -0
- package/src/orchestrator/epoch-proving-state.ts +187 -112
- package/src/orchestrator/orchestrator.ts +592 -299
- package/src/orchestrator/orchestrator_metrics.ts +20 -1
- package/src/orchestrator/tx-proving-state.ts +50 -64
- package/src/prover-client/prover-client.ts +16 -14
- package/src/prover-client/server-epoch-prover.ts +39 -21
- package/src/proving_broker/broker_prover_facade.ts +214 -126
- package/src/proving_broker/config.ts +24 -6
- package/src/proving_broker/factory.ts +2 -1
- package/src/proving_broker/fixtures.ts +7 -2
- package/src/proving_broker/proof_store/factory.ts +1 -1
- package/src/proving_broker/proof_store/gcs_proof_store.ts +5 -1
- package/src/proving_broker/proof_store/index.ts +1 -0
- package/src/proving_broker/proof_store/inline_proof_store.ts +1 -1
- package/src/proving_broker/proving_agent.ts +89 -47
- package/src/proving_broker/proving_broker.ts +53 -33
- package/src/proving_broker/proving_broker_database/memory.ts +3 -2
- package/src/proving_broker/proving_broker_database/persisted.ts +14 -12
- package/src/proving_broker/proving_broker_database.ts +2 -1
- package/src/proving_broker/proving_job_controller.ts +94 -82
- package/src/proving_broker/rpc.ts +1 -6
- package/src/test/mock_proof_store.ts +14 -0
- package/src/test/mock_prover.ts +164 -60
- package/dest/bin/get-proof-inputs.d.ts +0 -2
- package/dest/bin/get-proof-inputs.d.ts.map +0 -1
- package/dest/bin/get-proof-inputs.js +0 -51
- package/dest/block_builder/index.d.ts +0 -6
- package/dest/block_builder/index.d.ts.map +0 -1
- package/dest/block_builder/light.d.ts +0 -33
- package/dest/block_builder/light.d.ts.map +0 -1
- package/dest/block_builder/light.js +0 -82
- package/src/bin/get-proof-inputs.ts +0 -59
- package/src/block_builder/index.ts +0 -6
- package/src/block_builder/light.ts +0 -101
- /package/dest/{block_builder → block-factory}/index.js +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
import { AbortError } from '@aztec/foundation/error';
|
|
1
2
|
import { createLogger } from '@aztec/foundation/log';
|
|
2
3
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
3
4
|
import { truncate } from '@aztec/foundation/string';
|
|
4
|
-
import { Timer } from '@aztec/foundation/timer';
|
|
5
5
|
import { ProvingError } from '@aztec/stdlib/errors';
|
|
6
6
|
import type {
|
|
7
|
-
|
|
7
|
+
GetProvingJobResponse,
|
|
8
|
+
ProverAgentStatus,
|
|
8
9
|
ProvingJobConsumer,
|
|
9
10
|
ProvingJobId,
|
|
10
11
|
ProvingJobInputs,
|
|
@@ -31,7 +32,6 @@ export class ProvingAgent implements Traceable {
|
|
|
31
32
|
private currentJobController?: ProvingJobController;
|
|
32
33
|
private runningPromise: RunningPromise;
|
|
33
34
|
private instrumentation: ProvingAgentInstrumentation;
|
|
34
|
-
private idleTimer: Timer | undefined;
|
|
35
35
|
|
|
36
36
|
public readonly tracer: Tracer;
|
|
37
37
|
|
|
@@ -64,7 +64,6 @@ export class ProvingAgent implements Traceable {
|
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
public start(): void {
|
|
67
|
-
this.idleTimer = new Timer();
|
|
68
67
|
this.runningPromise.start();
|
|
69
68
|
}
|
|
70
69
|
|
|
@@ -73,41 +72,78 @@ export class ProvingAgent implements Traceable {
|
|
|
73
72
|
await this.runningPromise.stop();
|
|
74
73
|
}
|
|
75
74
|
|
|
75
|
+
public getStatus(): ProverAgentStatus {
|
|
76
|
+
if (this.currentJobController) {
|
|
77
|
+
return {
|
|
78
|
+
status: 'proving',
|
|
79
|
+
jobId: this.currentJobController.getJobId(),
|
|
80
|
+
proofType: this.currentJobController.getProofType(),
|
|
81
|
+
startedAtISO: new Date(this.currentJobController.getStartedAt()).toISOString(),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return this.runningPromise.isRunning() ? { status: 'running' } : { status: 'stopped' };
|
|
86
|
+
}
|
|
87
|
+
|
|
76
88
|
@trackSpan('ProvingAgent.safeWork')
|
|
77
89
|
private async work() {
|
|
78
|
-
// every tick we need to
|
|
79
|
-
//
|
|
80
|
-
//
|
|
81
|
-
//
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
90
|
+
// every tick we need to take one of the following actions:
|
|
91
|
+
// 1. send a hearbeat to the broker that we're working on some job
|
|
92
|
+
// 2. if the job is complete, send its result to the broker
|
|
93
|
+
// 3. get a job from the broker
|
|
94
|
+
// Any one of these actions could give us a new job to work on. If that happens we abort the current job.
|
|
95
|
+
//
|
|
96
|
+
// This loop gets triggered in one of two ways:
|
|
97
|
+
// - either on a timer (see pollIntervalMs)
|
|
98
|
+
// - or when a proof completes
|
|
99
|
+
let maybeJob: GetProvingJobResponse | undefined;
|
|
100
|
+
|
|
101
|
+
if (this.currentJobController) {
|
|
102
|
+
const status = this.currentJobController.getStatus();
|
|
103
|
+
const jobId = this.currentJobController.getJobId();
|
|
104
|
+
const proofType = this.currentJobController.getProofType();
|
|
105
|
+
const startedAt = this.currentJobController.getStartedAt();
|
|
106
|
+
const result = this.currentJobController.getResult();
|
|
107
|
+
|
|
108
|
+
if (status === ProvingJobControllerStatus.RUNNING) {
|
|
109
|
+
maybeJob = await this.broker.reportProvingJobProgress(jobId, startedAt, { allowList: this.proofAllowList });
|
|
110
|
+
} else if (status === ProvingJobControllerStatus.DONE) {
|
|
111
|
+
if (result) {
|
|
112
|
+
maybeJob = await this.reportResult(jobId, proofType, result);
|
|
113
|
+
} else {
|
|
114
|
+
this.log.warn(
|
|
115
|
+
`Job controller for job ${this.currentJobController.getJobId()} is done but doesn't have a result`,
|
|
116
|
+
{ jobId },
|
|
117
|
+
);
|
|
118
|
+
maybeJob = await this.reportResult(
|
|
119
|
+
jobId,
|
|
120
|
+
proofType,
|
|
121
|
+
new ProvingError('No result found after proving', undefined, /* retry */ true),
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
this.currentJobController = undefined;
|
|
126
|
+
} else {
|
|
127
|
+
// IDLE status should not be seen because a job is started as soon as it is created
|
|
128
|
+
this.log.warn(`Idle job controller for job: ${this.currentJobController.getJobId()}. Skipping main loop work`, {
|
|
129
|
+
jobId: this.currentJobController.getJobId(),
|
|
130
|
+
});
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
89
133
|
} else {
|
|
90
134
|
maybeJob = await this.broker.getProvingJob({ allowList: this.proofAllowList });
|
|
91
135
|
}
|
|
92
136
|
|
|
93
|
-
if (
|
|
94
|
-
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
if (this.idleTimer) {
|
|
98
|
-
this.instrumentation.recordIdleTime(this.idleTimer);
|
|
137
|
+
if (maybeJob) {
|
|
138
|
+
await this.startJob(maybeJob);
|
|
99
139
|
}
|
|
100
|
-
this.idleTimer = undefined;
|
|
101
|
-
|
|
102
|
-
const { job, time } = maybeJob;
|
|
103
|
-
await this.startJob(job, time);
|
|
104
140
|
}
|
|
105
141
|
|
|
106
|
-
private async startJob(job:
|
|
142
|
+
private async startJob({ job, time: startedAt }: GetProvingJobResponse): Promise<void> {
|
|
107
143
|
let abortedProofJobId: string | undefined;
|
|
108
144
|
let abortedProofName: string | undefined;
|
|
109
145
|
|
|
110
|
-
if (this.currentJobController?.getStatus() === ProvingJobControllerStatus.
|
|
146
|
+
if (this.currentJobController?.getStatus() === ProvingJobControllerStatus.RUNNING) {
|
|
111
147
|
abortedProofJobId = this.currentJobController.getJobId();
|
|
112
148
|
abortedProofName = this.currentJobController.getProofTypeName();
|
|
113
149
|
this.currentJobController?.abort();
|
|
@@ -116,13 +152,13 @@ export class ProvingAgent implements Traceable {
|
|
|
116
152
|
let inputs: ProvingJobInputs;
|
|
117
153
|
try {
|
|
118
154
|
inputs = await this.proofStore.getProofInput(job.inputsUri);
|
|
119
|
-
} catch
|
|
155
|
+
} catch {
|
|
120
156
|
const maybeJob = await this.broker.reportProvingJobError(job.id, 'Failed to load proof inputs', true, {
|
|
121
157
|
allowList: this.proofAllowList,
|
|
122
158
|
});
|
|
123
159
|
|
|
124
160
|
if (maybeJob) {
|
|
125
|
-
return this.startJob(maybeJob
|
|
161
|
+
return this.startJob(maybeJob);
|
|
126
162
|
}
|
|
127
163
|
|
|
128
164
|
return;
|
|
@@ -134,7 +170,11 @@ export class ProvingAgent implements Traceable {
|
|
|
134
170
|
job.epochNumber,
|
|
135
171
|
startedAt,
|
|
136
172
|
this.circuitProver,
|
|
137
|
-
|
|
173
|
+
() => {
|
|
174
|
+
// trigger a run of the main work loop when proving completes
|
|
175
|
+
// no need to await this here. The controller will stay alive (in DONE state) until the result is send to the broker
|
|
176
|
+
void this.runningPromise.trigger();
|
|
177
|
+
},
|
|
138
178
|
);
|
|
139
179
|
|
|
140
180
|
if (abortedProofJobId) {
|
|
@@ -154,28 +194,30 @@ export class ProvingAgent implements Traceable {
|
|
|
154
194
|
this.currentJobController.start();
|
|
155
195
|
}
|
|
156
196
|
|
|
157
|
-
|
|
197
|
+
private async reportResult<T extends ProvingRequestType>(
|
|
158
198
|
jobId: ProvingJobId,
|
|
159
199
|
type: T,
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
200
|
+
result: ProvingJobResultsMap[T] | Error,
|
|
201
|
+
): Promise<GetProvingJobResponse | undefined> {
|
|
202
|
+
let maybeJob: GetProvingJobResponse | undefined;
|
|
203
|
+
if (result instanceof AbortError) {
|
|
204
|
+
// no-op
|
|
205
|
+
this.log.warn(`Job id=${jobId} was aborted. Not reporting result back to broker`, result);
|
|
206
|
+
} else if (result instanceof Error) {
|
|
207
|
+
const retry = result.name === ProvingError.NAME ? (result as ProvingError).retry : false;
|
|
208
|
+
this.log.error(
|
|
209
|
+
`Job id=${jobId} type=${ProvingRequestType[type]} failed err=${result.message} retry=${retry}`,
|
|
210
|
+
result,
|
|
211
|
+
);
|
|
212
|
+
maybeJob = await this.broker.reportProvingJobError(jobId, result.message, retry, {
|
|
213
|
+
allowList: this.proofAllowList,
|
|
214
|
+
});
|
|
215
|
+
} else {
|
|
169
216
|
const outputUri = await this.proofStore.saveProofOutput(jobId, type, result);
|
|
170
217
|
this.log.info(`Job id=${jobId} type=${ProvingRequestType[type]} completed outputUri=${truncate(outputUri)}`);
|
|
171
218
|
maybeJob = await this.broker.reportProvingJobSuccess(jobId, outputUri, { allowList: this.proofAllowList });
|
|
172
219
|
}
|
|
173
220
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
await this.startJob(job, time);
|
|
177
|
-
} else {
|
|
178
|
-
this.idleTimer = new Timer();
|
|
179
|
-
}
|
|
180
|
-
};
|
|
221
|
+
return maybeJob;
|
|
222
|
+
}
|
|
181
223
|
}
|
|
@@ -1,17 +1,19 @@
|
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import { createLogger } from '@aztec/foundation/log';
|
|
2
3
|
import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
3
4
|
import { PriorityMemoryQueue } from '@aztec/foundation/queue';
|
|
4
5
|
import { Timer } from '@aztec/foundation/timer';
|
|
5
|
-
import
|
|
6
|
-
GetProvingJobResponse,
|
|
7
|
-
ProofUri,
|
|
8
|
-
ProvingJob,
|
|
9
|
-
ProvingJobConsumer,
|
|
10
|
-
ProvingJobFilter,
|
|
11
|
-
ProvingJobId,
|
|
12
|
-
ProvingJobProducer,
|
|
13
|
-
ProvingJobSettledResult,
|
|
14
|
-
ProvingJobStatus,
|
|
6
|
+
import {
|
|
7
|
+
type GetProvingJobResponse,
|
|
8
|
+
type ProofUri,
|
|
9
|
+
type ProvingJob,
|
|
10
|
+
type ProvingJobConsumer,
|
|
11
|
+
type ProvingJobFilter,
|
|
12
|
+
type ProvingJobId,
|
|
13
|
+
type ProvingJobProducer,
|
|
14
|
+
type ProvingJobSettledResult,
|
|
15
|
+
type ProvingJobStatus,
|
|
16
|
+
tryStop,
|
|
15
17
|
} from '@aztec/stdlib/interfaces/server';
|
|
16
18
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
17
19
|
import {
|
|
@@ -43,20 +45,33 @@ type EnqueuedProvingJob = Pick<ProvingJob, 'id' | 'epochNumber'>;
|
|
|
43
45
|
export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable {
|
|
44
46
|
private queues: ProvingQueues = {
|
|
45
47
|
[ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
46
|
-
[ProvingRequestType.
|
|
48
|
+
[ProvingRequestType.PUBLIC_CHONK_VERIFIER]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
47
49
|
|
|
48
|
-
[ProvingRequestType.
|
|
49
|
-
[ProvingRequestType.
|
|
50
|
-
[ProvingRequestType.
|
|
50
|
+
[ProvingRequestType.PRIVATE_TX_BASE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
51
|
+
[ProvingRequestType.PUBLIC_TX_BASE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
52
|
+
[ProvingRequestType.TX_MERGE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
51
53
|
[ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
52
54
|
|
|
53
55
|
[ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
56
|
+
[ProvingRequestType.BLOCK_ROOT_FIRST_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
57
|
+
[ProvingRequestType.BLOCK_ROOT_SINGLE_TX_FIRST_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(
|
|
58
|
+
provingJobComparator,
|
|
59
|
+
),
|
|
60
|
+
[ProvingRequestType.BLOCK_ROOT_EMPTY_TX_FIRST_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(
|
|
61
|
+
provingJobComparator,
|
|
62
|
+
),
|
|
54
63
|
[ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
55
|
-
[ProvingRequestType.
|
|
56
|
-
|
|
64
|
+
[ProvingRequestType.BLOCK_ROOT_SINGLE_TX_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
65
|
+
|
|
66
|
+
[ProvingRequestType.CHECKPOINT_ROOT_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
67
|
+
[ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(
|
|
68
|
+
provingJobComparator,
|
|
69
|
+
),
|
|
70
|
+
[ProvingRequestType.CHECKPOINT_MERGE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
71
|
+
[ProvingRequestType.CHECKPOINT_PADDING_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
57
72
|
|
|
58
|
-
[ProvingRequestType.
|
|
59
|
-
[ProvingRequestType.
|
|
73
|
+
[ProvingRequestType.PARITY_BASE]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
74
|
+
[ProvingRequestType.PARITY_ROOT]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
60
75
|
};
|
|
61
76
|
|
|
62
77
|
// holds a copy of the database in memory in order to quickly fulfill requests
|
|
@@ -93,7 +108,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
93
108
|
/**
|
|
94
109
|
* The broker keeps track of the highest epoch its seen.
|
|
95
110
|
* This information is used for garbage collection: once it reaches the next epoch, it can start pruning the database of old state.
|
|
96
|
-
* It is important that this value is
|
|
111
|
+
* It is important that this value is initialized to zero. This ensures that we don't delete any old jobs until the current
|
|
97
112
|
* process instance receives a job request informing it of the actual current highest epoch
|
|
98
113
|
* Example:
|
|
99
114
|
* proving epoch 11 - the broker will wipe all jobs for epochs 9 and lower
|
|
@@ -184,7 +199,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
184
199
|
this.logger.warn('ProvingBroker not started');
|
|
185
200
|
return Promise.resolve();
|
|
186
201
|
}
|
|
187
|
-
await this.cleanupPromise
|
|
202
|
+
await tryStop(this.cleanupPromise);
|
|
188
203
|
}
|
|
189
204
|
|
|
190
205
|
public enqueueProvingJob(job: ProvingJob): Promise<ProvingJobStatus> {
|
|
@@ -313,7 +328,6 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
313
328
|
return Promise.resolve(notifications.concat(completedJobs));
|
|
314
329
|
}
|
|
315
330
|
|
|
316
|
-
// eslint-disable-next-line require-await
|
|
317
331
|
#getProvingJob(filter: ProvingJobFilter = { allowList: [] }): { job: ProvingJob; time: number } | undefined {
|
|
318
332
|
const allowedProofs: ProvingRequestType[] =
|
|
319
333
|
Array.isArray(filter.allowList) && filter.allowList.length > 0
|
|
@@ -554,7 +568,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
554
568
|
this.reEnqueueExpiredJobs();
|
|
555
569
|
const oldestEpochToKeep = this.oldestEpochToKeep();
|
|
556
570
|
if (oldestEpochToKeep > 0) {
|
|
557
|
-
await this.database.deleteAllProvingJobsOlderThanEpoch(oldestEpochToKeep);
|
|
571
|
+
await this.database.deleteAllProvingJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
|
|
558
572
|
this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
|
|
559
573
|
}
|
|
560
574
|
}
|
|
@@ -571,7 +585,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
571
585
|
|
|
572
586
|
if (jobsToClean.length > 0) {
|
|
573
587
|
this.cleanUpProvingJobState(jobsToClean);
|
|
574
|
-
this.logger.
|
|
588
|
+
this.logger.verbose(`Cleaned up proving jobs=${jobsToClean.length}`);
|
|
575
589
|
}
|
|
576
590
|
}
|
|
577
591
|
|
|
@@ -671,17 +685,23 @@ function proofTypeComparator(a: ProvingRequestType, b: ProvingRequestType): -1 |
|
|
|
671
685
|
* The aim is that this will speed up block proving as the closer we get to a block's root proof the more likely it
|
|
672
686
|
* is to get picked up by agents
|
|
673
687
|
*/
|
|
674
|
-
const PROOF_TYPES_IN_PRIORITY_ORDER: ProvingRequestType[] = [
|
|
688
|
+
export const PROOF_TYPES_IN_PRIORITY_ORDER: ProvingRequestType[] = [
|
|
689
|
+
ProvingRequestType.ROOT_ROLLUP,
|
|
690
|
+
ProvingRequestType.BLOCK_ROOT_FIRST_ROLLUP,
|
|
691
|
+
ProvingRequestType.BLOCK_ROOT_SINGLE_TX_FIRST_ROLLUP,
|
|
692
|
+
ProvingRequestType.BLOCK_ROOT_EMPTY_TX_FIRST_ROLLUP,
|
|
675
693
|
ProvingRequestType.BLOCK_ROOT_ROLLUP,
|
|
676
|
-
ProvingRequestType.
|
|
694
|
+
ProvingRequestType.BLOCK_ROOT_SINGLE_TX_ROLLUP,
|
|
677
695
|
ProvingRequestType.BLOCK_MERGE_ROLLUP,
|
|
678
|
-
ProvingRequestType.
|
|
679
|
-
ProvingRequestType.
|
|
680
|
-
ProvingRequestType.
|
|
681
|
-
ProvingRequestType.
|
|
696
|
+
ProvingRequestType.CHECKPOINT_ROOT_ROLLUP,
|
|
697
|
+
ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP,
|
|
698
|
+
ProvingRequestType.CHECKPOINT_MERGE_ROLLUP,
|
|
699
|
+
ProvingRequestType.CHECKPOINT_PADDING_ROLLUP,
|
|
700
|
+
ProvingRequestType.TX_MERGE_ROLLUP,
|
|
701
|
+
ProvingRequestType.PUBLIC_TX_BASE_ROLLUP,
|
|
702
|
+
ProvingRequestType.PRIVATE_TX_BASE_ROLLUP,
|
|
682
703
|
ProvingRequestType.PUBLIC_VM,
|
|
683
|
-
ProvingRequestType.
|
|
684
|
-
ProvingRequestType.
|
|
685
|
-
ProvingRequestType.
|
|
686
|
-
ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP,
|
|
704
|
+
ProvingRequestType.PUBLIC_CHONK_VERIFIER,
|
|
705
|
+
ProvingRequestType.PARITY_ROOT,
|
|
706
|
+
ProvingRequestType.PARITY_BASE,
|
|
687
707
|
];
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import {
|
|
2
3
|
type ProofUri,
|
|
3
4
|
type ProvingJob,
|
|
@@ -43,7 +44,7 @@ export class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
43
44
|
return Promise.resolve();
|
|
44
45
|
}
|
|
45
46
|
|
|
46
|
-
deleteAllProvingJobsOlderThanEpoch(epochNumber:
|
|
47
|
+
deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void> {
|
|
47
48
|
const toDelete = [
|
|
48
49
|
...Array.from(this.jobs.keys()).filter(x => getEpochFromProvingJobId(x) < epochNumber),
|
|
49
50
|
...Array.from(this.results.keys()).filter(x => getEpochFromProvingJobId(x) < epochNumber),
|
|
@@ -52,7 +53,7 @@ export class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
52
53
|
}
|
|
53
54
|
|
|
54
55
|
async *allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]> {
|
|
55
|
-
for
|
|
56
|
+
for (const item of this.jobs.values()) {
|
|
56
57
|
yield [item, this.results.get(item.id)] as const;
|
|
57
58
|
}
|
|
58
59
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
|
|
2
3
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
3
4
|
import { BatchQueue } from '@aztec/foundation/queue';
|
|
@@ -46,9 +47,9 @@ class SingleEpochDatabase {
|
|
|
46
47
|
|
|
47
48
|
async *allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]> {
|
|
48
49
|
for await (const jobStr of this.jobs.valuesAsync()) {
|
|
49
|
-
const job =
|
|
50
|
+
const job = jsonParseWithSchema(jobStr, ProvingJob);
|
|
50
51
|
const resultStr = await this.jobResults.getAsync(job.id);
|
|
51
|
-
const result = resultStr ?
|
|
52
|
+
const result = resultStr ? jsonParseWithSchema(resultStr, ProvingJobSettledResult) : undefined;
|
|
52
53
|
yield [job, result];
|
|
53
54
|
}
|
|
54
55
|
}
|
|
@@ -104,14 +105,15 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
104
105
|
const jobsToAdd = items.filter((item): item is ProvingJob => 'id' in item);
|
|
105
106
|
const resultsToAdd = items.filter((item): item is [ProvingJobId, ProvingJobSettledResult] => Array.isArray(item));
|
|
106
107
|
|
|
107
|
-
const db = await this.getEpochDatabase(epochNumber);
|
|
108
|
+
const db = await this.getEpochDatabase(EpochNumber(epochNumber));
|
|
108
109
|
await db.batchWrite(jobsToAdd, resultsToAdd);
|
|
109
110
|
}
|
|
110
111
|
|
|
111
112
|
private async estimateSize() {
|
|
112
113
|
const sizes = await Promise.all(Array.from(this.epochs.values()).map(x => x.estimateSize()));
|
|
113
114
|
return {
|
|
114
|
-
mappingSize: this.config.
|
|
115
|
+
mappingSize: this.config.dataStoreMapSizeKb,
|
|
116
|
+
physicalFileSize: sizes.reduce((prev, curr) => prev + curr.physicalFileSize, 0),
|
|
115
117
|
numItems: sizes.reduce((prev, curr) => prev + curr.numItems, 0),
|
|
116
118
|
actualSize: sizes.reduce((prev, curr) => prev + curr.actualSize, 0),
|
|
117
119
|
};
|
|
@@ -136,13 +138,13 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
136
138
|
continue;
|
|
137
139
|
}
|
|
138
140
|
logger.info(
|
|
139
|
-
`Loading broker database for epoch ${epochNumber} from ${fullDirectory} with map size ${config.
|
|
141
|
+
`Loading broker database for epoch ${epochNumber} from ${fullDirectory} with map size ${config.dataStoreMapSizeKb}KB`,
|
|
140
142
|
);
|
|
141
143
|
const db = await openVersionedStoreAt(
|
|
142
144
|
fullDirectory,
|
|
143
145
|
SingleEpochDatabase.SCHEMA_VERSION,
|
|
144
146
|
config.l1Contracts.rollupAddress,
|
|
145
|
-
config.
|
|
147
|
+
config.dataStoreMapSizeKb,
|
|
146
148
|
);
|
|
147
149
|
const epochDb = new SingleEpochDatabase(db);
|
|
148
150
|
epochs.set(epochNumber, epochDb);
|
|
@@ -163,14 +165,14 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
163
165
|
}
|
|
164
166
|
}
|
|
165
167
|
|
|
166
|
-
async deleteAllProvingJobsOlderThanEpoch(epochNumber:
|
|
167
|
-
const oldEpochs = Array.from(this.epochs.keys()).filter(e => e < epochNumber);
|
|
168
|
+
async deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void> {
|
|
169
|
+
const oldEpochs = Array.from(this.epochs.keys()).filter(e => e < Number(epochNumber));
|
|
168
170
|
for (const old of oldEpochs) {
|
|
169
171
|
const db = this.epochs.get(old);
|
|
170
172
|
if (!db) {
|
|
171
173
|
continue;
|
|
172
174
|
}
|
|
173
|
-
this.logger.
|
|
175
|
+
this.logger.verbose(`Deleting broker database for epoch ${old}`);
|
|
174
176
|
await db.delete();
|
|
175
177
|
this.epochs.delete(old);
|
|
176
178
|
}
|
|
@@ -195,19 +197,19 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
195
197
|
return this.batchQueue.put([id, { status: 'fulfilled', value }], getEpochFromProvingJobId(id));
|
|
196
198
|
}
|
|
197
199
|
|
|
198
|
-
private async getEpochDatabase(epochNumber:
|
|
200
|
+
private async getEpochDatabase(epochNumber: EpochNumber): Promise<SingleEpochDatabase> {
|
|
199
201
|
let epochDb = this.epochs.get(epochNumber);
|
|
200
202
|
if (!epochDb) {
|
|
201
203
|
const newEpochDirectory = join(this.config.dataDirectory!, epochNumber.toString());
|
|
202
204
|
await mkdir(newEpochDirectory, { recursive: true });
|
|
203
205
|
this.logger.info(
|
|
204
|
-
`Creating broker database for epoch ${epochNumber} at ${newEpochDirectory} with map size ${this.config.
|
|
206
|
+
`Creating broker database for epoch ${epochNumber} at ${newEpochDirectory} with map size ${this.config.dataStoreMapSizeKb}`,
|
|
205
207
|
);
|
|
206
208
|
const db = await openVersionedStoreAt(
|
|
207
209
|
newEpochDirectory,
|
|
208
210
|
SingleEpochDatabase.SCHEMA_VERSION,
|
|
209
211
|
this.config.l1Contracts.rollupAddress,
|
|
210
|
-
this.config.
|
|
212
|
+
this.config.dataStoreMapSizeKb,
|
|
211
213
|
);
|
|
212
214
|
epochDb = new SingleEpochDatabase(db);
|
|
213
215
|
this.epochs.set(epochNumber, epochDb);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import type { ProofUri, ProvingJob, ProvingJobId, ProvingJobSettledResult } from '@aztec/stdlib/interfaces/server';
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -14,7 +15,7 @@ export interface ProvingBrokerDatabase {
|
|
|
14
15
|
* Deletes all proving jobs belonging to epochs older than the given epoch
|
|
15
16
|
* @param epochNumber - The epoch number beyond which jobs should be deleted
|
|
16
17
|
*/
|
|
17
|
-
deleteAllProvingJobsOlderThanEpoch(epochNumber:
|
|
18
|
+
deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void>;
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
21
|
* Returns an iterator over all saved proving jobs
|