@aztec/prover-client 0.0.0-test.1 → 0.0.1-commit.03f7ef2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/block-factory/index.d.ts +2 -0
- package/dest/block-factory/index.d.ts.map +1 -0
- package/dest/block-factory/light.d.ts +38 -0
- package/dest/block-factory/light.d.ts.map +1 -0
- package/dest/block-factory/light.js +108 -0
- package/dest/config.d.ts +8 -8
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +12 -2
- package/dest/index.d.ts +1 -1
- package/dest/light/index.d.ts +2 -0
- package/dest/light/index.d.ts.map +1 -0
- package/dest/light/index.js +1 -0
- package/dest/light/lightweight_checkpoint_builder.d.ts +36 -0
- package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -0
- package/dest/light/lightweight_checkpoint_builder.js +147 -0
- package/dest/mocks/fixtures.d.ts +8 -8
- package/dest/mocks/fixtures.d.ts.map +1 -1
- package/dest/mocks/fixtures.js +34 -16
- package/dest/mocks/test_context.d.ts +41 -32
- package/dest/mocks/test_context.d.ts.map +1 -1
- package/dest/mocks/test_context.js +138 -87
- package/dest/orchestrator/block-building-helpers.d.ts +37 -30
- package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
- package/dest/orchestrator/block-building-helpers.js +170 -189
- package/dest/orchestrator/block-proving-state.d.ts +70 -48
- package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/block-proving-state.js +282 -177
- package/dest/orchestrator/checkpoint-proving-state.d.ts +63 -0
- package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -0
- package/dest/orchestrator/checkpoint-proving-state.js +210 -0
- package/dest/orchestrator/epoch-proving-state.d.ts +41 -27
- package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/epoch-proving-state.js +143 -73
- package/dest/orchestrator/index.d.ts +1 -1
- package/dest/orchestrator/orchestrator.d.ts +36 -33
- package/dest/orchestrator/orchestrator.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator.js +379 -250
- package/dest/orchestrator/orchestrator_metrics.d.ts +1 -1
- package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -1
- package/dest/orchestrator/tx-proving-state.d.ts +13 -11
- package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/tx-proving-state.js +23 -40
- package/dest/prover-client/factory.d.ts +1 -1
- package/dest/prover-client/index.d.ts +1 -1
- package/dest/prover-client/prover-client.d.ts +4 -4
- package/dest/prover-client/prover-client.d.ts.map +1 -1
- package/dest/prover-client/prover-client.js +5 -4
- package/dest/prover-client/server-epoch-prover.d.ts +16 -12
- package/dest/prover-client/server-epoch-prover.d.ts.map +1 -1
- package/dest/prover-client/server-epoch-prover.js +11 -11
- package/dest/proving_broker/broker_prover_facade.d.ts +23 -16
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
- package/dest/proving_broker/broker_prover_facade.js +59 -40
- package/dest/proving_broker/config.d.ts +19 -10
- package/dest/proving_broker/config.d.ts.map +1 -1
- package/dest/proving_broker/config.js +23 -6
- package/dest/proving_broker/factory.d.ts +2 -2
- package/dest/proving_broker/factory.d.ts.map +1 -1
- package/dest/proving_broker/factory.js +5 -1
- package/dest/proving_broker/fixtures.d.ts +3 -2
- package/dest/proving_broker/fixtures.d.ts.map +1 -1
- package/dest/proving_broker/fixtures.js +3 -2
- package/dest/proving_broker/index.d.ts +1 -1
- package/dest/proving_broker/proof_store/factory.d.ts +2 -2
- package/dest/proving_broker/proof_store/factory.js +1 -1
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +1 -1
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/gcs_proof_store.js +1 -0
- package/dest/proving_broker/proof_store/index.d.ts +2 -1
- package/dest/proving_broker/proof_store/index.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/index.js +1 -0
- package/dest/proving_broker/proof_store/inline_proof_store.d.ts +1 -1
- package/dest/proving_broker/proof_store/inline_proof_store.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/proof_store.d.ts +1 -1
- package/dest/proving_broker/proving_agent.d.ts +4 -4
- package/dest/proving_broker/proving_agent.d.ts.map +1 -1
- package/dest/proving_broker/proving_agent.js +83 -47
- package/dest/proving_broker/proving_agent_instrumentation.d.ts +1 -1
- package/dest/proving_broker/proving_agent_instrumentation.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.d.ts +13 -4
- package/dest/proving_broker/proving_broker.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.js +39 -23
- package/dest/proving_broker/proving_broker_database/memory.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/memory.js +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.js +12 -10
- package/dest/proving_broker/proving_broker_database.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
- package/dest/proving_broker/proving_job_controller.d.ts +9 -9
- package/dest/proving_broker/proving_job_controller.d.ts.map +1 -1
- package/dest/proving_broker/proving_job_controller.js +87 -60
- package/dest/proving_broker/rpc.d.ts +4 -6
- package/dest/proving_broker/rpc.d.ts.map +1 -1
- package/dest/proving_broker/rpc.js +1 -4
- package/dest/test/mock_proof_store.d.ts +9 -0
- package/dest/test/mock_proof_store.d.ts.map +1 -0
- package/dest/test/mock_proof_store.js +10 -0
- package/dest/test/mock_prover.d.ts +23 -17
- package/dest/test/mock_prover.d.ts.map +1 -1
- package/dest/test/mock_prover.js +37 -19
- package/package.json +34 -31
- package/src/block-factory/index.ts +1 -0
- package/src/block-factory/light.ts +137 -0
- package/src/config.ts +25 -9
- package/src/light/index.ts +1 -0
- package/src/light/lightweight_checkpoint_builder.ts +198 -0
- package/src/mocks/fixtures.ts +44 -39
- package/src/mocks/test_context.ts +213 -116
- package/src/orchestrator/block-building-helpers.ts +258 -334
- package/src/orchestrator/block-proving-state.ts +325 -231
- package/src/orchestrator/checkpoint-proving-state.ts +303 -0
- package/src/orchestrator/epoch-proving-state.ts +191 -113
- package/src/orchestrator/orchestrator.ts +582 -314
- package/src/orchestrator/tx-proving-state.ts +49 -64
- package/src/prover-client/prover-client.ts +16 -14
- package/src/prover-client/server-epoch-prover.ts +40 -22
- package/src/proving_broker/broker_prover_facade.ts +205 -126
- package/src/proving_broker/config.ts +25 -7
- package/src/proving_broker/factory.ts +2 -1
- package/src/proving_broker/fixtures.ts +8 -3
- package/src/proving_broker/proof_store/factory.ts +1 -1
- package/src/proving_broker/proof_store/gcs_proof_store.ts +5 -1
- package/src/proving_broker/proof_store/index.ts +1 -0
- package/src/proving_broker/proof_store/inline_proof_store.ts +1 -1
- package/src/proving_broker/proving_agent.ts +89 -47
- package/src/proving_broker/proving_broker.ts +56 -33
- package/src/proving_broker/proving_broker_database/memory.ts +3 -2
- package/src/proving_broker/proving_broker_database/persisted.ts +14 -12
- package/src/proving_broker/proving_broker_database.ts +2 -1
- package/src/proving_broker/proving_job_controller.ts +92 -81
- package/src/proving_broker/rpc.ts +1 -6
- package/src/test/mock_proof_store.ts +14 -0
- package/src/test/mock_prover.ts +158 -59
- package/dest/bin/get-proof-inputs.d.ts +0 -2
- package/dest/bin/get-proof-inputs.d.ts.map +0 -1
- package/dest/bin/get-proof-inputs.js +0 -51
- package/dest/block_builder/index.d.ts +0 -6
- package/dest/block_builder/index.d.ts.map +0 -1
- package/dest/block_builder/light.d.ts +0 -33
- package/dest/block_builder/light.d.ts.map +0 -1
- package/dest/block_builder/light.js +0 -82
- package/src/bin/get-proof-inputs.ts +0 -59
- package/src/block_builder/index.ts +0 -6
- package/src/block_builder/light.ts +0 -101
- /package/dest/{block_builder → block-factory}/index.js +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
import { AbortError } from '@aztec/foundation/error';
|
|
1
2
|
import { createLogger } from '@aztec/foundation/log';
|
|
2
3
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
3
4
|
import { truncate } from '@aztec/foundation/string';
|
|
4
|
-
import { Timer } from '@aztec/foundation/timer';
|
|
5
5
|
import { ProvingError } from '@aztec/stdlib/errors';
|
|
6
6
|
import type {
|
|
7
|
-
|
|
7
|
+
GetProvingJobResponse,
|
|
8
|
+
ProverAgentStatus,
|
|
8
9
|
ProvingJobConsumer,
|
|
9
10
|
ProvingJobId,
|
|
10
11
|
ProvingJobInputs,
|
|
@@ -31,7 +32,6 @@ export class ProvingAgent implements Traceable {
|
|
|
31
32
|
private currentJobController?: ProvingJobController;
|
|
32
33
|
private runningPromise: RunningPromise;
|
|
33
34
|
private instrumentation: ProvingAgentInstrumentation;
|
|
34
|
-
private idleTimer: Timer | undefined;
|
|
35
35
|
|
|
36
36
|
public readonly tracer: Tracer;
|
|
37
37
|
|
|
@@ -64,7 +64,6 @@ export class ProvingAgent implements Traceable {
|
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
public start(): void {
|
|
67
|
-
this.idleTimer = new Timer();
|
|
68
67
|
this.runningPromise.start();
|
|
69
68
|
}
|
|
70
69
|
|
|
@@ -73,41 +72,78 @@ export class ProvingAgent implements Traceable {
|
|
|
73
72
|
await this.runningPromise.stop();
|
|
74
73
|
}
|
|
75
74
|
|
|
75
|
+
public getStatus(): ProverAgentStatus {
|
|
76
|
+
if (this.currentJobController) {
|
|
77
|
+
return {
|
|
78
|
+
status: 'proving',
|
|
79
|
+
jobId: this.currentJobController.getJobId(),
|
|
80
|
+
proofType: this.currentJobController.getProofType(),
|
|
81
|
+
startedAtISO: new Date(this.currentJobController.getStartedAt()).toISOString(),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return this.runningPromise.isRunning() ? { status: 'running' } : { status: 'stopped' };
|
|
86
|
+
}
|
|
87
|
+
|
|
76
88
|
@trackSpan('ProvingAgent.safeWork')
|
|
77
89
|
private async work() {
|
|
78
|
-
// every tick we need to
|
|
79
|
-
//
|
|
80
|
-
//
|
|
81
|
-
//
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
90
|
+
// every tick we need to take one of the following actions:
|
|
91
|
+
// 1. send a hearbeat to the broker that we're working on some job
|
|
92
|
+
// 2. if the job is complete, send its result to the broker
|
|
93
|
+
// 3. get a job from the broker
|
|
94
|
+
// Any one of these actions could give us a new job to work on. If that happens we abort the current job.
|
|
95
|
+
//
|
|
96
|
+
// This loop gets triggered in one of two ways:
|
|
97
|
+
// - either on a timer (see pollIntervalMs)
|
|
98
|
+
// - or when a proof completes
|
|
99
|
+
let maybeJob: GetProvingJobResponse | undefined;
|
|
100
|
+
|
|
101
|
+
if (this.currentJobController) {
|
|
102
|
+
const status = this.currentJobController.getStatus();
|
|
103
|
+
const jobId = this.currentJobController.getJobId();
|
|
104
|
+
const proofType = this.currentJobController.getProofType();
|
|
105
|
+
const startedAt = this.currentJobController.getStartedAt();
|
|
106
|
+
const result = this.currentJobController.getResult();
|
|
107
|
+
|
|
108
|
+
if (status === ProvingJobControllerStatus.RUNNING) {
|
|
109
|
+
maybeJob = await this.broker.reportProvingJobProgress(jobId, startedAt, { allowList: this.proofAllowList });
|
|
110
|
+
} else if (status === ProvingJobControllerStatus.DONE) {
|
|
111
|
+
if (result) {
|
|
112
|
+
maybeJob = await this.reportResult(jobId, proofType, result);
|
|
113
|
+
} else {
|
|
114
|
+
this.log.warn(
|
|
115
|
+
`Job controller for job ${this.currentJobController.getJobId()} is done but doesn't have a result`,
|
|
116
|
+
{ jobId },
|
|
117
|
+
);
|
|
118
|
+
maybeJob = await this.reportResult(
|
|
119
|
+
jobId,
|
|
120
|
+
proofType,
|
|
121
|
+
new ProvingError('No result found after proving', undefined, /* retry */ true),
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
this.currentJobController = undefined;
|
|
126
|
+
} else {
|
|
127
|
+
// IDLE status should not be seen because a job is started as soon as it is created
|
|
128
|
+
this.log.warn(`Idle job controller for job: ${this.currentJobController.getJobId()}. Skipping main loop work`, {
|
|
129
|
+
jobId: this.currentJobController.getJobId(),
|
|
130
|
+
});
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
89
133
|
} else {
|
|
90
134
|
maybeJob = await this.broker.getProvingJob({ allowList: this.proofAllowList });
|
|
91
135
|
}
|
|
92
136
|
|
|
93
|
-
if (
|
|
94
|
-
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
if (this.idleTimer) {
|
|
98
|
-
this.instrumentation.recordIdleTime(this.idleTimer);
|
|
137
|
+
if (maybeJob) {
|
|
138
|
+
await this.startJob(maybeJob);
|
|
99
139
|
}
|
|
100
|
-
this.idleTimer = undefined;
|
|
101
|
-
|
|
102
|
-
const { job, time } = maybeJob;
|
|
103
|
-
await this.startJob(job, time);
|
|
104
140
|
}
|
|
105
141
|
|
|
106
|
-
private async startJob(job:
|
|
142
|
+
private async startJob({ job, time: startedAt }: GetProvingJobResponse): Promise<void> {
|
|
107
143
|
let abortedProofJobId: string | undefined;
|
|
108
144
|
let abortedProofName: string | undefined;
|
|
109
145
|
|
|
110
|
-
if (this.currentJobController?.getStatus() === ProvingJobControllerStatus.
|
|
146
|
+
if (this.currentJobController?.getStatus() === ProvingJobControllerStatus.RUNNING) {
|
|
111
147
|
abortedProofJobId = this.currentJobController.getJobId();
|
|
112
148
|
abortedProofName = this.currentJobController.getProofTypeName();
|
|
113
149
|
this.currentJobController?.abort();
|
|
@@ -116,13 +152,13 @@ export class ProvingAgent implements Traceable {
|
|
|
116
152
|
let inputs: ProvingJobInputs;
|
|
117
153
|
try {
|
|
118
154
|
inputs = await this.proofStore.getProofInput(job.inputsUri);
|
|
119
|
-
} catch
|
|
155
|
+
} catch {
|
|
120
156
|
const maybeJob = await this.broker.reportProvingJobError(job.id, 'Failed to load proof inputs', true, {
|
|
121
157
|
allowList: this.proofAllowList,
|
|
122
158
|
});
|
|
123
159
|
|
|
124
160
|
if (maybeJob) {
|
|
125
|
-
return this.startJob(maybeJob
|
|
161
|
+
return this.startJob(maybeJob);
|
|
126
162
|
}
|
|
127
163
|
|
|
128
164
|
return;
|
|
@@ -134,7 +170,11 @@ export class ProvingAgent implements Traceable {
|
|
|
134
170
|
job.epochNumber,
|
|
135
171
|
startedAt,
|
|
136
172
|
this.circuitProver,
|
|
137
|
-
|
|
173
|
+
() => {
|
|
174
|
+
// trigger a run of the main work loop when proving completes
|
|
175
|
+
// no need to await this here. The controller will stay alive (in DONE state) until the result is send to the broker
|
|
176
|
+
void this.runningPromise.trigger();
|
|
177
|
+
},
|
|
138
178
|
);
|
|
139
179
|
|
|
140
180
|
if (abortedProofJobId) {
|
|
@@ -154,28 +194,30 @@ export class ProvingAgent implements Traceable {
|
|
|
154
194
|
this.currentJobController.start();
|
|
155
195
|
}
|
|
156
196
|
|
|
157
|
-
|
|
197
|
+
private async reportResult<T extends ProvingRequestType>(
|
|
158
198
|
jobId: ProvingJobId,
|
|
159
199
|
type: T,
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
200
|
+
result: ProvingJobResultsMap[T] | Error,
|
|
201
|
+
): Promise<GetProvingJobResponse | undefined> {
|
|
202
|
+
let maybeJob: GetProvingJobResponse | undefined;
|
|
203
|
+
if (result instanceof AbortError) {
|
|
204
|
+
// no-op
|
|
205
|
+
this.log.warn(`Job id=${jobId} was aborted. Not reporting result back to broker`, result);
|
|
206
|
+
} else if (result instanceof Error) {
|
|
207
|
+
const retry = result.name === ProvingError.NAME ? (result as ProvingError).retry : false;
|
|
208
|
+
this.log.error(
|
|
209
|
+
`Job id=${jobId} type=${ProvingRequestType[type]} failed err=${result.message} retry=${retry}`,
|
|
210
|
+
result,
|
|
211
|
+
);
|
|
212
|
+
maybeJob = await this.broker.reportProvingJobError(jobId, result.message, retry, {
|
|
213
|
+
allowList: this.proofAllowList,
|
|
214
|
+
});
|
|
215
|
+
} else {
|
|
169
216
|
const outputUri = await this.proofStore.saveProofOutput(jobId, type, result);
|
|
170
217
|
this.log.info(`Job id=${jobId} type=${ProvingRequestType[type]} completed outputUri=${truncate(outputUri)}`);
|
|
171
218
|
maybeJob = await this.broker.reportProvingJobSuccess(jobId, outputUri, { allowList: this.proofAllowList });
|
|
172
219
|
}
|
|
173
220
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
await this.startJob(job, time);
|
|
177
|
-
} else {
|
|
178
|
-
this.idleTimer = new Timer();
|
|
179
|
-
}
|
|
180
|
-
};
|
|
221
|
+
return maybeJob;
|
|
222
|
+
}
|
|
181
223
|
}
|
|
@@ -1,17 +1,19 @@
|
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import { createLogger } from '@aztec/foundation/log';
|
|
2
3
|
import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
3
4
|
import { PriorityMemoryQueue } from '@aztec/foundation/queue';
|
|
4
5
|
import { Timer } from '@aztec/foundation/timer';
|
|
5
|
-
import
|
|
6
|
-
GetProvingJobResponse,
|
|
7
|
-
ProofUri,
|
|
8
|
-
ProvingJob,
|
|
9
|
-
ProvingJobConsumer,
|
|
10
|
-
ProvingJobFilter,
|
|
11
|
-
ProvingJobId,
|
|
12
|
-
ProvingJobProducer,
|
|
13
|
-
ProvingJobSettledResult,
|
|
14
|
-
ProvingJobStatus,
|
|
6
|
+
import {
|
|
7
|
+
type GetProvingJobResponse,
|
|
8
|
+
type ProofUri,
|
|
9
|
+
type ProvingJob,
|
|
10
|
+
type ProvingJobConsumer,
|
|
11
|
+
type ProvingJobFilter,
|
|
12
|
+
type ProvingJobId,
|
|
13
|
+
type ProvingJobProducer,
|
|
14
|
+
type ProvingJobSettledResult,
|
|
15
|
+
type ProvingJobStatus,
|
|
16
|
+
tryStop,
|
|
15
17
|
} from '@aztec/stdlib/interfaces/server';
|
|
16
18
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
17
19
|
import {
|
|
@@ -43,20 +45,33 @@ type EnqueuedProvingJob = Pick<ProvingJob, 'id' | 'epochNumber'>;
|
|
|
43
45
|
export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable {
|
|
44
46
|
private queues: ProvingQueues = {
|
|
45
47
|
[ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
46
|
-
[ProvingRequestType.
|
|
48
|
+
[ProvingRequestType.PUBLIC_CHONK_VERIFIER]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
47
49
|
|
|
48
|
-
[ProvingRequestType.
|
|
49
|
-
[ProvingRequestType.
|
|
50
|
-
[ProvingRequestType.
|
|
50
|
+
[ProvingRequestType.PRIVATE_TX_BASE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
51
|
+
[ProvingRequestType.PUBLIC_TX_BASE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
52
|
+
[ProvingRequestType.TX_MERGE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
51
53
|
[ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
52
54
|
|
|
53
55
|
[ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
56
|
+
[ProvingRequestType.BLOCK_ROOT_FIRST_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
57
|
+
[ProvingRequestType.BLOCK_ROOT_SINGLE_TX_FIRST_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(
|
|
58
|
+
provingJobComparator,
|
|
59
|
+
),
|
|
60
|
+
[ProvingRequestType.BLOCK_ROOT_EMPTY_TX_FIRST_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(
|
|
61
|
+
provingJobComparator,
|
|
62
|
+
),
|
|
54
63
|
[ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
55
|
-
[ProvingRequestType.
|
|
56
|
-
|
|
64
|
+
[ProvingRequestType.BLOCK_ROOT_SINGLE_TX_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
65
|
+
|
|
66
|
+
[ProvingRequestType.CHECKPOINT_ROOT_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
67
|
+
[ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(
|
|
68
|
+
provingJobComparator,
|
|
69
|
+
),
|
|
70
|
+
[ProvingRequestType.CHECKPOINT_MERGE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
71
|
+
[ProvingRequestType.CHECKPOINT_PADDING_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
57
72
|
|
|
58
|
-
[ProvingRequestType.
|
|
59
|
-
[ProvingRequestType.
|
|
73
|
+
[ProvingRequestType.PARITY_BASE]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
74
|
+
[ProvingRequestType.PARITY_ROOT]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
|
|
60
75
|
};
|
|
61
76
|
|
|
62
77
|
// holds a copy of the database in memory in order to quickly fulfill requests
|
|
@@ -93,7 +108,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
93
108
|
/**
|
|
94
109
|
* The broker keeps track of the highest epoch its seen.
|
|
95
110
|
* This information is used for garbage collection: once it reaches the next epoch, it can start pruning the database of old state.
|
|
96
|
-
* It is important that this value is
|
|
111
|
+
* It is important that this value is initialized to zero. This ensures that we don't delete any old jobs until the current
|
|
97
112
|
* process instance receives a job request informing it of the actual current highest epoch
|
|
98
113
|
* Example:
|
|
99
114
|
* proving epoch 11 - the broker will wipe all jobs for epochs 9 and lower
|
|
@@ -184,7 +199,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
184
199
|
this.logger.warn('ProvingBroker not started');
|
|
185
200
|
return Promise.resolve();
|
|
186
201
|
}
|
|
187
|
-
await this.cleanupPromise
|
|
202
|
+
await tryStop(this.cleanupPromise);
|
|
188
203
|
}
|
|
189
204
|
|
|
190
205
|
public enqueueProvingJob(job: ProvingJob): Promise<ProvingJobStatus> {
|
|
@@ -287,6 +302,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
287
302
|
this.resultsCache.delete(id);
|
|
288
303
|
this.inProgress.delete(id);
|
|
289
304
|
this.retries.delete(id);
|
|
305
|
+
this.enqueuedAt.delete(id);
|
|
290
306
|
}
|
|
291
307
|
}
|
|
292
308
|
|
|
@@ -313,7 +329,6 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
313
329
|
return Promise.resolve(notifications.concat(completedJobs));
|
|
314
330
|
}
|
|
315
331
|
|
|
316
|
-
// eslint-disable-next-line require-await
|
|
317
332
|
#getProvingJob(filter: ProvingJobFilter = { allowList: [] }): { job: ProvingJob; time: number } | undefined {
|
|
318
333
|
const allowedProofs: ProvingRequestType[] =
|
|
319
334
|
Array.isArray(filter.allowList) && filter.allowList.length > 0
|
|
@@ -340,6 +355,8 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
340
355
|
const enqueuedAt = this.enqueuedAt.get(job.id);
|
|
341
356
|
if (enqueuedAt) {
|
|
342
357
|
this.instrumentation.recordJobWait(job.type, enqueuedAt);
|
|
358
|
+
// we can clear this flag now.
|
|
359
|
+
this.enqueuedAt.delete(job.id);
|
|
343
360
|
}
|
|
344
361
|
|
|
345
362
|
return { job, time };
|
|
@@ -554,7 +571,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
554
571
|
this.reEnqueueExpiredJobs();
|
|
555
572
|
const oldestEpochToKeep = this.oldestEpochToKeep();
|
|
556
573
|
if (oldestEpochToKeep > 0) {
|
|
557
|
-
await this.database.deleteAllProvingJobsOlderThanEpoch(oldestEpochToKeep);
|
|
574
|
+
await this.database.deleteAllProvingJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
|
|
558
575
|
this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
|
|
559
576
|
}
|
|
560
577
|
}
|
|
@@ -571,7 +588,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
571
588
|
|
|
572
589
|
if (jobsToClean.length > 0) {
|
|
573
590
|
this.cleanUpProvingJobState(jobsToClean);
|
|
574
|
-
this.logger.
|
|
591
|
+
this.logger.verbose(`Cleaned up proving jobs=${jobsToClean.length}`);
|
|
575
592
|
}
|
|
576
593
|
}
|
|
577
594
|
|
|
@@ -671,17 +688,23 @@ function proofTypeComparator(a: ProvingRequestType, b: ProvingRequestType): -1 |
|
|
|
671
688
|
* The aim is that this will speed up block proving as the closer we get to a block's root proof the more likely it
|
|
672
689
|
* is to get picked up by agents
|
|
673
690
|
*/
|
|
674
|
-
const PROOF_TYPES_IN_PRIORITY_ORDER: ProvingRequestType[] = [
|
|
691
|
+
export const PROOF_TYPES_IN_PRIORITY_ORDER: ProvingRequestType[] = [
|
|
692
|
+
ProvingRequestType.ROOT_ROLLUP,
|
|
693
|
+
ProvingRequestType.BLOCK_ROOT_FIRST_ROLLUP,
|
|
694
|
+
ProvingRequestType.BLOCK_ROOT_SINGLE_TX_FIRST_ROLLUP,
|
|
695
|
+
ProvingRequestType.BLOCK_ROOT_EMPTY_TX_FIRST_ROLLUP,
|
|
675
696
|
ProvingRequestType.BLOCK_ROOT_ROLLUP,
|
|
676
|
-
ProvingRequestType.
|
|
697
|
+
ProvingRequestType.BLOCK_ROOT_SINGLE_TX_ROLLUP,
|
|
677
698
|
ProvingRequestType.BLOCK_MERGE_ROLLUP,
|
|
678
|
-
ProvingRequestType.
|
|
679
|
-
ProvingRequestType.
|
|
680
|
-
ProvingRequestType.
|
|
681
|
-
ProvingRequestType.
|
|
699
|
+
ProvingRequestType.CHECKPOINT_ROOT_ROLLUP,
|
|
700
|
+
ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP,
|
|
701
|
+
ProvingRequestType.CHECKPOINT_MERGE_ROLLUP,
|
|
702
|
+
ProvingRequestType.CHECKPOINT_PADDING_ROLLUP,
|
|
703
|
+
ProvingRequestType.TX_MERGE_ROLLUP,
|
|
704
|
+
ProvingRequestType.PUBLIC_TX_BASE_ROLLUP,
|
|
705
|
+
ProvingRequestType.PRIVATE_TX_BASE_ROLLUP,
|
|
682
706
|
ProvingRequestType.PUBLIC_VM,
|
|
683
|
-
ProvingRequestType.
|
|
684
|
-
ProvingRequestType.
|
|
685
|
-
ProvingRequestType.
|
|
686
|
-
ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP,
|
|
707
|
+
ProvingRequestType.PUBLIC_CHONK_VERIFIER,
|
|
708
|
+
ProvingRequestType.PARITY_ROOT,
|
|
709
|
+
ProvingRequestType.PARITY_BASE,
|
|
687
710
|
];
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import {
|
|
2
3
|
type ProofUri,
|
|
3
4
|
type ProvingJob,
|
|
@@ -43,7 +44,7 @@ export class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
43
44
|
return Promise.resolve();
|
|
44
45
|
}
|
|
45
46
|
|
|
46
|
-
deleteAllProvingJobsOlderThanEpoch(epochNumber:
|
|
47
|
+
deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void> {
|
|
47
48
|
const toDelete = [
|
|
48
49
|
...Array.from(this.jobs.keys()).filter(x => getEpochFromProvingJobId(x) < epochNumber),
|
|
49
50
|
...Array.from(this.results.keys()).filter(x => getEpochFromProvingJobId(x) < epochNumber),
|
|
@@ -52,7 +53,7 @@ export class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
52
53
|
}
|
|
53
54
|
|
|
54
55
|
async *allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]> {
|
|
55
|
-
for
|
|
56
|
+
for (const item of this.jobs.values()) {
|
|
56
57
|
yield [item, this.results.get(item.id)] as const;
|
|
57
58
|
}
|
|
58
59
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
|
|
2
3
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
3
4
|
import { BatchQueue } from '@aztec/foundation/queue';
|
|
@@ -46,9 +47,9 @@ class SingleEpochDatabase {
|
|
|
46
47
|
|
|
47
48
|
async *allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]> {
|
|
48
49
|
for await (const jobStr of this.jobs.valuesAsync()) {
|
|
49
|
-
const job =
|
|
50
|
+
const job = jsonParseWithSchema(jobStr, ProvingJob);
|
|
50
51
|
const resultStr = await this.jobResults.getAsync(job.id);
|
|
51
|
-
const result = resultStr ?
|
|
52
|
+
const result = resultStr ? jsonParseWithSchema(resultStr, ProvingJobSettledResult) : undefined;
|
|
52
53
|
yield [job, result];
|
|
53
54
|
}
|
|
54
55
|
}
|
|
@@ -104,14 +105,15 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
104
105
|
const jobsToAdd = items.filter((item): item is ProvingJob => 'id' in item);
|
|
105
106
|
const resultsToAdd = items.filter((item): item is [ProvingJobId, ProvingJobSettledResult] => Array.isArray(item));
|
|
106
107
|
|
|
107
|
-
const db = await this.getEpochDatabase(epochNumber);
|
|
108
|
+
const db = await this.getEpochDatabase(EpochNumber(epochNumber));
|
|
108
109
|
await db.batchWrite(jobsToAdd, resultsToAdd);
|
|
109
110
|
}
|
|
110
111
|
|
|
111
112
|
private async estimateSize() {
|
|
112
113
|
const sizes = await Promise.all(Array.from(this.epochs.values()).map(x => x.estimateSize()));
|
|
113
114
|
return {
|
|
114
|
-
mappingSize: this.config.
|
|
115
|
+
mappingSize: this.config.dataStoreMapSizeKb,
|
|
116
|
+
physicalFileSize: sizes.reduce((prev, curr) => prev + curr.physicalFileSize, 0),
|
|
115
117
|
numItems: sizes.reduce((prev, curr) => prev + curr.numItems, 0),
|
|
116
118
|
actualSize: sizes.reduce((prev, curr) => prev + curr.actualSize, 0),
|
|
117
119
|
};
|
|
@@ -136,13 +138,13 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
136
138
|
continue;
|
|
137
139
|
}
|
|
138
140
|
logger.info(
|
|
139
|
-
`Loading broker database for epoch ${epochNumber} from ${fullDirectory} with map size ${config.
|
|
141
|
+
`Loading broker database for epoch ${epochNumber} from ${fullDirectory} with map size ${config.dataStoreMapSizeKb}KB`,
|
|
140
142
|
);
|
|
141
143
|
const db = await openVersionedStoreAt(
|
|
142
144
|
fullDirectory,
|
|
143
145
|
SingleEpochDatabase.SCHEMA_VERSION,
|
|
144
146
|
config.l1Contracts.rollupAddress,
|
|
145
|
-
config.
|
|
147
|
+
config.dataStoreMapSizeKb,
|
|
146
148
|
);
|
|
147
149
|
const epochDb = new SingleEpochDatabase(db);
|
|
148
150
|
epochs.set(epochNumber, epochDb);
|
|
@@ -163,14 +165,14 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
163
165
|
}
|
|
164
166
|
}
|
|
165
167
|
|
|
166
|
-
async deleteAllProvingJobsOlderThanEpoch(epochNumber:
|
|
167
|
-
const oldEpochs = Array.from(this.epochs.keys()).filter(e => e < epochNumber);
|
|
168
|
+
async deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void> {
|
|
169
|
+
const oldEpochs = Array.from(this.epochs.keys()).filter(e => e < Number(epochNumber));
|
|
168
170
|
for (const old of oldEpochs) {
|
|
169
171
|
const db = this.epochs.get(old);
|
|
170
172
|
if (!db) {
|
|
171
173
|
continue;
|
|
172
174
|
}
|
|
173
|
-
this.logger.
|
|
175
|
+
this.logger.verbose(`Deleting broker database for epoch ${old}`);
|
|
174
176
|
await db.delete();
|
|
175
177
|
this.epochs.delete(old);
|
|
176
178
|
}
|
|
@@ -195,19 +197,19 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
195
197
|
return this.batchQueue.put([id, { status: 'fulfilled', value }], getEpochFromProvingJobId(id));
|
|
196
198
|
}
|
|
197
199
|
|
|
198
|
-
private async getEpochDatabase(epochNumber:
|
|
200
|
+
private async getEpochDatabase(epochNumber: EpochNumber): Promise<SingleEpochDatabase> {
|
|
199
201
|
let epochDb = this.epochs.get(epochNumber);
|
|
200
202
|
if (!epochDb) {
|
|
201
203
|
const newEpochDirectory = join(this.config.dataDirectory!, epochNumber.toString());
|
|
202
204
|
await mkdir(newEpochDirectory, { recursive: true });
|
|
203
205
|
this.logger.info(
|
|
204
|
-
`Creating broker database for epoch ${epochNumber} at ${newEpochDirectory} with map size ${this.config.
|
|
206
|
+
`Creating broker database for epoch ${epochNumber} at ${newEpochDirectory} with map size ${this.config.dataStoreMapSizeKb}`,
|
|
205
207
|
);
|
|
206
208
|
const db = await openVersionedStoreAt(
|
|
207
209
|
newEpochDirectory,
|
|
208
210
|
SingleEpochDatabase.SCHEMA_VERSION,
|
|
209
211
|
this.config.l1Contracts.rollupAddress,
|
|
210
|
-
this.config.
|
|
212
|
+
this.config.dataStoreMapSizeKb,
|
|
211
213
|
);
|
|
212
214
|
epochDb = new SingleEpochDatabase(db);
|
|
213
215
|
this.epochs.set(epochNumber, epochDb);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import type { ProofUri, ProvingJob, ProvingJobId, ProvingJobSettledResult } from '@aztec/stdlib/interfaces/server';
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -14,7 +15,7 @@ export interface ProvingBrokerDatabase {
|
|
|
14
15
|
* Deletes all proving jobs belonging to epochs older than the given epoch
|
|
15
16
|
* @param epochNumber - The epoch number beyond which jobs should be deleted
|
|
16
17
|
*/
|
|
17
|
-
deleteAllProvingJobsOlderThanEpoch(epochNumber:
|
|
18
|
+
deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void>;
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
21
|
* Returns an iterator over all saved proving jobs
|