@aztec/prover-client 0.0.0-test.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dest/bin/get-proof-inputs.d.ts +2 -0
- package/dest/bin/get-proof-inputs.d.ts.map +1 -0
- package/dest/bin/get-proof-inputs.js +51 -0
- package/dest/block_builder/index.d.ts +6 -0
- package/dest/block_builder/index.d.ts.map +1 -0
- package/dest/block_builder/index.js +1 -0
- package/dest/block_builder/light.d.ts +33 -0
- package/dest/block_builder/light.d.ts.map +1 -0
- package/dest/block_builder/light.js +82 -0
- package/dest/config.d.ts +17 -0
- package/dest/config.d.ts.map +1 -0
- package/dest/config.js +39 -0
- package/dest/index.d.ts +4 -0
- package/dest/index.d.ts.map +1 -0
- package/dest/index.js +2 -0
- package/dest/mocks/fixtures.d.ts +20 -0
- package/dest/mocks/fixtures.d.ts.map +1 -0
- package/dest/mocks/fixtures.js +77 -0
- package/dest/mocks/test_context.d.ts +55 -0
- package/dest/mocks/test_context.d.ts.map +1 -0
- package/dest/mocks/test_context.js +193 -0
- package/dest/orchestrator/block-building-helpers.d.ts +55 -0
- package/dest/orchestrator/block-building-helpers.d.ts.map +1 -0
- package/dest/orchestrator/block-building-helpers.js +285 -0
- package/dest/orchestrator/block-proving-state.d.ts +76 -0
- package/dest/orchestrator/block-proving-state.d.ts.map +1 -0
- package/dest/orchestrator/block-proving-state.js +269 -0
- package/dest/orchestrator/epoch-proving-state.d.ts +60 -0
- package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -0
- package/dest/orchestrator/epoch-proving-state.js +163 -0
- package/dest/orchestrator/index.d.ts +2 -0
- package/dest/orchestrator/index.d.ts.map +1 -0
- package/dest/orchestrator/index.js +1 -0
- package/dest/orchestrator/orchestrator.d.ts +110 -0
- package/dest/orchestrator/orchestrator.d.ts.map +1 -0
- package/dest/orchestrator/orchestrator.js +690 -0
- package/dest/orchestrator/orchestrator_metrics.d.ts +8 -0
- package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -0
- package/dest/orchestrator/orchestrator_metrics.js +17 -0
- package/dest/orchestrator/tx-proving-state.d.ts +34 -0
- package/dest/orchestrator/tx-proving-state.d.ts.map +1 -0
- package/dest/orchestrator/tx-proving-state.js +94 -0
- package/dest/prover-client/factory.d.ts +6 -0
- package/dest/prover-client/factory.d.ts.map +1 -0
- package/dest/prover-client/factory.js +5 -0
- package/dest/prover-client/index.d.ts +3 -0
- package/dest/prover-client/index.d.ts.map +1 -0
- package/dest/prover-client/index.js +2 -0
- package/dest/prover-client/prover-client.d.ts +42 -0
- package/dest/prover-client/prover-client.d.ts.map +1 -0
- package/dest/prover-client/prover-client.js +110 -0
- package/dest/prover-client/server-epoch-prover.d.ts +28 -0
- package/dest/prover-client/server-epoch-prover.d.ts.map +1 -0
- package/dest/prover-client/server-epoch-prover.js +40 -0
- package/dest/proving_broker/broker_prover_facade.d.ts +46 -0
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -0
- package/dest/proving_broker/broker_prover_facade.js +344 -0
- package/dest/proving_broker/config.d.ts +83 -0
- package/dest/proving_broker/config.d.ts.map +1 -0
- package/dest/proving_broker/config.js +104 -0
- package/dest/proving_broker/factory.d.ts +5 -0
- package/dest/proving_broker/factory.d.ts.map +1 -0
- package/dest/proving_broker/factory.js +9 -0
- package/dest/proving_broker/fixtures.d.ts +5 -0
- package/dest/proving_broker/fixtures.d.ts.map +1 -0
- package/dest/proving_broker/fixtures.js +12 -0
- package/dest/proving_broker/index.d.ts +10 -0
- package/dest/proving_broker/index.d.ts.map +1 -0
- package/dest/proving_broker/index.js +9 -0
- package/dest/proving_broker/proof_store/factory.d.ts +6 -0
- package/dest/proving_broker/proof_store/factory.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/factory.js +36 -0
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +14 -0
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/gcs_proof_store.js +51 -0
- package/dest/proving_broker/proof_store/index.d.ts +4 -0
- package/dest/proving_broker/proof_store/index.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/index.js +3 -0
- package/dest/proving_broker/proof_store/inline_proof_store.d.ts +15 -0
- package/dest/proving_broker/proof_store/inline_proof_store.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/inline_proof_store.js +41 -0
- package/dest/proving_broker/proof_store/proof_store.d.ts +36 -0
- package/dest/proving_broker/proof_store/proof_store.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/proof_store.js +3 -0
- package/dest/proving_broker/proving_agent.d.ts +46 -0
- package/dest/proving_broker/proving_agent.d.ts.map +1 -0
- package/dest/proving_broker/proving_agent.js +134 -0
- package/dest/proving_broker/proving_agent_instrumentation.d.ts +8 -0
- package/dest/proving_broker/proving_agent_instrumentation.d.ts.map +1 -0
- package/dest/proving_broker/proving_agent_instrumentation.js +16 -0
- package/dest/proving_broker/proving_broker.d.ts +64 -0
- package/dest/proving_broker/proving_broker.d.ts.map +1 -0
- package/dest/proving_broker/proving_broker.js +570 -0
- package/dest/proving_broker/proving_broker_database/memory.d.ts +16 -0
- package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -0
- package/dest/proving_broker/proving_broker_database/memory.js +54 -0
- package/dest/proving_broker/proving_broker_database/persisted.d.ts +25 -0
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -0
- package/dest/proving_broker/proving_broker_database/persisted.js +182 -0
- package/dest/proving_broker/proving_broker_database.d.ts +39 -0
- package/dest/proving_broker/proving_broker_database.d.ts.map +1 -0
- package/dest/proving_broker/proving_broker_database.js +3 -0
- package/dest/proving_broker/proving_broker_instrumentation.d.ts +29 -0
- package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -0
- package/dest/proving_broker/proving_broker_instrumentation.js +110 -0
- package/dest/proving_broker/proving_job_controller.d.ts +33 -0
- package/dest/proving_broker/proving_job_controller.d.ts.map +1 -0
- package/dest/proving_broker/proving_job_controller.js +166 -0
- package/dest/proving_broker/rpc.d.ts +27 -0
- package/dest/proving_broker/rpc.d.ts.map +1 -0
- package/dest/proving_broker/rpc.js +66 -0
- package/dest/test/mock_prover.d.ts +35 -0
- package/dest/test/mock_prover.d.ts.map +1 -0
- package/dest/test/mock_prover.js +82 -0
- package/package.json +112 -0
- package/src/bin/get-proof-inputs.ts +59 -0
- package/src/block_builder/index.ts +6 -0
- package/src/block_builder/light.ts +101 -0
- package/src/config.ts +55 -0
- package/src/index.ts +4 -0
- package/src/mocks/fixtures.ts +117 -0
- package/src/mocks/test_context.ts +257 -0
- package/src/orchestrator/block-building-helpers.ts +553 -0
- package/src/orchestrator/block-proving-state.ts +379 -0
- package/src/orchestrator/epoch-proving-state.ts +252 -0
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/orchestrator.ts +971 -0
- package/src/orchestrator/orchestrator_metrics.ts +22 -0
- package/src/orchestrator/tx-proving-state.ts +139 -0
- package/src/prover-client/factory.ts +14 -0
- package/src/prover-client/index.ts +2 -0
- package/src/prover-client/prover-client.ts +162 -0
- package/src/prover-client/server-epoch-prover.ts +51 -0
- package/src/proving_broker/broker_prover_facade.ts +585 -0
- package/src/proving_broker/config.ts +138 -0
- package/src/proving_broker/factory.ts +18 -0
- package/src/proving_broker/fixtures.ts +15 -0
- package/src/proving_broker/index.ts +9 -0
- package/src/proving_broker/proof_store/factory.ts +42 -0
- package/src/proving_broker/proof_store/gcs_proof_store.ts +72 -0
- package/src/proving_broker/proof_store/index.ts +3 -0
- package/src/proving_broker/proof_store/inline_proof_store.ts +63 -0
- package/src/proving_broker/proof_store/proof_store.ts +54 -0
- package/src/proving_broker/proving_agent.ts +181 -0
- package/src/proving_broker/proving_agent_instrumentation.ts +21 -0
- package/src/proving_broker/proving_broker.ts +687 -0
- package/src/proving_broker/proving_broker_database/memory.ts +63 -0
- package/src/proving_broker/proving_broker_database/persisted.ts +218 -0
- package/src/proving_broker/proving_broker_database.ts +44 -0
- package/src/proving_broker/proving_broker_instrumentation.ts +145 -0
- package/src/proving_broker/proving_job_controller.ts +194 -0
- package/src/proving_broker/rpc.ts +95 -0
- package/src/test/mock_prover.ts +253 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type ProofUri,
|
|
3
|
+
type ProvingJob,
|
|
4
|
+
type ProvingJobId,
|
|
5
|
+
type ProvingJobSettledResult,
|
|
6
|
+
getEpochFromProvingJobId,
|
|
7
|
+
} from '@aztec/stdlib/interfaces/server';
|
|
8
|
+
|
|
9
|
+
import type { ProvingBrokerDatabase } from '../proving_broker_database.js';
|
|
10
|
+
|
|
11
|
+
export class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
|
|
12
|
+
private jobs = new Map<ProvingJobId, ProvingJob>();
|
|
13
|
+
private results = new Map<ProvingJobId, ProvingJobSettledResult>();
|
|
14
|
+
|
|
15
|
+
getProvingJob(id: ProvingJobId): ProvingJob | undefined {
|
|
16
|
+
return this.jobs.get(id);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
getProvingJobResult(id: ProvingJobId): ProvingJobSettledResult | undefined {
|
|
20
|
+
return this.results.get(id);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
addProvingJob(job: ProvingJob): Promise<void> {
|
|
24
|
+
this.jobs.set(job.id, job);
|
|
25
|
+
return Promise.resolve();
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void> {
|
|
29
|
+
this.results.set(id, { status: 'fulfilled', value });
|
|
30
|
+
return Promise.resolve();
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
setProvingJobError(id: ProvingJobId, reason: string): Promise<void> {
|
|
34
|
+
this.results.set(id, { status: 'rejected', reason });
|
|
35
|
+
return Promise.resolve();
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
deleteProvingJobs(ids: ProvingJobId[]): Promise<void> {
|
|
39
|
+
for (const id of ids) {
|
|
40
|
+
this.jobs.delete(id);
|
|
41
|
+
this.results.delete(id);
|
|
42
|
+
}
|
|
43
|
+
return Promise.resolve();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
deleteAllProvingJobsOlderThanEpoch(epochNumber: number): Promise<void> {
|
|
47
|
+
const toDelete = [
|
|
48
|
+
...Array.from(this.jobs.keys()).filter(x => getEpochFromProvingJobId(x) < epochNumber),
|
|
49
|
+
...Array.from(this.results.keys()).filter(x => getEpochFromProvingJobId(x) < epochNumber),
|
|
50
|
+
];
|
|
51
|
+
return this.deleteProvingJobs(toDelete);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async *allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]> {
|
|
55
|
+
for await (const item of this.jobs.values()) {
|
|
56
|
+
yield [item, this.results.get(item.id)] as const;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
close(): Promise<void> {
|
|
61
|
+
return Promise.resolve();
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
|
|
2
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
3
|
+
import { BatchQueue } from '@aztec/foundation/queue';
|
|
4
|
+
import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
|
|
5
|
+
import { openVersionedStoreAt } from '@aztec/kv-store/lmdb-v2';
|
|
6
|
+
import {
|
|
7
|
+
type ProofUri,
|
|
8
|
+
ProvingJob,
|
|
9
|
+
type ProvingJobId,
|
|
10
|
+
ProvingJobSettledResult,
|
|
11
|
+
getEpochFromProvingJobId,
|
|
12
|
+
} from '@aztec/stdlib/interfaces/server';
|
|
13
|
+
import { Attributes, LmdbMetrics, type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
14
|
+
|
|
15
|
+
import { mkdir, readdir } from 'fs/promises';
|
|
16
|
+
import { join } from 'path';
|
|
17
|
+
|
|
18
|
+
import type { ProverBrokerConfig } from '../config.js';
|
|
19
|
+
import type { ProvingBrokerDatabase } from '../proving_broker_database.js';
|
|
20
|
+
|
|
21
|
+
class SingleEpochDatabase {
|
|
22
|
+
public static readonly SCHEMA_VERSION = 1;
|
|
23
|
+
|
|
24
|
+
private jobs: AztecAsyncMap<ProvingJobId, string>;
|
|
25
|
+
private jobResults: AztecAsyncMap<ProvingJobId, string>;
|
|
26
|
+
|
|
27
|
+
constructor(public readonly store: AztecAsyncKVStore) {
|
|
28
|
+
this.jobs = store.openMap('proving_jobs');
|
|
29
|
+
this.jobResults = store.openMap('proving_job_results');
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
estimateSize() {
|
|
33
|
+
return this.store.estimateSize();
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async batchWrite(jobs: ProvingJob[], results: Array<[ProvingJobId, ProvingJobSettledResult]>) {
|
|
37
|
+
await this.store.transactionAsync(async () => {
|
|
38
|
+
for (const job of jobs) {
|
|
39
|
+
await this.jobs.set(job.id, jsonStringify(job));
|
|
40
|
+
}
|
|
41
|
+
for (const [id, result] of results) {
|
|
42
|
+
await this.jobResults.set(id, jsonStringify(result));
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async *allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]> {
|
|
48
|
+
for await (const jobStr of this.jobs.valuesAsync()) {
|
|
49
|
+
const job = await jsonParseWithSchema(jobStr, ProvingJob);
|
|
50
|
+
const resultStr = await this.jobResults.getAsync(job.id);
|
|
51
|
+
const result = resultStr ? await jsonParseWithSchema(resultStr, ProvingJobSettledResult) : undefined;
|
|
52
|
+
yield [job, result];
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async setProvingJobError(id: ProvingJobId, reason: string): Promise<void> {
|
|
57
|
+
const result: ProvingJobSettledResult = { status: 'rejected', reason };
|
|
58
|
+
await this.jobResults.set(id, jsonStringify(result));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void> {
|
|
62
|
+
const result: ProvingJobSettledResult = { status: 'fulfilled', value };
|
|
63
|
+
await this.jobResults.set(id, jsonStringify(result));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
delete() {
|
|
67
|
+
return this.store.delete();
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
close() {
|
|
71
|
+
return this.store.close();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
76
|
+
private metrics: LmdbMetrics;
|
|
77
|
+
|
|
78
|
+
private batchQueue: BatchQueue<ProvingJob | [ProvingJobId, ProvingJobSettledResult], number>;
|
|
79
|
+
|
|
80
|
+
private constructor(
|
|
81
|
+
private epochs: Map<number, SingleEpochDatabase>,
|
|
82
|
+
private config: ProverBrokerConfig,
|
|
83
|
+
client: TelemetryClient = getTelemetryClient(),
|
|
84
|
+
private logger: Logger,
|
|
85
|
+
) {
|
|
86
|
+
this.metrics = new LmdbMetrics(
|
|
87
|
+
client.getMeter('KVBrokerDatabase'),
|
|
88
|
+
{
|
|
89
|
+
[Attributes.DB_DATA_TYPE]: 'prover-broker',
|
|
90
|
+
},
|
|
91
|
+
() => this.estimateSize(),
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
this.batchQueue = new BatchQueue(
|
|
95
|
+
(items, key) => this.commitWrites(items, key),
|
|
96
|
+
config.proverBrokerBatchSize,
|
|
97
|
+
config.proverBrokerBatchIntervalMs,
|
|
98
|
+
createLogger('proving-client:proving-broker-database:batch-queue'),
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// exposed for testing
|
|
103
|
+
public async commitWrites(items: Array<ProvingJob | [ProvingJobId, ProvingJobSettledResult]>, epochNumber: number) {
|
|
104
|
+
const jobsToAdd = items.filter((item): item is ProvingJob => 'id' in item);
|
|
105
|
+
const resultsToAdd = items.filter((item): item is [ProvingJobId, ProvingJobSettledResult] => Array.isArray(item));
|
|
106
|
+
|
|
107
|
+
const db = await this.getEpochDatabase(epochNumber);
|
|
108
|
+
await db.batchWrite(jobsToAdd, resultsToAdd);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
private async estimateSize() {
|
|
112
|
+
const sizes = await Promise.all(Array.from(this.epochs.values()).map(x => x.estimateSize()));
|
|
113
|
+
return {
|
|
114
|
+
mappingSize: this.config.dataStoreMapSizeKB,
|
|
115
|
+
numItems: sizes.reduce((prev, curr) => prev + curr.numItems, 0),
|
|
116
|
+
actualSize: sizes.reduce((prev, curr) => prev + curr.actualSize, 0),
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
public static async new(
|
|
121
|
+
config: ProverBrokerConfig,
|
|
122
|
+
client: TelemetryClient = getTelemetryClient(),
|
|
123
|
+
logger = createLogger('prover-client:proving-broker-database'),
|
|
124
|
+
) {
|
|
125
|
+
const epochs: Map<number, SingleEpochDatabase> = new Map<number, SingleEpochDatabase>();
|
|
126
|
+
const files = await readdir(config.dataDirectory!, { recursive: false, withFileTypes: true });
|
|
127
|
+
for (const file of files) {
|
|
128
|
+
if (!file.isDirectory()) {
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const fullDirectory = join(config.dataDirectory!, file.name);
|
|
132
|
+
const epochDirectory = file.name;
|
|
133
|
+
const epochNumber = parseInt(epochDirectory, 10);
|
|
134
|
+
if (!Number.isSafeInteger(epochNumber) || epochNumber < 0) {
|
|
135
|
+
logger.warn(`Found invalid epoch directory ${fullDirectory} when loading epoch databases, ignoring`);
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
logger.info(
|
|
139
|
+
`Loading broker database for epoch ${epochNumber} from ${fullDirectory} with map size ${config.dataStoreMapSizeKB}KB`,
|
|
140
|
+
);
|
|
141
|
+
const db = await openVersionedStoreAt(
|
|
142
|
+
fullDirectory,
|
|
143
|
+
SingleEpochDatabase.SCHEMA_VERSION,
|
|
144
|
+
config.l1Contracts.rollupAddress,
|
|
145
|
+
config.dataStoreMapSizeKB,
|
|
146
|
+
);
|
|
147
|
+
const epochDb = new SingleEpochDatabase(db);
|
|
148
|
+
epochs.set(epochNumber, epochDb);
|
|
149
|
+
}
|
|
150
|
+
const db = new KVBrokerDatabase(epochs, config, client, logger);
|
|
151
|
+
db.start();
|
|
152
|
+
return db;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
private start(): void {
|
|
156
|
+
this.batchQueue.start();
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async close(): Promise<void> {
|
|
160
|
+
await this.batchQueue.stop();
|
|
161
|
+
for (const [_, v] of this.epochs) {
|
|
162
|
+
await v.close();
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async deleteAllProvingJobsOlderThanEpoch(epochNumber: number): Promise<void> {
|
|
167
|
+
const oldEpochs = Array.from(this.epochs.keys()).filter(e => e < epochNumber);
|
|
168
|
+
for (const old of oldEpochs) {
|
|
169
|
+
const db = this.epochs.get(old);
|
|
170
|
+
if (!db) {
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
this.logger.info(`Deleting broker database for epoch ${old}`);
|
|
174
|
+
await db.delete();
|
|
175
|
+
this.epochs.delete(old);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
addProvingJob(job: ProvingJob): Promise<void> {
|
|
180
|
+
return this.batchQueue.put(job, job.epochNumber);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async *allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]> {
|
|
184
|
+
const iterators = Array.from(this.epochs.values()).map(x => x.allProvingJobs());
|
|
185
|
+
for (const it of iterators) {
|
|
186
|
+
yield* it;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
setProvingJobError(id: ProvingJobId, reason: string): Promise<void> {
|
|
191
|
+
return this.batchQueue.put([id, { status: 'rejected', reason }], getEpochFromProvingJobId(id));
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void> {
|
|
195
|
+
return this.batchQueue.put([id, { status: 'fulfilled', value }], getEpochFromProvingJobId(id));
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
private async getEpochDatabase(epochNumber: number): Promise<SingleEpochDatabase> {
|
|
199
|
+
let epochDb = this.epochs.get(epochNumber);
|
|
200
|
+
if (!epochDb) {
|
|
201
|
+
const newEpochDirectory = join(this.config.dataDirectory!, epochNumber.toString());
|
|
202
|
+
await mkdir(newEpochDirectory, { recursive: true });
|
|
203
|
+
this.logger.info(
|
|
204
|
+
`Creating broker database for epoch ${epochNumber} at ${newEpochDirectory} with map size ${this.config.dataStoreMapSizeKB}`,
|
|
205
|
+
);
|
|
206
|
+
const db = await openVersionedStoreAt(
|
|
207
|
+
newEpochDirectory,
|
|
208
|
+
SingleEpochDatabase.SCHEMA_VERSION,
|
|
209
|
+
this.config.l1Contracts.rollupAddress,
|
|
210
|
+
this.config.dataStoreMapSizeKB,
|
|
211
|
+
);
|
|
212
|
+
epochDb = new SingleEpochDatabase(db);
|
|
213
|
+
this.epochs.set(epochNumber, epochDb);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
return epochDb;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { ProofUri, ProvingJob, ProvingJobId, ProvingJobSettledResult } from '@aztec/stdlib/interfaces/server';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A database for storing proof requests and their results
|
|
5
|
+
*/
|
|
6
|
+
export interface ProvingBrokerDatabase {
|
|
7
|
+
/**
|
|
8
|
+
* Saves a proof request so it can be retrieved later
|
|
9
|
+
* @param job - The proof request to save
|
|
10
|
+
*/
|
|
11
|
+
addProvingJob(job: ProvingJob): Promise<void>;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Deletes all proving jobs belonging to epochs older than the given epoch
|
|
15
|
+
* @param epochNumber - The epoch number beyond which jobs should be deleted
|
|
16
|
+
*/
|
|
17
|
+
deleteAllProvingJobsOlderThanEpoch(epochNumber: number): Promise<void>;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Returns an iterator over all saved proving jobs
|
|
21
|
+
*/
|
|
22
|
+
allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Saves the result of a proof request
|
|
26
|
+
* @param id - The ID of the proof request to save the result for
|
|
27
|
+
* @param ProvingRequestType - The type of proof that was requested
|
|
28
|
+
* @param value - The result of the proof request
|
|
29
|
+
*/
|
|
30
|
+
setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Saves an error that occurred while processing a proof request
|
|
34
|
+
* @param id - The ID of the proof request to save the error for
|
|
35
|
+
* @param ProvingRequestType - The type of proof that was requested
|
|
36
|
+
* @param err - The error that occurred while processing the proof request
|
|
37
|
+
*/
|
|
38
|
+
setProvingJobError(id: ProvingJobId, err: string): Promise<void>;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Closes the database
|
|
42
|
+
*/
|
|
43
|
+
close(): Promise<void>;
|
|
44
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import type { Timer } from '@aztec/foundation/timer';
|
|
2
|
+
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
3
|
+
import {
|
|
4
|
+
Attributes,
|
|
5
|
+
type Histogram,
|
|
6
|
+
Metrics,
|
|
7
|
+
type ObservableGauge,
|
|
8
|
+
type ObservableResult,
|
|
9
|
+
type TelemetryClient,
|
|
10
|
+
type UpDownCounter,
|
|
11
|
+
ValueType,
|
|
12
|
+
} from '@aztec/telemetry-client';
|
|
13
|
+
|
|
14
|
+
export type MonitorCallback = (proofType: ProvingRequestType) => number;
|
|
15
|
+
|
|
16
|
+
export class ProvingBrokerInstrumentation {
|
|
17
|
+
private queueSize: ObservableGauge;
|
|
18
|
+
private activeJobs: ObservableGauge;
|
|
19
|
+
private resolvedJobs: UpDownCounter;
|
|
20
|
+
private rejectedJobs: UpDownCounter;
|
|
21
|
+
private timedOutJobs: UpDownCounter;
|
|
22
|
+
private cachedJobs: UpDownCounter;
|
|
23
|
+
private totalJobs: UpDownCounter;
|
|
24
|
+
private jobWait: Histogram;
|
|
25
|
+
private jobDuration: Histogram;
|
|
26
|
+
private retriedJobs: UpDownCounter;
|
|
27
|
+
|
|
28
|
+
constructor(client: TelemetryClient, name = 'ProvingBroker') {
|
|
29
|
+
const meter = client.getMeter(name);
|
|
30
|
+
|
|
31
|
+
this.queueSize = meter.createObservableGauge(Metrics.PROVING_QUEUE_SIZE, {
|
|
32
|
+
valueType: ValueType.INT,
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS, {
|
|
36
|
+
valueType: ValueType.INT,
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
this.resolvedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RESOLVED_JOBS, {
|
|
40
|
+
valueType: ValueType.INT,
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
this.rejectedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_REJECTED_JOBS, {
|
|
44
|
+
valueType: ValueType.INT,
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
this.retriedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RETRIED_JOBS, {
|
|
48
|
+
valueType: ValueType.INT,
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
this.timedOutJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, {
|
|
52
|
+
valueType: ValueType.INT,
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
this.cachedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_CACHED_JOBS, {
|
|
56
|
+
valueType: ValueType.INT,
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
this.totalJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TOTAL_JOBS, {
|
|
60
|
+
valueType: ValueType.INT,
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT, {
|
|
64
|
+
description: 'Records how long a job sits in the queue',
|
|
65
|
+
unit: 'ms',
|
|
66
|
+
valueType: ValueType.INT,
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
this.jobDuration = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_DURATION, {
|
|
70
|
+
description: 'Records how long a job takes to complete',
|
|
71
|
+
unit: 'ms',
|
|
72
|
+
valueType: ValueType.INT,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
monitorQueueDepth(fn: MonitorCallback) {
|
|
77
|
+
this.queueSize.addCallback(obs => this.observe(obs, fn));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
monitorActiveJobs(fn: MonitorCallback) {
|
|
81
|
+
this.activeJobs.addCallback(obs => this.observe(obs, fn));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
incResolvedJobs(proofType: ProvingRequestType) {
|
|
85
|
+
this.resolvedJobs.add(1, {
|
|
86
|
+
[Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
incRejectedJobs(proofType: ProvingRequestType) {
|
|
91
|
+
this.rejectedJobs.add(1, {
|
|
92
|
+
[Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
incRetriedJobs(proofType: ProvingRequestType) {
|
|
97
|
+
this.retriedJobs.add(1, {
|
|
98
|
+
[Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
incTimedOutJobs(proofType: ProvingRequestType) {
|
|
103
|
+
this.timedOutJobs.add(1, {
|
|
104
|
+
[Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
incCachedJobs(proofType: ProvingRequestType) {
|
|
109
|
+
this.cachedJobs.add(1, {
|
|
110
|
+
[Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
incTotalJobs(proofType: ProvingRequestType) {
|
|
115
|
+
this.totalJobs.add(1, {
|
|
116
|
+
[Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
recordJobWait(proofType: ProvingRequestType, msOrTimer: Timer | number) {
|
|
121
|
+
const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
|
|
122
|
+
this.jobWait.record(duration, {
|
|
123
|
+
[Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
recordJobDuration(proofType: ProvingRequestType, msOrTimer: Timer | number) {
|
|
128
|
+
const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
|
|
129
|
+
this.jobDuration.record(duration, {
|
|
130
|
+
[Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
private observe(obs: ObservableResult, fn: MonitorCallback) {
|
|
135
|
+
for (const proofType of Object.values(ProvingRequestType)) {
|
|
136
|
+
// a type predicate for TypeScript to recognize that we're only iterating over enum values
|
|
137
|
+
if (typeof proofType !== 'number') {
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
obs.observe(fn(proofType), {
|
|
141
|
+
[Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import type {
|
|
3
|
+
ProvingJobId,
|
|
4
|
+
ProvingJobInputs,
|
|
5
|
+
ProvingJobResultsMap,
|
|
6
|
+
ServerCircuitProver,
|
|
7
|
+
} from '@aztec/stdlib/interfaces/server';
|
|
8
|
+
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
9
|
+
|
|
10
|
+
export enum ProvingJobControllerStatus {
|
|
11
|
+
IDLE = 'idle',
|
|
12
|
+
PROVING = 'proving',
|
|
13
|
+
DONE = 'done',
|
|
14
|
+
ABORTED = 'aborted',
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
interface ProvingJobCompletionCallback<T extends ProvingRequestType = ProvingRequestType> {
|
|
18
|
+
(
|
|
19
|
+
jobId: ProvingJobId,
|
|
20
|
+
type: T,
|
|
21
|
+
error: Error | undefined,
|
|
22
|
+
result: ProvingJobResultsMap[T] | undefined,
|
|
23
|
+
): void | Promise<void>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export class ProvingJobController {
|
|
27
|
+
private status: ProvingJobControllerStatus = ProvingJobControllerStatus.IDLE;
|
|
28
|
+
private promise?: Promise<void>;
|
|
29
|
+
private abortController = new AbortController();
|
|
30
|
+
|
|
31
|
+
constructor(
|
|
32
|
+
private jobId: ProvingJobId,
|
|
33
|
+
private inputs: ProvingJobInputs,
|
|
34
|
+
private epochNumber: number,
|
|
35
|
+
private startedAt: number,
|
|
36
|
+
private circuitProver: ServerCircuitProver,
|
|
37
|
+
private onComplete: ProvingJobCompletionCallback,
|
|
38
|
+
private log = createLogger('prover-client:proving-agent:job-controller'),
|
|
39
|
+
) {}
|
|
40
|
+
|
|
41
|
+
public start(): void {
|
|
42
|
+
if (this.status !== ProvingJobControllerStatus.IDLE) {
|
|
43
|
+
this.log.verbose(
|
|
44
|
+
`Job controller for jobId=${this.jobId} not starting because it is not idle currentStatus=${this.status}`,
|
|
45
|
+
{
|
|
46
|
+
currentStatus: this.status,
|
|
47
|
+
jobId: this.jobId,
|
|
48
|
+
},
|
|
49
|
+
);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
this.status = ProvingJobControllerStatus.PROVING;
|
|
54
|
+
this.log.verbose(`Job controller started jobId=${this.jobId}`, {
|
|
55
|
+
jobId: this.jobId,
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
this.promise = this.generateProof()
|
|
59
|
+
.then(
|
|
60
|
+
result => {
|
|
61
|
+
if (this.status === ProvingJobControllerStatus.ABORTED) {
|
|
62
|
+
this.log.warn(`Job controller for jobId=${this.jobId} completed successfully but job was aborted`, {
|
|
63
|
+
currentStatus: this.status,
|
|
64
|
+
jobId: this.jobId,
|
|
65
|
+
});
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
this.status = ProvingJobControllerStatus.DONE;
|
|
69
|
+
this.log.verbose(`Job controller for jobId=${this.jobId} completed successfully`, {
|
|
70
|
+
jobId: this.jobId,
|
|
71
|
+
});
|
|
72
|
+
return this.onComplete(this.jobId, this.inputs.type, undefined, result);
|
|
73
|
+
},
|
|
74
|
+
error => {
|
|
75
|
+
if (this.status === ProvingJobControllerStatus.ABORTED) {
|
|
76
|
+
this.log.warn(`Job controller for jobId=${this.jobId} finished with an error but job was aborted`, {
|
|
77
|
+
currentStatus: this.status,
|
|
78
|
+
jobId: this.jobId,
|
|
79
|
+
});
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (error.name === 'AbortError') {
|
|
84
|
+
// Ignore abort errors
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
this.log.verbose(`Job controller for jobId=${this.jobId} finished with an error`, {
|
|
89
|
+
jobId: this.jobId,
|
|
90
|
+
err: error,
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
this.status = ProvingJobControllerStatus.DONE;
|
|
94
|
+
return this.onComplete(this.jobId, this.inputs.type, error, undefined);
|
|
95
|
+
},
|
|
96
|
+
)
|
|
97
|
+
.catch(err => {
|
|
98
|
+
this.log.error(`Job constroller failed to send result for jobId=${this.jobId}: ${err}`, err, {
|
|
99
|
+
jobId: this.jobId,
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
public getStatus(): ProvingJobControllerStatus {
|
|
105
|
+
return this.status;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
public abort(): void {
|
|
109
|
+
if (this.status !== ProvingJobControllerStatus.PROVING) {
|
|
110
|
+
this.log.warn(`Tried to abort job controller for jobId=${this.jobId} but it is not running`, {
|
|
111
|
+
currentStatus: this.status,
|
|
112
|
+
jobId: this.jobId,
|
|
113
|
+
});
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
this.status = ProvingJobControllerStatus.ABORTED;
|
|
118
|
+
this.abortController.abort();
|
|
119
|
+
this.log.verbose(`Aborted job controller for jobId=${this.jobId}`, {
|
|
120
|
+
jobId: this.jobId,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
public getJobId(): ProvingJobId {
|
|
125
|
+
return this.jobId;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
public getStartedAt(): number {
|
|
129
|
+
return this.startedAt;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
public getProofTypeName(): string {
|
|
133
|
+
return ProvingRequestType[this.inputs.type];
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
private async generateProof(): Promise<ProvingJobResultsMap[ProvingRequestType]> {
|
|
137
|
+
const { type, inputs } = this.inputs;
|
|
138
|
+
const signal = this.abortController.signal;
|
|
139
|
+
switch (type) {
|
|
140
|
+
case ProvingRequestType.PUBLIC_VM: {
|
|
141
|
+
return await this.circuitProver.getAvmProof(inputs, signal, this.epochNumber);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
case ProvingRequestType.PRIVATE_BASE_ROLLUP: {
|
|
145
|
+
return await this.circuitProver.getPrivateBaseRollupProof(inputs, signal, this.epochNumber);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
case ProvingRequestType.PUBLIC_BASE_ROLLUP: {
|
|
149
|
+
return await this.circuitProver.getPublicBaseRollupProof(inputs, signal, this.epochNumber);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
case ProvingRequestType.MERGE_ROLLUP: {
|
|
153
|
+
return await this.circuitProver.getMergeRollupProof(inputs, signal, this.epochNumber);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
case ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP: {
|
|
157
|
+
return await this.circuitProver.getEmptyBlockRootRollupProof(inputs, signal, this.epochNumber);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
case ProvingRequestType.BLOCK_ROOT_ROLLUP: {
|
|
161
|
+
return await this.circuitProver.getBlockRootRollupProof(inputs, signal, this.epochNumber);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
case ProvingRequestType.SINGLE_TX_BLOCK_ROOT_ROLLUP: {
|
|
165
|
+
return await this.circuitProver.getSingleTxBlockRootRollupProof(inputs, signal, this.epochNumber);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
case ProvingRequestType.BLOCK_MERGE_ROLLUP: {
|
|
169
|
+
return await this.circuitProver.getBlockMergeRollupProof(inputs, signal, this.epochNumber);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
case ProvingRequestType.ROOT_ROLLUP: {
|
|
173
|
+
return await this.circuitProver.getRootRollupProof(inputs, signal, this.epochNumber);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
case ProvingRequestType.BASE_PARITY: {
|
|
177
|
+
return await this.circuitProver.getBaseParityProof(inputs, signal, this.epochNumber);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
case ProvingRequestType.ROOT_PARITY: {
|
|
181
|
+
return await this.circuitProver.getRootParityProof(inputs, signal, this.epochNumber);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
case ProvingRequestType.TUBE_PROOF: {
|
|
185
|
+
return await this.circuitProver.getTubeProof(inputs, signal, this.epochNumber);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
default: {
|
|
189
|
+
const _exhaustive: never = type;
|
|
190
|
+
return Promise.reject(new Error(`Invalid proof request type: ${type}`));
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|