@aztec/prover-client 0.0.1-commit.6c91f13 → 0.0.1-commit.96bb3f7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/block-factory/light.d.ts +2 -2
- package/dest/block-factory/light.d.ts.map +1 -1
- package/dest/block-factory/light.js +3 -5
- package/dest/light/lightweight_checkpoint_builder.d.ts +8 -1
- package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -1
- package/dest/light/lightweight_checkpoint_builder.js +40 -4
- package/dest/mocks/test_context.d.ts +2 -1
- package/dest/mocks/test_context.d.ts.map +1 -1
- package/dest/mocks/test_context.js +8 -2
- package/dest/orchestrator/block-building-helpers.js +1 -1
- package/dest/orchestrator/orchestrator.js +413 -31
- package/dest/orchestrator/orchestrator_metrics.d.ts +1 -1
- package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator_metrics.js +2 -6
- package/dest/orchestrator/tx-proving-state.d.ts +5 -4
- package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/tx-proving-state.js +6 -6
- package/dest/prover-client/prover-client.d.ts +1 -1
- package/dest/prover-client/prover-client.d.ts.map +1 -1
- package/dest/prover-client/prover-client.js +1 -1
- package/dest/proving_broker/broker_prover_facade.d.ts +4 -3
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
- package/dest/proving_broker/proving_agent.d.ts +3 -8
- package/dest/proving_broker/proving_agent.d.ts.map +1 -1
- package/dest/proving_broker/proving_agent.js +1 -16
- package/dest/proving_broker/proving_broker.d.ts +1 -1
- package/dest/proving_broker/proving_broker.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.js +1 -10
- package/dest/proving_broker/proving_broker_database/persisted.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.js +389 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.js +11 -35
- package/dest/test/mock_prover.d.ts +2 -2
- package/dest/test/mock_prover.d.ts.map +1 -1
- package/dest/test/mock_prover.js +3 -3
- package/package.json +15 -15
- package/src/block-factory/light.ts +4 -5
- package/src/light/lightweight_checkpoint_builder.ts +56 -5
- package/src/mocks/test_context.ts +6 -1
- package/src/orchestrator/block-building-helpers.ts +1 -1
- package/src/orchestrator/orchestrator.ts +2 -2
- package/src/orchestrator/orchestrator_metrics.ts +2 -6
- package/src/orchestrator/tx-proving-state.ts +8 -11
- package/src/prover-client/prover-client.ts +1 -9
- package/src/proving_broker/broker_prover_facade.ts +2 -3
- package/src/proving_broker/proving_agent.ts +1 -17
- package/src/proving_broker/proving_broker.ts +1 -8
- package/src/proving_broker/proving_broker_database/persisted.ts +15 -1
- package/src/proving_broker/proving_broker_instrumentation.ts +10 -35
- package/src/test/mock_prover.ts +1 -8
- package/dest/proving_broker/proving_agent_instrumentation.d.ts +0 -8
- package/dest/proving_broker/proving_agent_instrumentation.d.ts.map +0 -1
- package/dest/proving_broker/proving_agent_instrumentation.js +0 -16
- package/src/proving_broker/proving_agent_instrumentation.ts +0 -21
|
@@ -7,12 +7,11 @@ import { createLogger } from '@aztec/foundation/log';
|
|
|
7
7
|
import { L2BlockNew } from '@aztec/stdlib/block';
|
|
8
8
|
import { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
9
9
|
import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
|
|
10
|
-
import {
|
|
10
|
+
import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
|
|
11
11
|
import { CheckpointHeader, computeBlockHeadersHash } from '@aztec/stdlib/rollup';
|
|
12
12
|
import { AppendOnlyTreeSnapshot, MerkleTreeId } from '@aztec/stdlib/trees';
|
|
13
13
|
import {
|
|
14
14
|
type CheckpointGlobalVariables,
|
|
15
|
-
ContentCommitment,
|
|
16
15
|
type GlobalVariables,
|
|
17
16
|
type ProcessedTx,
|
|
18
17
|
StateReference,
|
|
@@ -63,6 +62,59 @@ export class LightweightCheckpointBuilder {
|
|
|
63
62
|
return new LightweightCheckpointBuilder(checkpointNumber, constants, l1ToL2Messages, db);
|
|
64
63
|
}
|
|
65
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Resumes building a checkpoint from existing blocks. This is used for validator re-execution
|
|
67
|
+
* where blocks have already been built and their effects are already in the database.
|
|
68
|
+
* Unlike startNewCheckpoint, this does NOT append l1ToL2Messages to the tree since they
|
|
69
|
+
* were already added when the blocks were originally built.
|
|
70
|
+
*/
|
|
71
|
+
static async resumeCheckpoint(
|
|
72
|
+
checkpointNumber: CheckpointNumber,
|
|
73
|
+
constants: CheckpointGlobalVariables,
|
|
74
|
+
l1ToL2Messages: Fr[],
|
|
75
|
+
db: MerkleTreeWriteOperations,
|
|
76
|
+
existingBlocks: L2BlockNew[],
|
|
77
|
+
): Promise<LightweightCheckpointBuilder> {
|
|
78
|
+
const builder = new LightweightCheckpointBuilder(checkpointNumber, constants, l1ToL2Messages, db);
|
|
79
|
+
|
|
80
|
+
builder.logger.debug('Resuming checkpoint from existing blocks', {
|
|
81
|
+
checkpointNumber,
|
|
82
|
+
numExistingBlocks: existingBlocks.length,
|
|
83
|
+
blockNumbers: existingBlocks.map(b => b.header.getBlockNumber()),
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
// Validate block order and consistency
|
|
87
|
+
for (let i = 1; i < existingBlocks.length; i++) {
|
|
88
|
+
const prev = existingBlocks[i - 1];
|
|
89
|
+
const curr = existingBlocks[i];
|
|
90
|
+
if (curr.number !== prev.number + 1) {
|
|
91
|
+
throw new Error(`Non-sequential block numbers in resumeCheckpoint: ${prev.number} -> ${curr.number}`);
|
|
92
|
+
}
|
|
93
|
+
if (!prev.archive.root.equals(curr.header.lastArchive.root)) {
|
|
94
|
+
throw new Error(`Archive root mismatch between blocks ${prev.number} and ${curr.number}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
for (let i = 0; i < existingBlocks.length; i++) {
|
|
99
|
+
const block = existingBlocks[i];
|
|
100
|
+
const isFirstBlock = i === 0;
|
|
101
|
+
|
|
102
|
+
if (isFirstBlock) {
|
|
103
|
+
builder.lastArchives.push(block.header.lastArchive);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
builder.lastArchives.push(block.archive);
|
|
107
|
+
|
|
108
|
+
const blockBlobFields = block.toBlobFields();
|
|
109
|
+
await builder.spongeBlob.absorb(blockBlobFields);
|
|
110
|
+
builder.blobFields.push(...blockBlobFields);
|
|
111
|
+
|
|
112
|
+
builder.blocks.push(block);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return builder;
|
|
116
|
+
}
|
|
117
|
+
|
|
66
118
|
/**
|
|
67
119
|
* Adds a new block to the checkpoint. The tx effects must have already been inserted into the db if
|
|
68
120
|
* this is called after tx processing, if that's not the case, then set `insertTxsEffects` to true.
|
|
@@ -157,8 +209,6 @@ export class LightweightCheckpointBuilder {
|
|
|
157
209
|
|
|
158
210
|
const inHash = computeInHashFromL1ToL2Messages(this.l1ToL2Messages);
|
|
159
211
|
|
|
160
|
-
const outHash = computeCheckpointOutHash(blocks.map(block => block.body.txEffects.map(tx => tx.l2ToL1Msgs)));
|
|
161
|
-
|
|
162
212
|
const { slotNumber, coinbase, feeRecipient, gasFees } = this.constants;
|
|
163
213
|
|
|
164
214
|
// TODO(palla/mbps): Should we source this from the constants instead?
|
|
@@ -169,7 +219,8 @@ export class LightweightCheckpointBuilder {
|
|
|
169
219
|
|
|
170
220
|
const header = CheckpointHeader.from({
|
|
171
221
|
lastArchiveRoot: this.lastArchives[0].root,
|
|
172
|
-
|
|
222
|
+
blobsHash,
|
|
223
|
+
inHash,
|
|
173
224
|
blockHeadersHash,
|
|
174
225
|
slotNumber,
|
|
175
226
|
timestamp,
|
|
@@ -45,6 +45,7 @@ export class TestContext {
|
|
|
45
45
|
private headers: Map<number, BlockHeader> = new Map();
|
|
46
46
|
private checkpoints: Checkpoint[] = [];
|
|
47
47
|
private nextCheckpointIndex = 0;
|
|
48
|
+
private nextCheckpointNumber = CheckpointNumber(1);
|
|
48
49
|
private nextBlockNumber = 1;
|
|
49
50
|
private epochNumber = 1;
|
|
50
51
|
private feePayerBalance: Fr;
|
|
@@ -187,7 +188,8 @@ export class TestContext {
|
|
|
187
188
|
}
|
|
188
189
|
|
|
189
190
|
const checkpointIndex = this.nextCheckpointIndex++;
|
|
190
|
-
const checkpointNumber =
|
|
191
|
+
const checkpointNumber = this.nextCheckpointNumber;
|
|
192
|
+
this.nextCheckpointNumber++;
|
|
191
193
|
const slotNumber = checkpointNumber * 15; // times an arbitrary number to make it different to the checkpoint number
|
|
192
194
|
|
|
193
195
|
const constants = makeCheckpointConstants(slotNumber, constantOpts);
|
|
@@ -204,6 +206,8 @@ export class TestContext {
|
|
|
204
206
|
|
|
205
207
|
const startBlockNumber = this.nextBlockNumber;
|
|
206
208
|
const previousBlockHeader = this.getBlockHeader(BlockNumber(startBlockNumber - 1));
|
|
209
|
+
// All blocks in the same slot/checkpoint share the same timestamp.
|
|
210
|
+
const timestamp = BigInt(slotNumber * 26);
|
|
207
211
|
|
|
208
212
|
// Build global variables.
|
|
209
213
|
const blockGlobalVariables = times(numBlocks, i =>
|
|
@@ -211,6 +215,7 @@ export class TestContext {
|
|
|
211
215
|
coinbase: constants.coinbase,
|
|
212
216
|
feeRecipient: constants.feeRecipient,
|
|
213
217
|
gasFees: constants.gasFees,
|
|
218
|
+
timestamp,
|
|
214
219
|
}),
|
|
215
220
|
);
|
|
216
221
|
this.nextBlockNumber += numBlocks;
|
|
@@ -282,7 +282,7 @@ export const buildHeaderFromCircuitOutputs = runInSpan(
|
|
|
282
282
|
chainId: constants.chainId,
|
|
283
283
|
version: constants.version,
|
|
284
284
|
blockNumber: BlockNumber(blockRootRollupOutput.previousArchive.nextAvailableLeafIndex),
|
|
285
|
-
timestamp: blockRootRollupOutput.
|
|
285
|
+
timestamp: blockRootRollupOutput.timestamp,
|
|
286
286
|
slotNumber: constants.slotNumber,
|
|
287
287
|
coinbase: constants.coinbase,
|
|
288
288
|
feeRecipient: constants.feeRecipient,
|
|
@@ -1219,9 +1219,9 @@ export class ProvingOrchestrator implements EpochProver {
|
|
|
1219
1219
|
},
|
|
1220
1220
|
);
|
|
1221
1221
|
|
|
1222
|
-
this.deferredProving(provingState, doAvmProving,
|
|
1222
|
+
this.deferredProving(provingState, doAvmProving, proof => {
|
|
1223
1223
|
logger.debug(`Proven VM for tx index: ${txIndex}`);
|
|
1224
|
-
txProvingState.setAvmProof(
|
|
1224
|
+
txProvingState.setAvmProof(proof);
|
|
1225
1225
|
this.checkAndEnqueueBaseRollup(provingState, txIndex);
|
|
1226
1226
|
});
|
|
1227
1227
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Histogram, Metrics, type TelemetryClient, type Tracer
|
|
1
|
+
import { type Histogram, Metrics, type TelemetryClient, type Tracer } from '@aztec/telemetry-client';
|
|
2
2
|
|
|
3
3
|
export class ProvingOrchestratorMetrics {
|
|
4
4
|
public readonly tracer: Tracer;
|
|
@@ -9,11 +9,7 @@ export class ProvingOrchestratorMetrics {
|
|
|
9
9
|
this.tracer = client.getTracer(name);
|
|
10
10
|
const meter = client.getMeter(name);
|
|
11
11
|
|
|
12
|
-
this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION
|
|
13
|
-
unit: 'ms',
|
|
14
|
-
description: 'Duration to build base rollup inputs',
|
|
15
|
-
valueType: ValueType.INT,
|
|
16
|
-
});
|
|
12
|
+
this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION);
|
|
17
13
|
}
|
|
18
14
|
|
|
19
15
|
recordBaseRollupInputs(durationMs: number) {
|
|
@@ -2,8 +2,8 @@ import { AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED, NESTED_RECURSIVE_ROLLUP_HONK_PROO
|
|
|
2
2
|
import type { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
3
|
import { getVkData } from '@aztec/noir-protocol-circuits-types/server/vks';
|
|
4
4
|
import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
|
|
5
|
-
import type {
|
|
6
|
-
import { ProofData, ProofDataForFixedVk } from '@aztec/stdlib/proofs';
|
|
5
|
+
import type { PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
|
|
6
|
+
import { ProofData, ProofDataForFixedVk, RecursiveProof } from '@aztec/stdlib/proofs';
|
|
7
7
|
import {
|
|
8
8
|
type BaseRollupHints,
|
|
9
9
|
PrivateBaseRollupHints,
|
|
@@ -32,7 +32,7 @@ export class TxProvingState {
|
|
|
32
32
|
PublicChonkVerifierPublicInputs,
|
|
33
33
|
typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH
|
|
34
34
|
>;
|
|
35
|
-
private
|
|
35
|
+
private avmProof?: RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>;
|
|
36
36
|
|
|
37
37
|
constructor(
|
|
38
38
|
public readonly processedTx: ProcessedTx,
|
|
@@ -46,7 +46,7 @@ export class TxProvingState {
|
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
public ready() {
|
|
49
|
-
return !this.requireAvmProof || (!!this.
|
|
49
|
+
return !this.requireAvmProof || (!!this.avmProof && !!this.publicChonkVerifier);
|
|
50
50
|
}
|
|
51
51
|
|
|
52
52
|
public getAvmInputs(): AvmCircuitInputs {
|
|
@@ -80,8 +80,8 @@ export class TxProvingState {
|
|
|
80
80
|
this.publicChonkVerifier = publicChonkVerifierProofAndVk;
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
public setAvmProof(
|
|
84
|
-
this.
|
|
83
|
+
public setAvmProof(avmProof: RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>) {
|
|
84
|
+
this.avmProof = avmProof;
|
|
85
85
|
}
|
|
86
86
|
|
|
87
87
|
#getPrivateBaseInputs() {
|
|
@@ -105,7 +105,7 @@ export class TxProvingState {
|
|
|
105
105
|
if (!this.publicChonkVerifier) {
|
|
106
106
|
throw new Error('Tx not ready for proving base rollup: public chonk verifier proof undefined');
|
|
107
107
|
}
|
|
108
|
-
if (!this.
|
|
108
|
+
if (!this.avmProof) {
|
|
109
109
|
throw new Error('Tx not ready for proving base rollup: avm proof undefined');
|
|
110
110
|
}
|
|
111
111
|
if (!(this.baseRollupHints instanceof PublicBaseRollupHints)) {
|
|
@@ -114,10 +114,7 @@ export class TxProvingState {
|
|
|
114
114
|
|
|
115
115
|
const publicChonkVerifierProofData = toProofData(this.publicChonkVerifier);
|
|
116
116
|
|
|
117
|
-
const avmProofData = new ProofDataForFixedVk(
|
|
118
|
-
this.processedTx.avmProvingRequest.inputs.publicInputs,
|
|
119
|
-
this.avm.proof,
|
|
120
|
-
);
|
|
117
|
+
const avmProofData = new ProofDataForFixedVk(this.processedTx.avmProvingRequest.inputs.publicInputs, this.avmProof);
|
|
121
118
|
|
|
122
119
|
return new PublicTxBaseRollupPrivateInputs(publicChonkVerifierProofData, avmProofData, this.baseRollupHints);
|
|
123
120
|
}
|
|
@@ -130,15 +130,7 @@ export class ProverClient implements EpochProverManager {
|
|
|
130
130
|
const prover = await buildServerCircuitProver(this.config, this.telemetry);
|
|
131
131
|
this.agents = times(
|
|
132
132
|
this.config.proverAgentCount,
|
|
133
|
-
() =>
|
|
134
|
-
new ProvingAgent(
|
|
135
|
-
this.agentClient!,
|
|
136
|
-
proofStore,
|
|
137
|
-
prover,
|
|
138
|
-
[],
|
|
139
|
-
this.config.proverAgentPollIntervalMs,
|
|
140
|
-
this.telemetry,
|
|
141
|
-
),
|
|
133
|
+
() => new ProvingAgent(this.agentClient!, proofStore, prover, [], this.config.proverAgentPollIntervalMs),
|
|
142
134
|
);
|
|
143
135
|
|
|
144
136
|
await Promise.all(this.agents.map(agent => agent.start()));
|
|
@@ -11,7 +11,6 @@ import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from
|
|
|
11
11
|
import { truncate } from '@aztec/foundation/string';
|
|
12
12
|
import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
|
|
13
13
|
import {
|
|
14
|
-
type ProofAndVerificationKey,
|
|
15
14
|
type ProofUri,
|
|
16
15
|
type ProvingJobId,
|
|
17
16
|
type ProvingJobInputsMap,
|
|
@@ -23,7 +22,7 @@ import {
|
|
|
23
22
|
makeProvingJobId,
|
|
24
23
|
} from '@aztec/stdlib/interfaces/server';
|
|
25
24
|
import type { ParityBasePrivateInputs, ParityPublicInputs, ParityRootPrivateInputs } from '@aztec/stdlib/parity';
|
|
26
|
-
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
25
|
+
import { ProvingRequestType, RecursiveProof } from '@aztec/stdlib/proofs';
|
|
27
26
|
import type {
|
|
28
27
|
BlockMergeRollupPrivateInputs,
|
|
29
28
|
BlockRollupPublicInputs,
|
|
@@ -399,7 +398,7 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
|
|
|
399
398
|
inputs: AvmCircuitInputs,
|
|
400
399
|
signal?: AbortSignal,
|
|
401
400
|
epochNumber?: EpochNumber,
|
|
402
|
-
): Promise<
|
|
401
|
+
): Promise<RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>> {
|
|
403
402
|
return this.enqueueJob(
|
|
404
403
|
this.generateId(ProvingRequestType.PUBLIC_VM, inputs, epochNumber),
|
|
405
404
|
ProvingRequestType.PUBLIC_VM,
|
|
@@ -13,27 +13,16 @@ import type {
|
|
|
13
13
|
ServerCircuitProver,
|
|
14
14
|
} from '@aztec/stdlib/interfaces/server';
|
|
15
15
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
16
|
-
import {
|
|
17
|
-
type TelemetryClient,
|
|
18
|
-
type Traceable,
|
|
19
|
-
type Tracer,
|
|
20
|
-
getTelemetryClient,
|
|
21
|
-
trackSpan,
|
|
22
|
-
} from '@aztec/telemetry-client';
|
|
23
16
|
|
|
24
17
|
import type { ProofStore } from './proof_store/index.js';
|
|
25
|
-
import { ProvingAgentInstrumentation } from './proving_agent_instrumentation.js';
|
|
26
18
|
import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_controller.js';
|
|
27
19
|
|
|
28
20
|
/**
|
|
29
21
|
* A helper class that encapsulates a circuit prover and connects it to a job source.
|
|
30
22
|
*/
|
|
31
|
-
export class ProvingAgent
|
|
23
|
+
export class ProvingAgent {
|
|
32
24
|
private currentJobController?: ProvingJobController;
|
|
33
25
|
private runningPromise: RunningPromise;
|
|
34
|
-
private instrumentation: ProvingAgentInstrumentation;
|
|
35
|
-
|
|
36
|
-
public readonly tracer: Tracer;
|
|
37
26
|
|
|
38
27
|
constructor(
|
|
39
28
|
/** The source of proving jobs */
|
|
@@ -46,12 +35,8 @@ export class ProvingAgent implements Traceable {
|
|
|
46
35
|
private proofAllowList: Array<ProvingRequestType> = [],
|
|
47
36
|
/** How long to wait between jobs */
|
|
48
37
|
private pollIntervalMs = 1000,
|
|
49
|
-
/** A telemetry client through which to emit metrics */
|
|
50
|
-
client: TelemetryClient = getTelemetryClient(),
|
|
51
38
|
private log = createLogger('prover-client:proving-agent'),
|
|
52
39
|
) {
|
|
53
|
-
this.tracer = client.getTracer('ProvingAgent');
|
|
54
|
-
this.instrumentation = new ProvingAgentInstrumentation(client);
|
|
55
40
|
this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollIntervalMs);
|
|
56
41
|
}
|
|
57
42
|
|
|
@@ -85,7 +70,6 @@ export class ProvingAgent implements Traceable {
|
|
|
85
70
|
return this.runningPromise.isRunning() ? { status: 'running' } : { status: 'stopped' };
|
|
86
71
|
}
|
|
87
72
|
|
|
88
|
-
@trackSpan('ProvingAgent.safeWork')
|
|
89
73
|
private async work() {
|
|
90
74
|
// every tick we need to take one of the following actions:
|
|
91
75
|
// 1. send a hearbeat to the broker that we're working on some job
|
|
@@ -16,13 +16,7 @@ import {
|
|
|
16
16
|
tryStop,
|
|
17
17
|
} from '@aztec/stdlib/interfaces/server';
|
|
18
18
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
19
|
-
import {
|
|
20
|
-
type TelemetryClient,
|
|
21
|
-
type Traceable,
|
|
22
|
-
type Tracer,
|
|
23
|
-
getTelemetryClient,
|
|
24
|
-
trackSpan,
|
|
25
|
-
} from '@aztec/telemetry-client';
|
|
19
|
+
import { type TelemetryClient, type Traceable, type Tracer, getTelemetryClient } from '@aztec/telemetry-client';
|
|
26
20
|
|
|
27
21
|
import assert from 'assert';
|
|
28
22
|
|
|
@@ -565,7 +559,6 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
|
|
|
565
559
|
return this.#getProvingJob(filter);
|
|
566
560
|
}
|
|
567
561
|
|
|
568
|
-
@trackSpan('ProvingBroker.cleanupPass')
|
|
569
562
|
private async cleanupPass() {
|
|
570
563
|
this.cleanupStaleJobs();
|
|
571
564
|
this.reEnqueueExpiredJobs();
|
|
@@ -11,7 +11,14 @@ import {
|
|
|
11
11
|
ProvingJobSettledResult,
|
|
12
12
|
getEpochFromProvingJobId,
|
|
13
13
|
} from '@aztec/stdlib/interfaces/server';
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
Attributes,
|
|
16
|
+
LmdbMetrics,
|
|
17
|
+
type TelemetryClient,
|
|
18
|
+
type Tracer,
|
|
19
|
+
getTelemetryClient,
|
|
20
|
+
trackSpan,
|
|
21
|
+
} from '@aztec/telemetry-client';
|
|
15
22
|
|
|
16
23
|
import { mkdir, readdir } from 'fs/promises';
|
|
17
24
|
import { join } from 'path';
|
|
@@ -78,6 +85,8 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
78
85
|
|
|
79
86
|
private batchQueue: BatchQueue<ProvingJob | [ProvingJobId, ProvingJobSettledResult], number>;
|
|
80
87
|
|
|
88
|
+
public readonly tracer: Tracer;
|
|
89
|
+
|
|
81
90
|
private constructor(
|
|
82
91
|
private epochs: Map<number, SingleEpochDatabase>,
|
|
83
92
|
private config: ProverBrokerConfig,
|
|
@@ -92,6 +101,8 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
92
101
|
() => this.estimateSize(),
|
|
93
102
|
);
|
|
94
103
|
|
|
104
|
+
this.tracer = client.getTracer('KVBrokerDatabase');
|
|
105
|
+
|
|
95
106
|
this.batchQueue = new BatchQueue(
|
|
96
107
|
(items, key) => this.commitWrites(items, key),
|
|
97
108
|
config.proverBrokerBatchSize,
|
|
@@ -165,6 +176,9 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
165
176
|
}
|
|
166
177
|
}
|
|
167
178
|
|
|
179
|
+
@trackSpan('KVBrokerDatabase.deleteAllProvingJobsOlderThanEpoch', epochNumber => ({
|
|
180
|
+
[Attributes.EPOCH_NUMBER]: epochNumber,
|
|
181
|
+
}))
|
|
168
182
|
async deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void> {
|
|
169
183
|
const oldEpochs = Array.from(this.epochs.keys()).filter(e => e < Number(epochNumber));
|
|
170
184
|
for (const old of oldEpochs) {
|
|
@@ -8,7 +8,6 @@ import {
|
|
|
8
8
|
type ObservableResult,
|
|
9
9
|
type TelemetryClient,
|
|
10
10
|
type UpDownCounter,
|
|
11
|
-
ValueType,
|
|
12
11
|
} from '@aztec/telemetry-client';
|
|
13
12
|
|
|
14
13
|
export type MonitorCallback = (proofType: ProvingRequestType) => number;
|
|
@@ -28,49 +27,25 @@ export class ProvingBrokerInstrumentation {
|
|
|
28
27
|
constructor(client: TelemetryClient, name = 'ProvingBroker') {
|
|
29
28
|
const meter = client.getMeter(name);
|
|
30
29
|
|
|
31
|
-
this.queueSize = meter.createObservableGauge(Metrics.PROVING_QUEUE_SIZE
|
|
32
|
-
valueType: ValueType.INT,
|
|
33
|
-
});
|
|
30
|
+
this.queueSize = meter.createObservableGauge(Metrics.PROVING_QUEUE_SIZE);
|
|
34
31
|
|
|
35
|
-
this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS
|
|
36
|
-
valueType: ValueType.INT,
|
|
37
|
-
});
|
|
32
|
+
this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS);
|
|
38
33
|
|
|
39
|
-
this.resolvedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RESOLVED_JOBS
|
|
40
|
-
valueType: ValueType.INT,
|
|
41
|
-
});
|
|
34
|
+
this.resolvedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RESOLVED_JOBS);
|
|
42
35
|
|
|
43
|
-
this.rejectedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_REJECTED_JOBS
|
|
44
|
-
valueType: ValueType.INT,
|
|
45
|
-
});
|
|
36
|
+
this.rejectedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_REJECTED_JOBS);
|
|
46
37
|
|
|
47
|
-
this.retriedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RETRIED_JOBS
|
|
48
|
-
valueType: ValueType.INT,
|
|
49
|
-
});
|
|
38
|
+
this.retriedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RETRIED_JOBS);
|
|
50
39
|
|
|
51
|
-
this.timedOutJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TIMED_OUT_JOBS
|
|
52
|
-
valueType: ValueType.INT,
|
|
53
|
-
});
|
|
40
|
+
this.timedOutJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TIMED_OUT_JOBS);
|
|
54
41
|
|
|
55
|
-
this.cachedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_CACHED_JOBS
|
|
56
|
-
valueType: ValueType.INT,
|
|
57
|
-
});
|
|
42
|
+
this.cachedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_CACHED_JOBS);
|
|
58
43
|
|
|
59
|
-
this.totalJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TOTAL_JOBS
|
|
60
|
-
valueType: ValueType.INT,
|
|
61
|
-
});
|
|
44
|
+
this.totalJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TOTAL_JOBS);
|
|
62
45
|
|
|
63
|
-
this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT
|
|
64
|
-
description: 'Records how long a job sits in the queue',
|
|
65
|
-
unit: 'ms',
|
|
66
|
-
valueType: ValueType.INT,
|
|
67
|
-
});
|
|
46
|
+
this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT);
|
|
68
47
|
|
|
69
|
-
this.jobDuration = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_DURATION
|
|
70
|
-
description: 'Records how long a job takes to complete',
|
|
71
|
-
unit: 'ms',
|
|
72
|
-
valueType: ValueType.INT,
|
|
73
|
-
});
|
|
48
|
+
this.jobDuration = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_DURATION);
|
|
74
49
|
}
|
|
75
50
|
|
|
76
51
|
monitorQueueDepth(fn: MonitorCallback) {
|
package/src/test/mock_prover.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
2
|
AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED,
|
|
3
|
-
AVM_V2_VERIFICATION_KEY_LENGTH_IN_FIELDS_PADDED,
|
|
4
3
|
NESTED_RECURSIVE_PROOF_LENGTH,
|
|
5
4
|
NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
|
|
6
5
|
RECURSIVE_PROOF_LENGTH,
|
|
@@ -14,7 +13,6 @@ import {
|
|
|
14
13
|
type ProvingJobStatus,
|
|
15
14
|
type PublicInputsAndRecursiveProof,
|
|
16
15
|
type ServerCircuitProver,
|
|
17
|
-
makeProofAndVerificationKey,
|
|
18
16
|
makePublicInputsAndRecursiveProof,
|
|
19
17
|
} from '@aztec/stdlib/interfaces/server';
|
|
20
18
|
import type { ParityBasePrivateInputs, ParityRootPrivateInputs } from '@aztec/stdlib/parity';
|
|
@@ -106,12 +104,7 @@ export class MockProver implements ServerCircuitProver {
|
|
|
106
104
|
constructor() {}
|
|
107
105
|
|
|
108
106
|
getAvmProof(_inputs: AvmCircuitInputs, _signal?: AbortSignal, _epochNumber?: number) {
|
|
109
|
-
return Promise.resolve(
|
|
110
|
-
makeProofAndVerificationKey(
|
|
111
|
-
makeEmptyRecursiveProof(AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED),
|
|
112
|
-
VerificationKeyData.makeFake(AVM_V2_VERIFICATION_KEY_LENGTH_IN_FIELDS_PADDED),
|
|
113
|
-
),
|
|
114
|
-
);
|
|
107
|
+
return Promise.resolve(makeEmptyRecursiveProof(AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED));
|
|
115
108
|
}
|
|
116
109
|
|
|
117
110
|
getBaseParityProof(_inputs: ParityBasePrivateInputs, _signal?: AbortSignal, _epochNumber?: number) {
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { Timer } from '@aztec/foundation/timer';
|
|
2
|
-
import { type TelemetryClient } from '@aztec/telemetry-client';
|
|
3
|
-
export declare class ProvingAgentInstrumentation {
|
|
4
|
-
private idleTime;
|
|
5
|
-
constructor(client: TelemetryClient, name?: string);
|
|
6
|
-
recordIdleTime(msOrTimer: Timer | number): void;
|
|
7
|
-
}
|
|
8
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19hZ2VudF9pbnN0cnVtZW50YXRpb24uZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9wcm92aW5nX2FnZW50X2luc3RydW1lbnRhdGlvbi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxLQUFLLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUNyRCxPQUFPLEVBQTJCLEtBQUssZUFBZSxFQUFhLE1BQU0seUJBQXlCLENBQUM7QUFFbkcscUJBQWEsMkJBQTJCO0lBQ3RDLE9BQU8sQ0FBQyxRQUFRLENBQVk7SUFFNUIsWUFBWSxNQUFNLEVBQUUsZUFBZSxFQUFFLElBQUksU0FBaUIsRUFRekQ7SUFFRCxjQUFjLENBQUMsU0FBUyxFQUFFLEtBQUssR0FBRyxNQUFNLFFBR3ZDO0NBQ0YifQ==
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"proving_agent_instrumentation.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_agent_instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAA2B,KAAK,eAAe,EAAa,MAAM,yBAAyB,CAAC;AAEnG,qBAAa,2BAA2B;IACtC,OAAO,CAAC,QAAQ,CAAY;IAE5B,YAAY,MAAM,EAAE,eAAe,EAAE,IAAI,SAAiB,EAQzD;IAED,cAAc,CAAC,SAAS,EAAE,KAAK,GAAG,MAAM,QAGvC;CACF"}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { Metrics, ValueType } from '@aztec/telemetry-client';
|
|
2
|
-
export class ProvingAgentInstrumentation {
|
|
3
|
-
idleTime;
|
|
4
|
-
constructor(client, name = 'ProvingAgent'){
|
|
5
|
-
const meter = client.getMeter(name);
|
|
6
|
-
this.idleTime = meter.createHistogram(Metrics.PROVING_AGENT_IDLE, {
|
|
7
|
-
description: 'Records how long an agent was idle',
|
|
8
|
-
unit: 's',
|
|
9
|
-
valueType: ValueType.DOUBLE
|
|
10
|
-
});
|
|
11
|
-
}
|
|
12
|
-
recordIdleTime(msOrTimer) {
|
|
13
|
-
const duration = typeof msOrTimer === 'number' ? msOrTimer : msOrTimer.ms();
|
|
14
|
-
this.idleTime.record(duration / 1000);
|
|
15
|
-
}
|
|
16
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import type { Timer } from '@aztec/foundation/timer';
|
|
2
|
-
import { type Histogram, Metrics, type TelemetryClient, ValueType } from '@aztec/telemetry-client';
|
|
3
|
-
|
|
4
|
-
export class ProvingAgentInstrumentation {
|
|
5
|
-
private idleTime: Histogram;
|
|
6
|
-
|
|
7
|
-
constructor(client: TelemetryClient, name = 'ProvingAgent') {
|
|
8
|
-
const meter = client.getMeter(name);
|
|
9
|
-
|
|
10
|
-
this.idleTime = meter.createHistogram(Metrics.PROVING_AGENT_IDLE, {
|
|
11
|
-
description: 'Records how long an agent was idle',
|
|
12
|
-
unit: 's',
|
|
13
|
-
valueType: ValueType.DOUBLE,
|
|
14
|
-
});
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
recordIdleTime(msOrTimer: Timer | number) {
|
|
18
|
-
const duration = typeof msOrTimer === 'number' ? msOrTimer : msOrTimer.ms();
|
|
19
|
-
this.idleTime.record(duration / 1000);
|
|
20
|
-
}
|
|
21
|
-
}
|