@aztec/prover-client 0.0.1-commit.a89ec08 → 0.0.1-commit.aa0c64f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/config.d.ts +1 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +16 -2
- package/dest/light/lightweight_checkpoint_builder.d.ts +5 -2
- package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -1
- package/dest/light/lightweight_checkpoint_builder.js +36 -12
- package/dest/mocks/test_context.d.ts +11 -11
- package/dest/mocks/test_context.d.ts.map +1 -1
- package/dest/mocks/test_context.js +27 -29
- package/dest/orchestrator/block-building-helpers.d.ts +1 -1
- package/dest/orchestrator/checkpoint-proving-state.d.ts +20 -36
- package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/checkpoint-proving-state.js +22 -120
- package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts +161 -0
- package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts.map +1 -0
- package/dest/orchestrator/{orchestrator.js → checkpoint-sub-tree-orchestrator.js} +267 -451
- package/dest/orchestrator/chonk-cache.d.ts +39 -0
- package/dest/orchestrator/chonk-cache.d.ts.map +1 -0
- package/dest/orchestrator/chonk-cache.js +80 -0
- package/dest/orchestrator/index.d.ts +4 -2
- package/dest/orchestrator/index.d.ts.map +1 -1
- package/dest/orchestrator/index.js +3 -1
- package/dest/orchestrator/proving-scheduler.d.ts +81 -0
- package/dest/orchestrator/proving-scheduler.d.ts.map +1 -0
- package/dest/orchestrator/proving-scheduler.js +120 -0
- package/dest/orchestrator/top-tree-orchestrator.d.ts +88 -0
- package/dest/orchestrator/top-tree-orchestrator.d.ts.map +1 -0
- package/dest/orchestrator/top-tree-orchestrator.js +232 -0
- package/dest/orchestrator/top-tree-proving-state.d.ts +61 -0
- package/dest/orchestrator/top-tree-proving-state.d.ts.map +1 -0
- package/dest/orchestrator/top-tree-proving-state.js +185 -0
- package/dest/orchestrator/tx-proving-state.d.ts +3 -3
- package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
- package/dest/prover-client/prover-client.d.ts +61 -4
- package/dest/prover-client/prover-client.d.ts.map +1 -1
- package/dest/prover-client/prover-client.js +54 -7
- package/dest/proving_broker/broker_prover_facade.d.ts +3 -3
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
- package/dest/proving_broker/broker_prover_facade.js +15 -16
- package/dest/proving_broker/config.d.ts +9 -73
- package/dest/proving_broker/config.d.ts.map +1 -1
- package/dest/proving_broker/config.js +3 -3
- package/dest/proving_broker/index.d.ts +2 -1
- package/dest/proving_broker/index.d.ts.map +1 -1
- package/dest/proving_broker/index.js +1 -0
- package/dest/proving_broker/proving_broker.d.ts +2 -2
- package/dest/proving_broker/proving_broker.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.js +92 -24
- package/dest/proving_broker/proving_broker_database/memory.d.ts +3 -1
- package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/memory.js +10 -0
- package/dest/proving_broker/proving_broker_database/persisted.d.ts +13 -2
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.js +32 -5
- package/dest/proving_broker/proving_broker_database.d.ts +15 -1
- package/dest/proving_broker/proving_broker_database.d.ts.map +1 -1
- package/dest/proving_broker/rpc.d.ts +1 -1
- package/dest/proving_broker/rpc.d.ts.map +1 -1
- package/dest/proving_broker/rpc.js +64 -9
- package/dest/test/epoch_settlement.d.ts +36 -0
- package/dest/test/epoch_settlement.d.ts.map +1 -0
- package/dest/test/epoch_settlement.js +52 -0
- package/dest/test/index.d.ts +2 -0
- package/dest/test/index.d.ts.map +1 -0
- package/dest/test/index.js +1 -0
- package/dest/test/mock_prover.d.ts +4 -4
- package/dest/test/mock_prover.js +2 -2
- package/package.json +20 -19
- package/src/config.ts +18 -2
- package/src/light/lightweight_checkpoint_builder.ts +37 -20
- package/src/mocks/test_context.ts +26 -42
- package/src/orchestrator/checkpoint-proving-state.ts +21 -162
- package/src/orchestrator/{orchestrator.ts → checkpoint-sub-tree-orchestrator.ts} +415 -661
- package/src/orchestrator/chonk-cache.ts +99 -0
- package/src/orchestrator/index.ts +8 -1
- package/src/orchestrator/proving-scheduler.ts +160 -0
- package/src/orchestrator/top-tree-orchestrator.ts +384 -0
- package/src/orchestrator/top-tree-proving-state.ts +219 -0
- package/src/orchestrator/tx-proving-state.ts +3 -3
- package/src/prover-client/prover-client.ts +128 -18
- package/src/proving_broker/broker_prover_facade.ts +12 -18
- package/src/proving_broker/config.ts +3 -2
- package/src/proving_broker/index.ts +1 -0
- package/src/proving_broker/proving_broker.ts +84 -22
- package/src/proving_broker/proving_broker_database/memory.ts +10 -0
- package/src/proving_broker/proving_broker_database/persisted.ts +42 -7
- package/src/proving_broker/proving_broker_database.ts +16 -0
- package/src/proving_broker/rpc.ts +24 -21
- package/src/test/epoch_settlement.ts +82 -0
- package/src/test/index.ts +1 -0
- package/src/test/mock_prover.ts +2 -2
- package/dest/orchestrator/epoch-proving-state.d.ts +0 -75
- package/dest/orchestrator/epoch-proving-state.d.ts.map +0 -1
- package/dest/orchestrator/epoch-proving-state.js +0 -269
- package/dest/orchestrator/orchestrator.d.ts +0 -130
- package/dest/orchestrator/orchestrator.d.ts.map +0 -1
- package/dest/prover-client/server-epoch-prover.d.ts +0 -32
- package/dest/prover-client/server-epoch-prover.d.ts.map +0 -1
- package/dest/prover-client/server-epoch-prover.js +0 -40
- package/src/orchestrator/epoch-proving-state.ts +0 -380
- package/src/prover-client/server-epoch-prover.ts +0 -69
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { BBNativeRollupProver, TestCircuitProver } from '@aztec/bb-prover';
|
|
2
2
|
import { times } from '@aztec/foundation/collection';
|
|
3
3
|
import { createLogger } from '@aztec/foundation/log';
|
|
4
|
+
import { SerialQueue } from '@aztec/foundation/queue';
|
|
4
5
|
import { NativeACVMSimulator } from '@aztec/simulator/server';
|
|
5
6
|
import { tryStop } from '@aztec/stdlib/interfaces/server';
|
|
6
7
|
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
7
|
-
import {
|
|
8
|
+
import { CheckpointSubTreeOrchestrator } from '../orchestrator/checkpoint-sub-tree-orchestrator.js';
|
|
9
|
+
import { TopTreeOrchestrator } from '../orchestrator/top-tree-orchestrator.js';
|
|
8
10
|
import { BrokerCircuitProverFacade } from '../proving_broker/broker_prover_facade.js';
|
|
9
11
|
import { InlineProofStore, createProofStore } from '../proving_broker/proof_store/index.js';
|
|
10
12
|
import { ProvingAgent } from '../proving_broker/proving_agent.js';
|
|
11
|
-
import { ServerEpochProver } from './server-epoch-prover.js';
|
|
12
13
|
/** Manages proving of epochs by orchestrating the proving of individual blocks relying on a pool of prover agents. */ export class ProverClient {
|
|
13
14
|
config;
|
|
14
15
|
worldState;
|
|
@@ -20,6 +21,17 @@ import { ServerEpochProver } from './server-epoch-prover.js';
|
|
|
20
21
|
log;
|
|
21
22
|
running;
|
|
22
23
|
agents;
|
|
24
|
+
/**
|
|
25
|
+
* The single broker facade shared by every orchestrator created from this client.
|
|
26
|
+
* Constructed lazily on `start()` and torn down on `stop()` — see the comment on
|
|
27
|
+
* `EpochProverFactory` for why a single shared facade is required.
|
|
28
|
+
*/ facade;
|
|
29
|
+
/**
|
|
30
|
+
* Single deferred-proving-job queue shared across every orchestrator (sub-trees and
|
|
31
|
+
* top-trees, across every concurrent epoch session). Throttles the total rate of job
|
|
32
|
+
* submission to the broker once, rather than once per orchestrator. Started lazily
|
|
33
|
+
* alongside the facade and cancelled on `stop()`.
|
|
34
|
+
*/ deferredJobQueue;
|
|
23
35
|
constructor(config, worldState, orchestratorClient, proofStore, failedProofStore, agentClient, telemetry = getTelemetryClient(), log = createLogger('prover-client:tx-prover')){
|
|
24
36
|
this.config = config;
|
|
25
37
|
this.worldState = worldState;
|
|
@@ -32,11 +44,38 @@ import { ServerEpochProver } from './server-epoch-prover.js';
|
|
|
32
44
|
this.running = false;
|
|
33
45
|
this.agents = [];
|
|
34
46
|
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
47
|
+
/**
|
|
48
|
+
* Lazy-init the shared facade. The broker delivers each completed-job notification
|
|
49
|
+
* exactly once (drained on the first `getCompletedJobs` poll), so we cannot start
|
|
50
|
+
* a shared facade alongside the per-call facades that `createEpochProver` builds —
|
|
51
|
+
* they would race for notifications and one side would silently drop them. Starting
|
|
52
|
+
* the shared facade only on first use of one of the new factory methods keeps the
|
|
53
|
+
* legacy `createEpochProver` path race-free.
|
|
54
|
+
*/ getFacade() {
|
|
55
|
+
if (!this.running) {
|
|
56
|
+
throw new Error('ProverClient is not running; call start() before constructing orchestrators.');
|
|
57
|
+
}
|
|
58
|
+
if (!this.facade) {
|
|
59
|
+
this.facade = new BrokerCircuitProverFacade(this.orchestratorClient, this.proofStore, this.failedProofStore, undefined, this.log.getBindings());
|
|
60
|
+
this.facade.start();
|
|
61
|
+
}
|
|
62
|
+
return this.facade;
|
|
63
|
+
}
|
|
64
|
+
/** Lazy-init the shared deferred-job queue, started with the configured enqueue concurrency. */ getDeferredJobQueue() {
|
|
65
|
+
if (!this.running) {
|
|
66
|
+
throw new Error('ProverClient is not running; call start() before constructing orchestrators.');
|
|
67
|
+
}
|
|
68
|
+
if (!this.deferredJobQueue) {
|
|
69
|
+
this.deferredJobQueue = new SerialQueue();
|
|
70
|
+
this.deferredJobQueue.start(this.config.enqueueConcurrency);
|
|
71
|
+
}
|
|
72
|
+
return this.deferredJobQueue;
|
|
73
|
+
}
|
|
74
|
+
createCheckpointSubTreeOrchestrator(chonkCache, epochNumber, checkpointConstants, l1ToL2Messages, totalNumBlocks, headerOfLastBlockInPreviousCheckpoint) {
|
|
75
|
+
return CheckpointSubTreeOrchestrator.start(this.worldState, this.getFacade(), this.config.proverId, chonkCache, epochNumber, this.config.cancelJobsOnStop, this.getDeferredJobQueue(), checkpointConstants, l1ToL2Messages, totalNumBlocks, headerOfLastBlockInPreviousCheckpoint, this.telemetry, this.log.getBindings());
|
|
76
|
+
}
|
|
77
|
+
createTopTreeOrchestrator() {
|
|
78
|
+
return new TopTreeOrchestrator(this.getFacade(), this.config.proverId, this.getDeferredJobQueue(), this.telemetry, this.log.getBindings());
|
|
40
79
|
}
|
|
41
80
|
getProverId() {
|
|
42
81
|
return this.config.proverId;
|
|
@@ -69,6 +108,14 @@ import { ServerEpochProver } from './server-epoch-prover.js';
|
|
|
69
108
|
}
|
|
70
109
|
this.running = false;
|
|
71
110
|
await this.stopAgents();
|
|
111
|
+
if (this.deferredJobQueue) {
|
|
112
|
+
await this.deferredJobQueue.cancel();
|
|
113
|
+
this.deferredJobQueue = undefined;
|
|
114
|
+
}
|
|
115
|
+
if (this.facade) {
|
|
116
|
+
await tryStop(this.facade, this.log);
|
|
117
|
+
this.facade = undefined;
|
|
118
|
+
}
|
|
72
119
|
await tryStop(this.orchestratorClient);
|
|
73
120
|
}
|
|
74
121
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { AVM_V2_PROOF_LENGTH_IN_FIELDS, NESTED_RECURSIVE_PROOF_LENGTH, NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH, RECURSIVE_PROOF_LENGTH } from '@aztec/constants';
|
|
2
2
|
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
3
3
|
import { type LoggerBindings } from '@aztec/foundation/log';
|
|
4
4
|
import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
|
|
@@ -32,7 +32,7 @@ export declare class BrokerCircuitProverFacade implements ServerCircuitProver {
|
|
|
32
32
|
private retrieveJobsThatShouldBeReady;
|
|
33
33
|
private backupFailedProofInputs;
|
|
34
34
|
private monitorForCompletedJobs;
|
|
35
|
-
getAvmProof(inputs: AvmCircuitInputs, signal?: AbortSignal, epochNumber?: EpochNumber): Promise<RecursiveProof<typeof
|
|
35
|
+
getAvmProof(inputs: AvmCircuitInputs, signal?: AbortSignal, epochNumber?: EpochNumber): Promise<RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS>>;
|
|
36
36
|
getBaseParityProof(inputs: ParityBasePrivateInputs, signal?: AbortSignal, epochNumber?: EpochNumber): Promise<PublicInputsAndRecursiveProof<ParityPublicInputs, typeof RECURSIVE_PROOF_LENGTH>>;
|
|
37
37
|
getTxMergeRollupProof(input: TxMergeRollupPrivateInputs, signal?: AbortSignal, epochNumber?: EpochNumber): Promise<PublicInputsAndRecursiveProof<TxRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
|
|
38
38
|
getPublicChonkVerifierProof(inputs: PublicChonkVerifierPrivateInputs, signal?: AbortSignal, epochNumber?: EpochNumber): Promise<PublicInputsAndRecursiveProof<PublicChonkVerifierPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
|
|
@@ -52,4 +52,4 @@ export declare class BrokerCircuitProverFacade implements ServerCircuitProver {
|
|
|
52
52
|
getRootRollupProof(input: RootRollupPrivateInputs, signal?: AbortSignal, epochNumber?: EpochNumber): Promise<PublicInputsAndRecursiveProof<RootRollupPublicInputs, typeof RECURSIVE_PROOF_LENGTH>>;
|
|
53
53
|
private generateId;
|
|
54
54
|
}
|
|
55
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
55
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYnJva2VyX3Byb3Zlcl9mYWNhZGUuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9icm9rZXJfcHJvdmVyX2ZhY2FkZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFDViw2QkFBNkIsRUFDN0IsNkJBQTZCLEVBQzdCLHlDQUF5QyxFQUN6QyxzQkFBc0IsRUFDdkIsTUFBTSxrQkFBa0IsQ0FBQztBQUUxQixPQUFPLEVBQUUsV0FBVyxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFFOUQsT0FBTyxFQUFlLEtBQUssY0FBYyxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBR3ZGLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDMUQsT0FBTyxFQUlMLEtBQUssa0JBQWtCLEVBR3ZCLEtBQUssNkJBQTZCLEVBQ2xDLEtBQUssbUJBQW1CLEVBRXpCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxLQUFLLEVBQUUsdUJBQXVCLEVBQUUsa0JBQWtCLEVBQUUsdUJBQXVCLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUNqSCxPQUFPLEVBQXNCLGNBQWMsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQzFFLE9BQU8sS0FBSyxFQUNWLDZCQUE2QixFQUM3Qix1QkFBdUIsRUFDdkIsd0NBQXdDLEVBQ3hDLGlDQUFpQyxFQUNqQyw0QkFBNEIsRUFDNUIseUNBQXlDLEVBQ3pDLG9DQUFvQyxFQUNwQyxrQ0FBa0MsRUFDbEMsb0NBQW9DLEVBQ3BDLDRCQUE0QixFQUM1QixpQ0FBaUMsRUFDakMsNENBQTRDLEVBQzVDLGdDQUFnQyxFQUNoQyxnQ0FBZ0MsRUFDaEMsK0JBQStCLEVBQy9CLCtCQUErQixFQUMvQix1QkFBdUIsRUFDdkIsc0JBQXNCLEVBQ3RCLDBCQUEwQixFQUMxQixvQkFBb0IsRUFDckIsTUFBTSxzQkFBc0IsQ0FBQztBQUk5QixPQUFPLEVBQW9CLEtBQUssVUFBVSxFQUFFLE1BQU0sd0JBQXdCLENBQUM7QUFpQjNFLHFCQUFhLHlCQUEwQixZQUFXLG1CQUFtQjtJQVFqRSxPQUFPLENBQUMsTUFBTTtJQUNkLE9BQU8sQ0FBQyxVQUFVO0lBQ2xCLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQztJQUN6QixPQUFPLENBQUMsY0FBYztJQVZ4QixPQUFPLENBQUMsSUFBSSxDQUE0QztJQUN4RCxPQUFPLENBQUMsY0FBYyxDQUFDLENBQWlCO0lBQ3hDLE9BQU8sQ0FBQyxzQkFBc0IsQ0FBYztJQUM1QyxPQUFPLENBQUMsY0FBYyxDQUFnQztJQUN0RCxPQUFPLENBQUMsR0FBRyxDQUFTO0lBRXBCLFlBQ1UsTUFBTSxFQUFFLGtCQUFrQixFQUMxQixVQUFVLEdBQUUsVUFBbUMsRUFDL0MsZ0JBQWdCLENBQUMsd0JBQVksRUFDN0IsY0FBYyxTQUFPLEVBQzdCLFFBQVEsQ0FBQyxFQUFFLGNBQWMsRUFHMUI7SUFFRDs7Ozs7T0FLRztJQUNILE9BQU8sQ0FBQyxxQkFBcUI7WUEwQ2YsVUFBVTtJQTBEakIsS0FBSyxTQVNYO0lBRVksSUFBSSxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FZakM7WUFFYSxtQkFBbUI7WUF3RG5CLDZCQUE2QjtZQXlGN0IsdUJBQXVCO1lBb0J2Qix1QkFBdUI7SUFhckMsV0FBVyxDQUNULE1BQU0sRUFBRSxnQkFBZ0IsRUFDeEIsTUFBTSxDQUFDLEVBQUUsV0FBVyxFQUNwQixXQUFXLENBQUMsRUFBRSxXQUFXLEdBQ3hCLE9BQU8sQ0FBQyxjQUFjLENBQUMsT0FBTyw2QkFBNkIsQ0FBQyxDQUFDLENBUS9EO0lBRUQsa0JBQWtCLENBQ2hCLE1BQU0sRUFBRSx1QkFBdUIsRUFDL0IsTUFBTSxDQUFDLEVBQUUsV0FBVyxFQUNwQixXQUFXLENBQUMsRUFBRSxXQUFXLEdBQ3hCLE9BQU8sQ0FBQyw2QkFBNkIsQ0FBQyxrQkFBa0IsRUFBRSxPQUFPLHNCQUFzQixDQUFDLENBQUMsQ0FRM0Y7SUFFRCxxQkFBcUIsQ0FDbkIsS0FBSyxFQUFFLDBCQUEwQixFQUNqQyxNQUFNLENBQUMsRUFBRSxXQUFXLEVBQ3BCLFdBQVcsQ0FBQyxFQUFFLFdBQVcsR0FDeEIsT0FBTyxDQUFDLDZCQUE2QixDQUFDLG9CQUFvQixFQUFFLE9BQU8seUNBQXlDLENBQUMsQ0FBQyxDQVFoSDtJQUVELDJCQUEyQixDQUN6QixNQUFNLEVBQUUsZ0NBQWdDLEVBQ3hDLE1BQU0sQ0FBQyxFQUFFLFdBQVcsRUFDcEIsV0FBVyxDQUFDLEVBQUUsV0FBVyxHQUN4QixPQUFPLENBQ1IsNkJBQTZCLENBQUMsK0JBQStCLEVBQUUsT0FBTyx5Q0FBeUMsQ0FBQyxDQUNqSCxDQVFBO0lBRUQsMkJBQTJCLENBQ3pCLGVBQWUsRUFBRSxnQ0FBZ0MsRUFDakQsTUFBTSxDQUFDLEVBQUUsV0FBVyxFQUNwQixXQUFXLENBQUMsRUFBRSxXQUFXLEdBQ3hCLE9BQU8sQ0FBQyw2QkFBNkIsQ0FBQyxvQkFBb0IsRUFBRSxPQUFPLHlDQUF5QyxDQUFDLENBQUMsQ0FRaEg7SUFFRCwwQkFBMEIsQ0FDeEIsTUFBTSxFQUFFLCtCQUErQixFQUN2QyxNQUFNLENBQUMsRUFBRSxXQUFXLEVBQ3BCLFdBQVcsQ0FBQyxFQUFFLFdBQVcsR0FDeEIsT0FBTyxDQUFDLDZCQUE2QixDQUFDLG9CQUFvQixFQUFFLE9BQU8seUNBQXlDLENBQUMsQ0FBQyxDQVFoSDtJQUVELGtCQUFrQixDQUNoQixNQUFNLEVBQUUsdUJBQXVCLEVBQy9CLE1BQU0sQ0FBQyxFQUFFLFdBQVcsRUFDcEIsV0FBVyxDQUFDLEVBQUUsV0FBVyxHQUN4QixPQUFPLENBQUMsNkJBQTZCLENBQUMsa0JBQWtCLEVBQUUsT0FBTyw2QkFBNkIsQ0FBQyxDQUFDLENBUWxHO0lBRUQsNEJBQTRCLENBQzFCLEtBQUssRUFBRSxpQ0FBaUMsRUFDeEMsTUFBTSxDQUFDLEVBQUUsV0FBVyxFQUNwQixXQUFXLENBQUMsRUFBRSxXQUFXLEdBQ3hCLE9BQU8sQ0FBQyw2QkFBNkIsQ0FBQyx1QkFBdUIsRUFBRSxPQUFPLHlDQUF5QyxDQUFDLENBQUMsQ0FRbkg7SUFFRCxvQ0FBb0MsQ0FDbEMsS0FBSyxFQUFFLHlDQUF5QyxFQUNoRCxNQUFNLENBQUMsRUFBRSxXQUFXLEVBQ3BCLFdBQVcsQ0FBQyxFQUFFLFdBQVcsR0FDeEIsT0FBTyxDQUFDLDZCQUE2QixDQUFDLHVCQUF1QixFQUFFLE9BQU8seUNBQXlDLENBQUMsQ0FBQyxDQVFuSDtJQUVELG1DQUFtQyxDQUNqQyxLQUFLLEVBQUUsd0NBQXdDLEVBQy9DLE1BQU0sQ0FBQyxFQUFFLFdBQVcsRUFDcEIsV0FBVyxDQUFDLEVBQUUsV0FBVyxHQUN4QixPQUFPLENBQUMsNkJBQTZCLENBQUMsdUJBQXVCLEVBQUUsT0FBTyx5Q0FBeUMsQ0FBQyxDQUFDLENBUW5IO0lBRUQsdUJBQXVCLENBQ3JCLEtBQUssRUFBRSw0QkFBNEIsRUFDbkMsTUFBTSxDQUFDLEVBQUUsV0FBVyxFQUNwQixXQUFXLENBQUMsRUFBRSxXQUFXLEdBQ3hCLE9BQU8sQ0FBQyw2QkFBNkIsQ0FBQyx1QkFBdUIsRUFBRSxPQUFPLHlDQUF5QyxDQUFDLENBQUMsQ0FRbkg7SUFFRCwrQkFBK0IsQ0FDN0IsS0FBSyxFQUFFLG9DQUFvQyxFQUMzQyxNQUFNLENBQUMsRUFBRSxXQUFXLEVBQ3BCLFdBQVcsQ0FBQyxFQUFFLFdBQVcsR0FDeEIsT0FBTyxDQUFDLDZCQUE2QixDQUFDLHVCQUF1QixFQUFFLE9BQU8seUNBQXlDLENBQUMsQ0FBQyxDQVFuSDtJQUVELHdCQUF3QixDQUN0QixLQUFLLEVBQUUsNkJBQTZCLEVBQ3BDLE1BQU0sQ0FBQyxFQUFFLFdBQVcsRUFDcEIsV0FBVyxDQUFDLEVBQUUsV0FBVyxHQUN4QixPQUFPLENBQUMsNkJBQTZCLENBQUMsdUJBQXVCLEVBQUUsT0FBTyx5Q0FBeUMsQ0FBQyxDQUFDLENBUW5IO0lBRUQsNEJBQTRCLENBQzFCLEtBQUssRUFBRSxpQ0FBaUMsRUFDeEMsTUFBTSxDQUFDLEVBQUUsV0FBVyxFQUNwQixXQUFXLENBQUMsRUFBRSxXQUFXLEdBQ3hCLE9BQU8sQ0FDUiw2QkFBNkIsQ0FBQyw0QkFBNEIsRUFBRSxPQUFPLHlDQUF5QyxDQUFDLENBQzlHLENBUUE7SUFFRCx1Q0FBdUMsQ0FDckMsS0FBSyxFQUFFLDRDQUE0QyxFQUNuRCxNQUFNLENBQUMsRUFBRSxXQUFXLEVBQ3BCLFdBQVcsQ0FBQyxFQUFFLFdBQVcsR0FDeEIsT0FBTyxDQUNSLDZCQUE2QixDQUFDLDRCQUE0QixFQUFFLE9BQU8seUNBQXlDLENBQUMsQ0FDOUcsQ0FRQTtJQUVELCtCQUErQixDQUM3QixLQUFLLEVBQUUsb0NBQW9DLEVBQzNDLE1BQU0sQ0FBQyxFQUFFLFdBQVcsRUFDcEIsV0FBVyxDQUFDLEVBQUUsV0FBVyxHQUN4QixPQUFPLENBQ1IsNkJBQTZCLENBQUMsNEJBQTRCLEVBQUUsT0FBTyx5Q0FBeUMsQ0FBQyxDQUM5RyxDQVFBO0lBRUQsNkJBQTZCLENBQzNCLEtBQUssRUFBRSxrQ0FBa0MsRUFDekMsTUFBTSxDQUFDLEVBQUUsV0FBVyxFQUNwQixXQUFXLENBQUMsRUFBRSxXQUFXLEdBQ3hCLE9BQU8sQ0FDUiw2QkFBNkIsQ0FBQyw0QkFBNEIsRUFBRSxPQUFPLHlDQUF5QyxDQUFDLENBQzlHLENBUUE7SUFFRCxrQkFBa0IsQ0FDaEIsS0FBSyxFQUFFLHVCQUF1QixFQUM5QixNQUFNLENBQUMsRUFBRSxXQUFXLEVBQ3BCLFdBQVcsQ0FBQyxFQUFFLFdBQVcsR0FDeEIsT0FBTyxDQUFDLDZCQUE2QixDQUFDLHNCQUFzQixFQUFFLE9BQU8sc0JBQXNCLENBQUMsQ0FBQyxDQVEvRjtJQUVELE9BQU8sQ0FBQyxVQUFVO0NBUW5CIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"broker_prover_facade.d.ts","sourceRoot":"","sources":["../../src/proving_broker/broker_prover_facade.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,
|
|
1
|
+
{"version":3,"file":"broker_prover_facade.d.ts","sourceRoot":"","sources":["../../src/proving_broker/broker_prover_facade.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,6BAA6B,EAC7B,6BAA6B,EAC7B,yCAAyC,EACzC,sBAAsB,EACvB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAe,KAAK,cAAc,EAAgB,MAAM,uBAAuB,CAAC;AAGvF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAIL,KAAK,kBAAkB,EAGvB,KAAK,6BAA6B,EAClC,KAAK,mBAAmB,EAEzB,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AACjH,OAAO,EAAsB,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,KAAK,EACV,6BAA6B,EAC7B,uBAAuB,EACvB,wCAAwC,EACxC,iCAAiC,EACjC,4BAA4B,EAC5B,yCAAyC,EACzC,oCAAoC,EACpC,kCAAkC,EAClC,oCAAoC,EACpC,4BAA4B,EAC5B,iCAAiC,EACjC,4CAA4C,EAC5C,gCAAgC,EAChC,gCAAgC,EAChC,+BAA+B,EAC/B,+BAA+B,EAC/B,uBAAuB,EACvB,sBAAsB,EACtB,0BAA0B,EAC1B,oBAAoB,EACrB,MAAM,sBAAsB,CAAC;AAI9B,OAAO,EAAoB,KAAK,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAiB3E,qBAAa,yBAA0B,YAAW,mBAAmB;IAQjE,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,gBAAgB,CAAC;IACzB,OAAO,CAAC,cAAc;IAVxB,OAAO,CAAC,IAAI,CAA4C;IACxD,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,sBAAsB,CAAc;IAC5C,OAAO,CAAC,cAAc,CAAgC;IACtD,OAAO,CAAC,GAAG,CAAS;IAEpB,YACU,MAAM,EAAE,kBAAkB,EAC1B,UAAU,GAAE,UAAmC,EAC/C,gBAAgB,CAAC,wBAAY,EAC7B,cAAc,SAAO,EAC7B,QAAQ,CAAC,EAAE,cAAc,EAG1B;IAED;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;YA0Cf,UAAU;IA0DjB,KAAK,SASX;IAEY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAYjC;YAEa,mBAAmB;YAwDnB,6BAA6B;YAyF7B,uBAAuB;YAoBvB,uBAAuB;IAarC,WAAW,CACT,MAAM,EAAE,gBAAgB,EACxB,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,cAAc,CAAC,OAAO,6BAA6B,CAAC,CAAC,CAQ/D;IAED,kBAAkB,CAChB,MAAM,EAAE,uBAAuB,EAC/B,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,kBAAkB,EAAE,OAAO,sBAAsB,CAAC,CAAC,CAQ3F;IAED,qBAAqB,CACnB,KAAK,EAAE,0BAA0B,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,oBAAoB,EAAE,OAAO,yCAAyC,CAAC,CAAC,CAQhH;IAED,2BAA2B,CACzB,MAAM,EAAE,gCAAgC,EACxC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CACR,6BAA6B,CAAC,+BAA+B,EAAE,OAAO,yCAAyC,CAAC,CACjH,CAQA;IAED,2BAA2B,CACzB,eAAe,EAAE,gCAAgC,EACjD,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,oBAAoB,EAAE,OAAO,yCAAyC,CAAC,CAAC,CAQhH;IAED,0BAA0B,CACxB,MAAM,EAAE,+BAA+B,EACvC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,oBAAoB,EAAE,OAAO,yCAAyC,CAAC,CAAC,CAQhH;IAED,kBAAkB,CAChB,MAAM,EAAE,uBAAuB,EAC/B,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,kBAAkB,EAAE,OAAO,6BAA6B,CAAC,CAAC,CAQlG;IAED,4BAA4B,CAC1B,KAAK,EAAE,iCAAiC,EACxC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,uBAAuB,EAAE,OAAO,yCAAyC,CAAC,CAAC,CAQnH;IAED,oCAAoC,CAClC,KAAK,EAAE,yCAAyC,EAChD,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,uBAAuB,EAAE,OAAO,yCAAyC,CAAC,CAAC,CAQnH;IAED,mCAAmC,CACjC,KAAK,EAAE,wCAAwC,EAC/C,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,uBAAuB,EAAE,OAAO,yCAAyC,CAAC,CAAC,CAQnH;IAED,uBAAuB,CACrB,KAAK,EAAE,4BAA4B,EACnC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,uBAAuB,EAAE,OAAO,yCAAyC,CAAC,CAAC,CAQnH;IAED,+BAA+B,CAC7B,KAAK,EAAE,oCAAoC,EAC3C,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,uBAAuB,EAAE,OAAO,yCAAyC,CAAC,CAAC,CAQnH;IAED,wBAAwB,CACtB,KAAK,EAAE,6BAA6B,EACpC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,uBAAuB,EAAE,OAAO,yCAAyC,CAAC,CAAC,CAQnH;IAED,4BAA4B,CAC1B,KAAK,EAAE,iCAAiC,EACxC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CACR,6BAA6B,CAAC,4BAA4B,EAAE,OAAO,yCAAyC,CAAC,CAC9G,CAQA;IAED,uCAAuC,CACrC,KAAK,EAAE,4CAA4C,EACnD,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CACR,6BAA6B,CAAC,4BAA4B,EAAE,OAAO,yCAAyC,CAAC,CAC9G,CAQA;IAED,+BAA+B,CAC7B,KAAK,EAAE,oCAAoC,EAC3C,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CACR,6BAA6B,CAAC,4BAA4B,EAAE,OAAO,yCAAyC,CAAC,CAC9G,CAQA;IAED,6BAA6B,CAC3B,KAAK,EAAE,kCAAkC,EACzC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CACR,6BAA6B,CAAC,4BAA4B,EAAE,OAAO,yCAAyC,CAAC,CAC9G,CAQA;IAED,kBAAkB,CAChB,KAAK,EAAE,uBAAuB,EAC9B,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,6BAA6B,CAAC,sBAAsB,EAAE,OAAO,sBAAsB,CAAC,CAAC,CAQ/F;IAED,OAAO,CAAC,UAAU;CAQnB"}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { asyncPool } from '@aztec/foundation/async-pool';
|
|
1
2
|
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
3
|
+
import { chunk } from '@aztec/foundation/collection';
|
|
2
4
|
import { createLogger } from '@aztec/foundation/log';
|
|
3
5
|
import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
4
6
|
import { truncate } from '@aztec/foundation/string';
|
|
@@ -156,17 +158,13 @@ export class BrokerCircuitProverFacade {
|
|
|
156
158
|
// We collect all returned notifications and return them
|
|
157
159
|
const allCompleted = new Set();
|
|
158
160
|
try {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
161
|
+
const batches = ids.length > 0 ? chunk(ids, SNAPSHOT_SYNC_CHECK_MAX_REQUEST_SIZE) : [
|
|
162
|
+
[]
|
|
163
|
+
];
|
|
164
|
+
await asyncPool(1, batches, async (batch)=>{
|
|
165
|
+
const completed = await this.broker.getCompletedJobs(batch);
|
|
163
166
|
completed.forEach((id)=>allCompleted.add(id));
|
|
164
|
-
|
|
165
|
-
}
|
|
166
|
-
if (numRequests === 0) {
|
|
167
|
-
const final = await this.broker.getCompletedJobs([]);
|
|
168
|
-
final.forEach((id)=>allCompleted.add(id));
|
|
169
|
-
}
|
|
167
|
+
});
|
|
170
168
|
} catch (err) {
|
|
171
169
|
this.log.error(`Error thrown when requesting completed job notifications from the broker`, err);
|
|
172
170
|
}
|
|
@@ -220,6 +218,11 @@ export class BrokerCircuitProverFacade {
|
|
|
220
218
|
success: false,
|
|
221
219
|
reason: result.reason
|
|
222
220
|
};
|
|
221
|
+
} else if (result.status === 'aborted') {
|
|
222
|
+
return {
|
|
223
|
+
success: false,
|
|
224
|
+
reason: 'Aborted'
|
|
225
|
+
};
|
|
223
226
|
} else {
|
|
224
227
|
throw new Error(`Unexpected proving job status ${result.status}`);
|
|
225
228
|
}
|
|
@@ -266,12 +269,8 @@ export class BrokerCircuitProverFacade {
|
|
|
266
269
|
};
|
|
267
270
|
const toBeRetrieved = Array.from(this.jobsToRetrieve.values()).map((id)=>this.jobs.get(id)).filter((x)=>x !== undefined);
|
|
268
271
|
const totalJobsToRetrieve = toBeRetrieved.length;
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
const slice = toBeRetrieved.splice(0, MAX_CONCURRENT_JOB_SETTLED_REQUESTS);
|
|
272
|
-
const results = await Promise.all(slice.map((job)=>processJob(job)));
|
|
273
|
-
totalJobsRetrieved += results.filter((x)=>x).length;
|
|
274
|
-
}
|
|
272
|
+
const results = await asyncPool(MAX_CONCURRENT_JOB_SETTLED_REQUESTS, toBeRetrieved, (job)=>processJob(job));
|
|
273
|
+
const totalJobsRetrieved = results.filter((x)=>x).length;
|
|
275
274
|
if (totalJobsToRetrieve > 0) {
|
|
276
275
|
this.log.verbose(`Successfully retrieved ${totalJobsRetrieved} of ${totalJobsToRetrieve} jobs that should be ready, total ready jobs is now: ${this.jobsToRetrieve.size}`);
|
|
277
276
|
}
|
|
@@ -1,104 +1,40 @@
|
|
|
1
1
|
import { type L1ReaderConfig } from '@aztec/ethereum/l1-reader';
|
|
2
2
|
import { type ConfigMappingsType } from '@aztec/foundation/config';
|
|
3
|
-
import { type DataStoreConfig } from '@aztec/kv-store/config';
|
|
4
3
|
import { type ChainConfig } from '@aztec/stdlib/config';
|
|
4
|
+
import { type DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
5
5
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
6
6
|
import { z } from 'zod';
|
|
7
7
|
export declare const ProverBrokerConfig: z.ZodObject<{
|
|
8
|
-
/** If starting a prover broker locally, the max number of retries per proving job */
|
|
9
8
|
proverBrokerJobMaxRetries: z.ZodNumber;
|
|
10
|
-
/** If starting a prover broker locally, the time after which a job times out and gets assigned to a different agent */
|
|
11
9
|
proverBrokerJobTimeoutMs: z.ZodNumber;
|
|
12
|
-
/** If starting a prover broker locally, the interval the broker checks for timed out jobs */
|
|
13
10
|
proverBrokerPollIntervalMs: z.ZodNumber;
|
|
14
|
-
/** If starting a prover broker locally, the directory to store broker data */
|
|
15
11
|
dataDirectory: z.ZodOptional<z.ZodString>;
|
|
16
|
-
/** The size of the data store map */
|
|
17
12
|
dataStoreMapSizeKb: z.ZodNumber;
|
|
18
|
-
/** The size of the prover broker's database. Will override the dataStoreMapSizeKb if set. */
|
|
19
13
|
proverBrokerStoreMapSizeKb: z.ZodOptional<z.ZodNumber>;
|
|
20
|
-
/** The prover broker may batch jobs together before writing to the database */
|
|
21
14
|
proverBrokerBatchSize: z.ZodNumber;
|
|
22
|
-
/** How often the job batches get flushed */
|
|
23
15
|
proverBrokerBatchIntervalMs: z.ZodNumber;
|
|
24
|
-
/** The maximum number of epochs to keep results for */
|
|
25
16
|
proverBrokerMaxEpochsToKeepResultsFor: z.ZodNumber;
|
|
26
|
-
/** Enable debug replay mode for replaying proving jobs from stored inputs */
|
|
27
17
|
proverBrokerDebugReplayEnabled: z.ZodBoolean;
|
|
28
|
-
},
|
|
29
|
-
proverBrokerJobMaxRetries: number;
|
|
30
|
-
proverBrokerJobTimeoutMs: number;
|
|
31
|
-
proverBrokerPollIntervalMs: number;
|
|
32
|
-
dataDirectory?: string | undefined;
|
|
33
|
-
dataStoreMapSizeKb: number;
|
|
34
|
-
proverBrokerStoreMapSizeKb?: number | undefined;
|
|
35
|
-
proverBrokerBatchSize: number;
|
|
36
|
-
proverBrokerBatchIntervalMs: number;
|
|
37
|
-
proverBrokerMaxEpochsToKeepResultsFor: number;
|
|
38
|
-
proverBrokerDebugReplayEnabled: boolean;
|
|
39
|
-
}, {
|
|
40
|
-
proverBrokerJobMaxRetries: number;
|
|
41
|
-
proverBrokerJobTimeoutMs: number;
|
|
42
|
-
proverBrokerPollIntervalMs: number;
|
|
43
|
-
dataDirectory?: string | undefined;
|
|
44
|
-
dataStoreMapSizeKb: number;
|
|
45
|
-
proverBrokerStoreMapSizeKb?: number | undefined;
|
|
46
|
-
proverBrokerBatchSize: number;
|
|
47
|
-
proverBrokerBatchIntervalMs: number;
|
|
48
|
-
proverBrokerMaxEpochsToKeepResultsFor: number;
|
|
49
|
-
proverBrokerDebugReplayEnabled: boolean;
|
|
50
|
-
}>;
|
|
18
|
+
}, z.core.$strip>;
|
|
51
19
|
export type ProverBrokerConfig = z.infer<typeof ProverBrokerConfig> & Pick<DataStoreConfig, 'dataStoreMapSizeKb' | 'dataDirectory'> & L1ReaderConfig & Pick<ChainConfig, 'rollupVersion'>;
|
|
52
20
|
export declare const proverBrokerConfigMappings: ConfigMappingsType<ProverBrokerConfig>;
|
|
53
21
|
export declare const defaultProverBrokerConfig: ProverBrokerConfig;
|
|
54
22
|
export declare const ProverAgentConfig: z.ZodObject<{
|
|
55
|
-
/** The number of prover agents to start */
|
|
56
23
|
proverAgentCount: z.ZodNumber;
|
|
57
|
-
|
|
58
|
-
proverAgentProofTypes: z.ZodArray<z.ZodNativeEnum<typeof ProvingRequestType>, "many">;
|
|
59
|
-
/** How often the prover agents poll for jobs */
|
|
24
|
+
proverAgentProofTypes: z.ZodArray<z.ZodEnum<typeof ProvingRequestType>>;
|
|
60
25
|
proverAgentPollIntervalMs: z.ZodNumber;
|
|
61
|
-
/** The URL where this agent takes jobs from */
|
|
62
26
|
proverBrokerUrl: z.ZodOptional<z.ZodString>;
|
|
63
|
-
/** Whether to construct real proofs */
|
|
64
27
|
realProofs: z.ZodBoolean;
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
28
|
+
proverTestDelayType: z.ZodEnum<{
|
|
29
|
+
fixed: "fixed";
|
|
30
|
+
realistic: "realistic";
|
|
31
|
+
}>;
|
|
68
32
|
proverTestDelayMs: z.ZodNumber;
|
|
69
|
-
/** If using realistic delays, what percentage of realistic times to apply. */
|
|
70
33
|
proverTestDelayFactor: z.ZodNumber;
|
|
71
|
-
/** The delay (ms) to inject during fake proof verification */
|
|
72
34
|
proverTestVerificationDelayMs: z.ZodOptional<z.ZodNumber>;
|
|
73
|
-
/** Whether to abort pending proving jobs when the orchestrator is cancelled */
|
|
74
35
|
cancelJobsOnStop: z.ZodBoolean;
|
|
75
|
-
/** Where to store proving results. Must be accessible to both prover node and agents. If not set will inline-encode the parameters */
|
|
76
36
|
proofStore: z.ZodOptional<z.ZodString>;
|
|
77
|
-
},
|
|
78
|
-
proverAgentCount: number;
|
|
79
|
-
proverAgentProofTypes: ProvingRequestType[];
|
|
80
|
-
proverAgentPollIntervalMs: number;
|
|
81
|
-
proverBrokerUrl?: string | undefined;
|
|
82
|
-
realProofs: boolean;
|
|
83
|
-
proverTestDelayType: "fixed" | "realistic";
|
|
84
|
-
proverTestDelayMs: number;
|
|
85
|
-
proverTestDelayFactor: number;
|
|
86
|
-
proverTestVerificationDelayMs?: number | undefined;
|
|
87
|
-
cancelJobsOnStop: boolean;
|
|
88
|
-
proofStore?: string | undefined;
|
|
89
|
-
}, {
|
|
90
|
-
proverAgentCount: number;
|
|
91
|
-
proverAgentProofTypes: ProvingRequestType[];
|
|
92
|
-
proverAgentPollIntervalMs: number;
|
|
93
|
-
proverBrokerUrl?: string | undefined;
|
|
94
|
-
realProofs: boolean;
|
|
95
|
-
proverTestDelayType: "fixed" | "realistic";
|
|
96
|
-
proverTestDelayMs: number;
|
|
97
|
-
proverTestDelayFactor: number;
|
|
98
|
-
proverTestVerificationDelayMs?: number | undefined;
|
|
99
|
-
cancelJobsOnStop: boolean;
|
|
100
|
-
proofStore?: string | undefined;
|
|
101
|
-
}>;
|
|
37
|
+
}, z.core.$strip>;
|
|
102
38
|
export type ProverAgentConfig = z.infer<typeof ProverAgentConfig>;
|
|
103
39
|
export declare const proverAgentConfigMappings: ConfigMappingsType<ProverAgentConfig>;
|
|
104
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
40
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvY29uZmlnLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxLQUFLLGNBQWMsRUFBMEIsTUFBTSwyQkFBMkIsQ0FBQztBQUN4RixPQUFPLEVBQ0wsS0FBSyxrQkFBa0IsRUFLeEIsTUFBTSwwQkFBMEIsQ0FBQztBQUVsQyxPQUFPLEVBQUUsS0FBSyxXQUFXLEVBQXVCLE1BQU0sc0JBQXNCLENBQUM7QUFDN0UsT0FBTyxFQUFFLEtBQUssZUFBZSxFQUFzQixNQUFNLHdCQUF3QixDQUFDO0FBQ2xGLE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBRTFELE9BQU8sRUFBRSxDQUFDLEVBQUUsTUFBTSxLQUFLLENBQUM7QUFFeEIsZUFBTyxNQUFNLGtCQUFrQjs7Ozs7Ozs7Ozs7aUJBcUI3QixDQUFDO0FBRUgsTUFBTSxNQUFNLGtCQUFrQixHQUFHLENBQUMsQ0FBQyxLQUFLLENBQUMsT0FBTyxrQkFBa0IsQ0FBQyxHQUNqRSxJQUFJLENBQUMsZUFBZSxFQUFFLG9CQUFvQixHQUFHLGVBQWUsQ0FBQyxHQUM3RCxjQUFjLEdBQ2QsSUFBSSxDQUFDLFdBQVcsRUFBRSxlQUFlLENBQUMsQ0FBQztBQUVyQyxlQUFPLE1BQU0sMEJBQTBCLEVBQUUsa0JBQWtCLENBQUMsa0JBQWtCLENBNEM3RSxDQUFDO0FBRUYsZUFBTyxNQUFNLHlCQUF5QixFQUFFLGtCQUFpRSxDQUFDO0FBRTFHLGVBQU8sTUFBTSxpQkFBaUI7Ozs7Ozs7Ozs7Ozs7OztpQkF1QjVCLENBQUM7QUFFSCxNQUFNLE1BQU0saUJBQWlCLEdBQUcsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLGlCQUFpQixDQUFDLENBQUM7QUFFbEUsZUFBTyxNQUFNLHlCQUF5QixFQUFFLGtCQUFrQixDQUFDLGlCQUFpQixDQTREM0UsQ0FBQyJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/proving_broker/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,cAAc,EAA0B,MAAM,2BAA2B,CAAC;AACxF,OAAO,EACL,KAAK,kBAAkB,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/proving_broker/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,cAAc,EAA0B,MAAM,2BAA2B,CAAC;AACxF,OAAO,EACL,KAAK,kBAAkB,EAKxB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,sBAAsB,CAAC;AAC7E,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,kBAAkB;;;;;;;;;;;iBAqB7B,CAAC;AAEH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,GACjE,IAAI,CAAC,eAAe,EAAE,oBAAoB,GAAG,eAAe,CAAC,GAC7D,cAAc,GACd,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;AAErC,eAAO,MAAM,0BAA0B,EAAE,kBAAkB,CAAC,kBAAkB,CA4C7E,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,kBAAiE,CAAC;AAE1G,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;iBAuB5B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAElE,eAAO,MAAM,yBAAyB,EAAE,kBAAkB,CAAC,iBAAiB,CA4D3E,CAAC"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { l1ReaderConfigMappings } from '@aztec/ethereum/l1-reader';
|
|
2
|
-
import { booleanConfigHelper, getDefaultConfig, numberConfigHelper } from '@aztec/foundation/config';
|
|
2
|
+
import { booleanConfigHelper, getDefaultConfig, numberConfigHelper, optionalNumberConfigHelper } from '@aztec/foundation/config';
|
|
3
3
|
import { pickConfigMappings } from '@aztec/foundation/config';
|
|
4
|
-
import { dataConfigMappings } from '@aztec/kv-store/config';
|
|
5
4
|
import { chainConfigMappings } from '@aztec/stdlib/config';
|
|
5
|
+
import { dataConfigMappings } from '@aztec/stdlib/kv-store';
|
|
6
6
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
7
7
|
import { z } from 'zod';
|
|
8
8
|
export const ProverBrokerConfig = z.object({
|
|
@@ -50,7 +50,7 @@ export const proverBrokerConfigMappings = {
|
|
|
50
50
|
},
|
|
51
51
|
proverBrokerStoreMapSizeKb: {
|
|
52
52
|
env: 'PROVER_BROKER_STORE_MAP_SIZE_KB',
|
|
53
|
-
|
|
53
|
+
...optionalNumberConfigHelper(),
|
|
54
54
|
description: "The size of the prover broker's database. Will override the dataStoreMapSizeKb if set."
|
|
55
55
|
},
|
|
56
56
|
proverBrokerDebugReplayEnabled: {
|
|
@@ -7,4 +7,5 @@ export * from './proving_broker_database/persisted.js';
|
|
|
7
7
|
export * from './proof_store/index.js';
|
|
8
8
|
export * from './factory.js';
|
|
9
9
|
export * from './config.js';
|
|
10
|
-
|
|
10
|
+
export { BrokerCircuitProverFacade } from './broker_prover_facade.js';
|
|
11
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxjQUFjLG9CQUFvQixDQUFDO0FBQ25DLGNBQWMscUJBQXFCLENBQUM7QUFDcEMsY0FBYyxVQUFVLENBQUM7QUFDekIsY0FBYyw4QkFBOEIsQ0FBQztBQUM3QyxjQUFjLHFDQUFxQyxDQUFDO0FBQ3BELGNBQWMsd0NBQXdDLENBQUM7QUFDdkQsY0FBYyx3QkFBd0IsQ0FBQztBQUN2QyxjQUFjLGNBQWMsQ0FBQztBQUM3QixjQUFjLGFBQWEsQ0FBQztBQUM1QixPQUFPLEVBQUUseUJBQXlCLEVBQUUsTUFBTSwyQkFBMkIsQ0FBQyJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/proving_broker/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,UAAU,CAAC;AACzB,cAAc,8BAA8B,CAAC;AAC7C,cAAc,qCAAqC,CAAC;AACpD,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC;AACvC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/proving_broker/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,UAAU,CAAC;AACzB,cAAc,8BAA8B,CAAC;AAC7C,cAAc,qCAAqC,CAAC;AACpD,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC;AACvC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,OAAO,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC"}
|
|
@@ -59,7 +59,7 @@ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobCons
|
|
|
59
59
|
replayProvingJob(jobId: ProvingJobId, type: ProvingRequestType, epochNumber: EpochNumber, inputsUri: ProofUri): Promise<ProvingJobStatus>;
|
|
60
60
|
private cleanUpProvingJobState;
|
|
61
61
|
private cleanupPass;
|
|
62
|
-
private
|
|
62
|
+
private cleanupJobsOlderThanEpoch;
|
|
63
63
|
private reEnqueueExpiredJobs;
|
|
64
64
|
private enqueueJobInternal;
|
|
65
65
|
private isJobStale;
|
|
@@ -73,4 +73,4 @@ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobCons
|
|
|
73
73
|
* is to get picked up by agents
|
|
74
74
|
*/
|
|
75
75
|
export declare const PROOF_TYPES_IN_PRIORITY_ORDER: ProvingRequestType[];
|
|
76
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
76
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXIuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9wcm92aW5nX2Jyb2tlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsV0FBVyxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFLOUQsT0FBTyxFQUNMLEtBQUsscUJBQXFCLEVBQzFCLEtBQUssUUFBUSxFQUNiLEtBQUssVUFBVSxFQUNmLEtBQUsscUJBQXFCLEVBQzFCLEtBQUssa0JBQWtCLEVBQ3ZCLEtBQUssZ0JBQWdCLEVBQ3JCLEtBQUssWUFBWSxFQUNqQixLQUFLLGtCQUFrQixFQUV2QixLQUFLLGdCQUFnQixFQUV0QixNQUFNLGlDQUFpQyxDQUFDO0FBQ3pDLE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQzFELE9BQU8sRUFBRSxLQUFLLGVBQWUsRUFBRSxLQUFLLFNBQVMsRUFBRSxLQUFLLE1BQU0sRUFBc0IsTUFBTSx5QkFBeUIsQ0FBQztBQUloSCxPQUFPLEVBQUUsS0FBSyxrQkFBa0IsRUFBNkIsTUFBTSxhQUFhLENBQUM7QUFDakYsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSw4QkFBOEIsQ0FBQztBQVcxRTs7O0dBR0c7QUFDSCxxQkFBYSxhQUFjLFlBQVcsa0JBQWtCLEVBQUUsa0JBQWtCLEVBQUUscUJBQXFCLEVBQUUsU0FBUzs7SUFpRjFHLE9BQU8sQ0FBQyxRQUFRO0lBa0JoQixPQUFPLENBQUMsTUFBTTtJQWxHaEIsT0FBTyxDQUFDLE1BQU0sQ0E2Qlo7SUFJRixPQUFPLENBQUMsU0FBUyxDQUF1QztJQUV4RCxPQUFPLENBQUMsWUFBWSxDQUFvRDtJQUd4RSxPQUFPLENBQUMsVUFBVSxDQUFrQztJQU1wRCxPQUFPLENBQUMsVUFBVSxDQUErQztJQUdqRSxPQUFPLENBQUMsT0FBTyxDQUFtQztJQUdsRCxPQUFPLENBQUMsUUFBUSxDQUEwRTtJQUUxRixPQUFPLENBQUMsY0FBYyxDQUFpQjtJQUN2QyxPQUFPLENBQUMsWUFBWSxDQUFvQjtJQUN4QyxPQUFPLENBQUMsWUFBWSxDQUFTO0lBQzdCLE9BQU8sQ0FBQyxVQUFVLENBQVM7SUFFM0IsT0FBTyxDQUFDLGVBQWUsQ0FBK0I7SUFDdEQsU0FBZ0IsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUUvQixPQUFPLENBQUMseUJBQXlCLENBQXNCO0lBRXZEOzs7Ozs7Ozs7T0FTRztJQUNILE9BQU8sQ0FBQyxXQUFXLENBQUs7SUFDeEIsT0FBTyxDQUFDLHlCQUF5QixDQUFLO0lBRXRDLE9BQU8sQ0FBQyxPQUFPLENBQVM7SUFFeEIsT0FBTyxDQUFDLGtCQUFrQixDQUFVO0lBRXBDLFlBQ1UsUUFBUSxFQUFFLHFCQUFxQixFQUN2QyxFQUNFLHdCQUF3QixFQUN4QiwwQkFBMEIsRUFDMUIseUJBQXlCLEVBQ3pCLHFDQUFxQyxFQUNyQyw4QkFBOEIsRUFDL0IsR0FBRSxRQUFRLENBQ1QsSUFBSSxDQUNGLGtCQUFrQixFQUNoQiwwQkFBMEIsR0FDMUIsNEJBQTRCLEdBQzVCLDJCQUEyQixHQUMzQix1Q0FBdUMsR0FDdkMsZ0NBQWdDLENBQ25DLENBQzBCLEVBQzdCLE1BQU0sR0FBRSxlQUFzQyxFQUN0QyxNQUFNLHlDQUErQyxFQVM5RDtJQUVELE9BQU8sQ0FBQyxpQkFBaUIsQ0FFdkI7SUFFRixPQUFPLENBQUMsZUFBZSxDQVVyQjtJQUVXLEtBQUssSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBNkJsQztJQUVZLElBQUksSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBTWpDO0lBRU0saUJBQWlCLENBQUMsR0FBRyxFQUFFLFVBQVUsR0FBRyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsQ0FFbkU7SUFFTSxnQkFBZ0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFdkQ7SUFFTSxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxDQUV0RTtJQUVNLGdCQUFnQixDQUFDLEdBQUcsRUFBRSxZQUFZLEVBQUUsR0FBRyxPQUFPLENBQUMsWUFBWSxFQUFFLENBQUMsQ0FFcEU7SUFFTSxhQUFhLENBQUMsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQUcsT0FBTyxDQUFDLHFCQUFxQixHQUFHLFNBQVMsQ0FBQyxDQUUxRjtJQUVNLHVCQUF1QixDQUM1QixFQUFFLEVBQUUsWUFBWSxFQUNoQixLQUFLLEVBQUUsUUFBUSxFQUNmLE1BQU0sQ0FBQyxFQUFFLGdCQUFnQixHQUN4QixPQUFPLENBQUMscUJBQXFCLEdBQUcsU0FBUyxDQUFDLENBRTVDO0lBRU0scUJBQXFCLENBQzFCLEVBQUUsRUFBRSxZQUFZLEVBQ2hCLEdBQUcsRUFBRSxNQUFNLEVBQ1gsS0FBSyxVQUFRLEVBQ2IsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQ3hCLE9BQU8sQ0FBQyxxQkFBcUIsR0FBRyxTQUFTLENBQUMsQ0FFNUM7SUFFTSx3QkFBd0IsQ0FDN0IsRUFBRSxFQUFFLFlBQVksRUFDaEIsU0FBUyxFQUFFLE1BQU0sRUFDakIsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQ3hCLE9BQU8sQ0FBQztRQUFFLEdBQUcsRUFBRSxVQUFVLENBQUM7UUFBQyxJQUFJLEVBQUUsTUFBTSxDQUFBO0tBQUUsR0FBRyxTQUFTLENBQUMsQ0FFeEQ7SUFFWSxnQkFBZ0IsQ0FDM0IsS0FBSyxFQUFFLFlBQVksRUFDbkIsSUFBSSxFQUFFLGtCQUFrQixFQUN4QixXQUFXLEVBQUUsV0FBVyxFQUN4QixTQUFTLEVBQUUsUUFBUSxHQUNsQixPQUFPLENBQUMsZ0JBQWdCLENBQUMsQ0FnQjNCO0lBK0ZELE9BQU8sQ0FBQyxzQkFBc0I7WUFvUmhCLFdBQVc7SUFVekIsT0FBTyxDQUFDLHlCQUF5QjtJQWdCakMsT0FBTyxDQUFDLG9CQUFvQjtJQXFDNUIsT0FBTyxDQUFDLGtCQUFrQjtJQVkxQixPQUFPLENBQUMsVUFBVTtJQUlsQixPQUFPLENBQUMsaUJBQWlCO0NBRzFCO0FBaUREOzs7Ozs7R0FNRztBQUNILGVBQU8sTUFBTSw2QkFBNkIsRUFBRSxrQkFBa0IsRUFtQjdELENBQUMifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proving_broker.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAK9D,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,kBAAkB,EAEvB,KAAK,gBAAgB,EAEtB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,SAAS,EAAE,KAAK,MAAM,EAAsB,MAAM,yBAAyB,CAAC;AAIhH,OAAO,EAAE,KAAK,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AACjF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAW1E;;;GAGG;AACH,qBAAa,aAAc,YAAW,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,SAAS;;IAiF1G,OAAO,CAAC,QAAQ;IAkBhB,OAAO,CAAC,MAAM;IAlGhB,OAAO,CAAC,MAAM,CA6BZ;IAIF,OAAO,CAAC,SAAS,CAAuC;IAExD,OAAO,CAAC,YAAY,CAAoD;IAGxE,OAAO,CAAC,UAAU,CAAkC;IAMpD,OAAO,CAAC,UAAU,CAA+C;IAGjE,OAAO,CAAC,OAAO,CAAmC;IAGlD,OAAO,CAAC,QAAQ,CAA0E;IAE1F,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAS;IAE3B,OAAO,CAAC,eAAe,CAA+B;IACtD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,yBAAyB,CAAsB;IAEvD;;;;;;;;;OASG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,yBAAyB,CAAK;IAEtC,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,CAAC,kBAAkB,CAAU;IAEpC,YACU,QAAQ,EAAE,qBAAqB,EACvC,EACE,wBAAwB,EACxB,0BAA0B,EAC1B,yBAAyB,EACzB,qCAAqC,EACrC,8BAA8B,EAC/B,GAAE,QAAQ,CACT,IAAI,CACF,kBAAkB,EAChB,0BAA0B,GAC1B,4BAA4B,GAC5B,2BAA2B,GAC3B,uCAAuC,GACvC,gCAAgC,CACnC,CAC0B,EAC7B,MAAM,GAAE,eAAsC,EACtC,MAAM,yCAA+C,EAS9D;IAED,OAAO,CAAC,iBAAiB,CAEvB;IAEF,OAAO,CAAC,eAAe,CAUrB;IAEW,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA6BlC;IAEY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAMjC;IAEM,iBAAiB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEnE;IAEM,gBAAgB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvD;IAEM,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEtE;IAEM,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAEpE;IAEM,aAAa,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE1F;IAEM,uBAAuB,CAC5B,EAAE,EAAE,YAAY,EAChB,KAAK,EAAE,QAAQ,EACf,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,qBAAqB,CAC1B,EAAE,EAAE,YAAY,EAChB,GAAG,EAAE,MAAM,EACX,KAAK,UAAQ,EACb,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,wBAAwB,CAC7B,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC;QAAE,GAAG,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,CAExD;IAEY,gBAAgB,CAC3B,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,kBAAkB,EACxB,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,QAAQ,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAgB3B;
|
|
1
|
+
{"version":3,"file":"proving_broker.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAK9D,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,kBAAkB,EAEvB,KAAK,gBAAgB,EAEtB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,SAAS,EAAE,KAAK,MAAM,EAAsB,MAAM,yBAAyB,CAAC;AAIhH,OAAO,EAAE,KAAK,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AACjF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAW1E;;;GAGG;AACH,qBAAa,aAAc,YAAW,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,SAAS;;IAiF1G,OAAO,CAAC,QAAQ;IAkBhB,OAAO,CAAC,MAAM;IAlGhB,OAAO,CAAC,MAAM,CA6BZ;IAIF,OAAO,CAAC,SAAS,CAAuC;IAExD,OAAO,CAAC,YAAY,CAAoD;IAGxE,OAAO,CAAC,UAAU,CAAkC;IAMpD,OAAO,CAAC,UAAU,CAA+C;IAGjE,OAAO,CAAC,OAAO,CAAmC;IAGlD,OAAO,CAAC,QAAQ,CAA0E;IAE1F,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAS;IAE3B,OAAO,CAAC,eAAe,CAA+B;IACtD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,yBAAyB,CAAsB;IAEvD;;;;;;;;;OASG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,yBAAyB,CAAK;IAEtC,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,CAAC,kBAAkB,CAAU;IAEpC,YACU,QAAQ,EAAE,qBAAqB,EACvC,EACE,wBAAwB,EACxB,0BAA0B,EAC1B,yBAAyB,EACzB,qCAAqC,EACrC,8BAA8B,EAC/B,GAAE,QAAQ,CACT,IAAI,CACF,kBAAkB,EAChB,0BAA0B,GAC1B,4BAA4B,GAC5B,2BAA2B,GAC3B,uCAAuC,GACvC,gCAAgC,CACnC,CAC0B,EAC7B,MAAM,GAAE,eAAsC,EACtC,MAAM,yCAA+C,EAS9D;IAED,OAAO,CAAC,iBAAiB,CAEvB;IAEF,OAAO,CAAC,eAAe,CAUrB;IAEW,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA6BlC;IAEY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAMjC;IAEM,iBAAiB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEnE;IAEM,gBAAgB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvD;IAEM,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEtE;IAEM,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAEpE;IAEM,aAAa,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE1F;IAEM,uBAAuB,CAC5B,EAAE,EAAE,YAAY,EAChB,KAAK,EAAE,QAAQ,EACf,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,qBAAqB,CAC1B,EAAE,EAAE,YAAY,EAChB,GAAG,EAAE,MAAM,EACX,KAAK,UAAQ,EACb,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,wBAAwB,CAC7B,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC;QAAE,GAAG,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,CAExD;IAEY,gBAAgB,CAC3B,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,kBAAkB,EACxB,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,QAAQ,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAgB3B;IA+FD,OAAO,CAAC,sBAAsB;YAoRhB,WAAW;IAUzB,OAAO,CAAC,yBAAyB;IAgBjC,OAAO,CAAC,oBAAoB;IAqC5B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;CAG1B;AAiDD;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,EAAE,kBAAkB,EAmB7D,CAAC"}
|
|
@@ -193,15 +193,41 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
193
193
|
}
|
|
194
194
|
async #enqueueProvingJob(job) {
|
|
195
195
|
// We return the job status at the start of this call
|
|
196
|
-
|
|
196
|
+
let jobStatus = this.#getProvingJobStatus(job.id);
|
|
197
197
|
if (this.jobsCache.has(job.id)) {
|
|
198
198
|
const existing = this.jobsCache.get(job.id);
|
|
199
199
|
assert.deepStrictEqual(job, existing, 'Duplicate proving job ID');
|
|
200
|
-
this.
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
200
|
+
if (this.resultsCache.get(job.id)?.status === 'aborted') {
|
|
201
|
+
// The producer is re-requesting a job it previously cancelled: revive it rather than
|
|
202
|
+
// returning the cached abort, clearing the aborted state in memory and in the database so the
|
|
203
|
+
// revival survives a restart.
|
|
204
|
+
//
|
|
205
|
+
// Concurrency model: `jobsCache` is the enqueue lock. Every path that puts a job on the queue
|
|
206
|
+
// populates `jobsCache` *synchronously, before its first await* (see the "New proving job"
|
|
207
|
+
// block below), so a second concurrent enqueue of the same id observes the entry at the top
|
|
208
|
+
// of this method and takes a cached, no-op branch instead of enqueuing a duplicate. The revive
|
|
209
|
+
// must keep holding that lock: we tear down the settled state and re-set `jobsCache` in a
|
|
210
|
+
// single synchronous span (no await in between), and only then await the database. Because a
|
|
211
|
+
// concurrent re-request can only interleave at that await — by which point `jobsCache` is
|
|
212
|
+
// populated again and the aborted result is gone — it falls into the cached branch and no-ops,
|
|
213
|
+
// so the job is enqueued exactly once. (`cleanUpProvingJobState` also drops the promise, which
|
|
214
|
+
// was resolved with the abort, so `enqueueJobInternal` below mints a fresh one for the retry.)
|
|
215
|
+
this.logger.info(`Reviving aborted proving job id=${job.id} epochNumber=${job.epochNumber}`, {
|
|
216
|
+
provingJobId: job.id
|
|
217
|
+
});
|
|
218
|
+
this.cleanUpProvingJobState([
|
|
219
|
+
job.id
|
|
220
|
+
]);
|
|
221
|
+
this.jobsCache.set(job.id, job);
|
|
222
|
+
await this.database.deleteProvingJobResult(job.id);
|
|
223
|
+
jobStatus = this.#getProvingJobStatus(job.id);
|
|
224
|
+
} else {
|
|
225
|
+
this.logger.warn(`Cached proving job id=${job.id} epochNumber=${job.epochNumber}. Not enqueuing again`, {
|
|
226
|
+
provingJobId: job.id
|
|
227
|
+
});
|
|
228
|
+
this.instrumentation.incCachedJobs(job.type);
|
|
229
|
+
return jobStatus;
|
|
230
|
+
}
|
|
205
231
|
}
|
|
206
232
|
if (this.isJobStale(job)) {
|
|
207
233
|
this.logger.warn(`Tried enqueueing stale proving job id=${job.id} epochNumber=${job.epochNumber}`, {
|
|
@@ -228,29 +254,59 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
228
254
|
return jobStatus;
|
|
229
255
|
}
|
|
230
256
|
async #cancelProvingJob(id) {
|
|
231
|
-
|
|
257
|
+
const job = this.jobsCache.get(id);
|
|
258
|
+
if (!job) {
|
|
232
259
|
this.logger.warn(`Can't cancel a job that doesn't exist id=${id}`, {
|
|
233
260
|
provingJobId: id
|
|
234
261
|
});
|
|
235
262
|
return;
|
|
236
263
|
}
|
|
237
|
-
//
|
|
238
|
-
if (
|
|
239
|
-
|
|
264
|
+
// Leave jobs that have already settled (completed or failed) alone: those results are terminal.
|
|
265
|
+
if (this.resultsCache.has(id)) {
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
this.logger.info(`Cancelling job id=${id}`, {
|
|
269
|
+
provingJobId: id
|
|
270
|
+
});
|
|
271
|
+
this.inProgress.delete(id);
|
|
272
|
+
// Record the cancellation as its own settled state and persist it, so it survives a restart and
|
|
273
|
+
// notifies the current waiter. Unlike a completion or failure this is not terminal: re-enqueuing
|
|
274
|
+
// the same job id revives it (see #enqueueProvingJob), so the abort never permanently blocks the
|
|
275
|
+
// proof.
|
|
276
|
+
const result = {
|
|
277
|
+
status: 'aborted'
|
|
278
|
+
};
|
|
279
|
+
this.resultsCache.set(id, result);
|
|
280
|
+
this.promises.get(id)?.resolve(result);
|
|
281
|
+
this.completedJobNotifications.push(id);
|
|
282
|
+
this.instrumentation.incAbortedJobs(job.type);
|
|
283
|
+
try {
|
|
284
|
+
await this.database.setProvingJobAborted(id);
|
|
285
|
+
} catch (saveErr) {
|
|
286
|
+
this.logger.error(`Failed to save proving job aborted status id=${id}`, saveErr, {
|
|
240
287
|
provingJobId: id
|
|
241
288
|
});
|
|
242
|
-
|
|
289
|
+
throw saveErr;
|
|
243
290
|
}
|
|
244
291
|
}
|
|
245
292
|
cleanUpProvingJobState(ids) {
|
|
293
|
+
const idsToClean = new Set(ids);
|
|
246
294
|
for (const id of ids){
|
|
247
295
|
this.jobsCache.delete(id);
|
|
296
|
+
const deferred = this.promises.get(id);
|
|
297
|
+
if (deferred) {
|
|
298
|
+
deferred.resolve({
|
|
299
|
+
status: 'rejected',
|
|
300
|
+
reason: 'Proving job cleaned up'
|
|
301
|
+
});
|
|
302
|
+
}
|
|
248
303
|
this.promises.delete(id);
|
|
249
304
|
this.resultsCache.delete(id);
|
|
250
305
|
this.inProgress.delete(id);
|
|
251
306
|
this.retries.delete(id);
|
|
252
307
|
this.enqueuedAt.delete(id);
|
|
253
308
|
}
|
|
309
|
+
this.completedJobNotifications = this.completedJobNotifications.filter((id)=>!idsToClean.has(id));
|
|
254
310
|
}
|
|
255
311
|
#getProvingJobStatus(id) {
|
|
256
312
|
const result = this.resultsCache.get(id);
|
|
@@ -313,7 +369,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
313
369
|
}
|
|
314
370
|
return undefined;
|
|
315
371
|
}
|
|
316
|
-
async #reportProvingJobError(id, err, retry = false, filter
|
|
372
|
+
async #reportProvingJobError(id, err, retry = false, filter) {
|
|
317
373
|
const info = this.inProgress.get(id);
|
|
318
374
|
const item = this.jobsCache.get(id);
|
|
319
375
|
const retries = this.retries.get(id) ?? 0;
|
|
@@ -360,11 +416,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
360
416
|
this.resultsCache.set(id, result);
|
|
361
417
|
this.promises.get(id).resolve(result);
|
|
362
418
|
this.completedJobNotifications.push(id);
|
|
363
|
-
|
|
364
|
-
this.instrumentation.incAbortedJobs(item.type);
|
|
365
|
-
} else {
|
|
366
|
-
this.instrumentation.incRejectedJobs(item.type);
|
|
367
|
-
}
|
|
419
|
+
this.instrumentation.incRejectedJobs(item.type);
|
|
368
420
|
if (info) {
|
|
369
421
|
const duration = this.msTimeSource() - info.startedAt;
|
|
370
422
|
this.instrumentation.recordJobDuration(item.type, duration);
|
|
@@ -479,20 +531,20 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
479
531
|
return this.#getProvingJob(filter);
|
|
480
532
|
}
|
|
481
533
|
async cleanupPass() {
|
|
482
|
-
this.cleanupStaleJobs();
|
|
483
534
|
this.reEnqueueExpiredJobs();
|
|
484
535
|
const oldestEpochToKeep = this.oldestEpochToKeep();
|
|
485
536
|
if (oldestEpochToKeep > 0) {
|
|
537
|
+
this.cleanupJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
|
|
486
538
|
await this.database.deleteAllProvingJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
|
|
487
539
|
this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
|
|
488
540
|
}
|
|
489
541
|
}
|
|
490
|
-
|
|
542
|
+
cleanupJobsOlderThanEpoch(epochNumber) {
|
|
491
543
|
const jobIds = Array.from(this.jobsCache.keys());
|
|
492
544
|
const jobsToClean = [];
|
|
493
545
|
for (const id of jobIds){
|
|
494
546
|
const job = this.jobsCache.get(id);
|
|
495
|
-
if (
|
|
547
|
+
if (job.epochNumber < epochNumber) {
|
|
496
548
|
jobsToClean.push(id);
|
|
497
549
|
}
|
|
498
550
|
}
|
|
@@ -515,12 +567,28 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
515
567
|
const now = this.msTimeSource();
|
|
516
568
|
const msSinceLastUpdate = now - metadata.lastUpdatedAt;
|
|
517
569
|
if (msSinceLastUpdate >= this.jobTimeoutMs) {
|
|
518
|
-
this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, {
|
|
519
|
-
provingJobId: id
|
|
520
|
-
});
|
|
521
570
|
this.inProgress.delete(id);
|
|
522
|
-
this.enqueueJobInternal(item);
|
|
523
571
|
this.instrumentation.incTimedOutJobs(item.type);
|
|
572
|
+
const retries = this.retries.get(id) ?? 0;
|
|
573
|
+
if (retries + 1 < this.maxRetries && !this.isJobStale(item)) {
|
|
574
|
+
this.logger.warn(`Proving job id=${id} timed out. Re-enqueueing (retry ${retries + 1}/${this.maxRetries}).`, {
|
|
575
|
+
provingJobId: id
|
|
576
|
+
});
|
|
577
|
+
this.retries.set(id, retries + 1);
|
|
578
|
+
this.enqueueJobInternal(item);
|
|
579
|
+
} else {
|
|
580
|
+
this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
|
|
581
|
+
provingJobId: id
|
|
582
|
+
});
|
|
583
|
+
const result = {
|
|
584
|
+
status: 'rejected',
|
|
585
|
+
reason: 'Timed out'
|
|
586
|
+
};
|
|
587
|
+
this.resultsCache.set(id, result);
|
|
588
|
+
this.promises.get(id)?.resolve(result);
|
|
589
|
+
this.completedJobNotifications.push(id);
|
|
590
|
+
this.instrumentation.incRejectedJobs(item.type);
|
|
591
|
+
}
|
|
524
592
|
}
|
|
525
593
|
}
|
|
526
594
|
}
|