@aztec/prover-client 0.0.1-commit.7d4e6cd → 0.0.1-commit.8afd444

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/dest/light/lightweight_checkpoint_builder.d.ts +11 -7
  2. package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -1
  3. package/dest/light/lightweight_checkpoint_builder.js +28 -11
  4. package/dest/mocks/fixtures.d.ts +1 -1
  5. package/dest/mocks/fixtures.d.ts.map +1 -1
  6. package/dest/mocks/fixtures.js +2 -1
  7. package/dest/mocks/test_context.d.ts +3 -2
  8. package/dest/mocks/test_context.d.ts.map +1 -1
  9. package/dest/mocks/test_context.js +6 -1
  10. package/dest/orchestrator/block-building-helpers.d.ts +2 -2
  11. package/dest/orchestrator/block-building-helpers.js +1 -1
  12. package/dest/orchestrator/checkpoint-proving-state.d.ts +15 -2
  13. package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -1
  14. package/dest/orchestrator/checkpoint-proving-state.js +34 -1
  15. package/dest/orchestrator/epoch-proving-state.d.ts +6 -5
  16. package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -1
  17. package/dest/orchestrator/epoch-proving-state.js +35 -1
  18. package/dest/orchestrator/orchestrator.d.ts +18 -3
  19. package/dest/orchestrator/orchestrator.d.ts.map +1 -1
  20. package/dest/orchestrator/orchestrator.js +120 -78
  21. package/dest/prover-client/prover-client.d.ts +3 -3
  22. package/dest/prover-client/prover-client.d.ts.map +1 -1
  23. package/dest/prover-client/prover-client.js +15 -10
  24. package/dest/proving_broker/broker_prover_facade.d.ts +4 -3
  25. package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
  26. package/dest/proving_broker/broker_prover_facade.js +3 -3
  27. package/dest/proving_broker/config.d.ts +13 -1
  28. package/dest/proving_broker/config.d.ts.map +1 -1
  29. package/dest/proving_broker/config.js +19 -2
  30. package/dest/proving_broker/proof_store/factory.d.ts +2 -5
  31. package/dest/proving_broker/proof_store/factory.d.ts.map +1 -1
  32. package/dest/proving_broker/proof_store/factory.js +7 -30
  33. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts +18 -0
  34. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts.map +1 -0
  35. package/dest/proving_broker/proof_store/file_store_proof_store.js +60 -0
  36. package/dest/proving_broker/proof_store/index.d.ts +2 -2
  37. package/dest/proving_broker/proof_store/index.d.ts.map +1 -1
  38. package/dest/proving_broker/proof_store/index.js +1 -1
  39. package/dest/proving_broker/proving_agent.d.ts +4 -3
  40. package/dest/proving_broker/proving_agent.d.ts.map +1 -1
  41. package/dest/proving_broker/proving_agent.js +4 -4
  42. package/dest/proving_broker/proving_broker.d.ts +7 -4
  43. package/dest/proving_broker/proving_broker.d.ts.map +1 -1
  44. package/dest/proving_broker/proving_broker.js +29 -1
  45. package/dest/proving_broker/proving_broker_instrumentation.d.ts +1 -1
  46. package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
  47. package/dest/proving_broker/proving_broker_instrumentation.js +11 -7
  48. package/dest/proving_broker/proving_job_controller.d.ts +4 -3
  49. package/dest/proving_broker/proving_job_controller.d.ts.map +1 -1
  50. package/dest/proving_broker/proving_job_controller.js +6 -3
  51. package/dest/proving_broker/rpc.d.ts +4 -2
  52. package/dest/proving_broker/rpc.d.ts.map +1 -1
  53. package/dest/proving_broker/rpc.js +8 -0
  54. package/dest/test/mock_proof_store.d.ts +3 -3
  55. package/dest/test/mock_proof_store.d.ts.map +1 -1
  56. package/dest/test/mock_prover.d.ts +3 -3
  57. package/package.json +16 -17
  58. package/src/light/lightweight_checkpoint_builder.ts +52 -12
  59. package/src/mocks/fixtures.ts +2 -1
  60. package/src/mocks/test_context.ts +5 -0
  61. package/src/orchestrator/block-building-helpers.ts +1 -1
  62. package/src/orchestrator/checkpoint-proving-state.ts +47 -1
  63. package/src/orchestrator/epoch-proving-state.ts +56 -8
  64. package/src/orchestrator/orchestrator.ts +124 -83
  65. package/src/prover-client/prover-client.ts +29 -14
  66. package/src/proving_broker/broker_prover_facade.ts +6 -3
  67. package/src/proving_broker/config.ts +22 -0
  68. package/src/proving_broker/proof_store/factory.ts +10 -32
  69. package/src/proving_broker/proof_store/file_store_proof_store.ts +78 -0
  70. package/src/proving_broker/proof_store/index.ts +1 -1
  71. package/src/proving_broker/proving_agent.ts +5 -2
  72. package/src/proving_broker/proving_broker.ts +30 -1
  73. package/src/proving_broker/proving_broker_instrumentation.ts +10 -6
  74. package/src/proving_broker/proving_job_controller.ts +9 -3
  75. package/src/proving_broker/rpc.ts +14 -0
  76. package/dest/block-factory/index.d.ts +0 -2
  77. package/dest/block-factory/index.d.ts.map +0 -1
  78. package/dest/block-factory/index.js +0 -1
  79. package/dest/block-factory/light.d.ts +0 -38
  80. package/dest/block-factory/light.d.ts.map +0 -1
  81. package/dest/block-factory/light.js +0 -106
  82. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +0 -14
  83. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +0 -1
  84. package/dest/proving_broker/proof_store/gcs_proof_store.js +0 -52
  85. package/src/block-factory/index.ts +0 -1
  86. package/src/block-factory/light.ts +0 -136
  87. package/src/proving_broker/proof_store/gcs_proof_store.ts +0 -76
@@ -1,7 +1,7 @@
1
1
  import { type ACVMConfig, type BBConfig, BBNativeRollupProver, TestCircuitProver } from '@aztec/bb-prover';
2
2
  import { times } from '@aztec/foundation/collection';
3
3
  import type { EthAddress } from '@aztec/foundation/eth-address';
4
- import { createLogger } from '@aztec/foundation/log';
4
+ import { type Logger, createLogger } from '@aztec/foundation/log';
5
5
  import { NativeACVMSimulator } from '@aztec/simulator/server';
6
6
  import {
7
7
  type ActualProverConfig,
@@ -29,24 +29,34 @@ export class ProverClient implements EpochProverManager {
29
29
  private running = false;
30
30
  private agents: ProvingAgent[] = [];
31
31
 
32
- private proofStore: ProofStore;
33
- private failedProofStore: ProofStore | undefined;
34
-
35
32
  private constructor(
36
33
  private config: ProverClientConfig,
37
34
  private worldState: ForkMerkleTreeOperations & ReadonlyWorldStateAccess,
38
35
  private orchestratorClient: ProvingJobProducer,
36
+ private proofStore: ProofStore,
37
+ private failedProofStore: ProofStore | undefined,
39
38
  private agentClient?: ProvingJobConsumer,
40
39
  private telemetry: TelemetryClient = getTelemetryClient(),
41
- private log = createLogger('prover-client:tx-prover'),
42
- ) {
43
- this.proofStore = new InlineProofStore();
44
- this.failedProofStore = this.config.failedProofStore ? createProofStore(this.config.failedProofStore) : undefined;
45
- }
40
+ private log: Logger = createLogger('prover-client:tx-prover'),
41
+ ) {}
46
42
 
47
43
  public createEpochProver(): EpochProver {
48
- const facade = new BrokerCircuitProverFacade(this.orchestratorClient, this.proofStore, this.failedProofStore);
49
- const orchestrator = new ProvingOrchestrator(this.worldState, facade, this.config.proverId, this.telemetry);
44
+ const bindings = this.log.getBindings();
45
+ const facade = new BrokerCircuitProverFacade(
46
+ this.orchestratorClient,
47
+ this.proofStore,
48
+ this.failedProofStore,
49
+ undefined,
50
+ bindings,
51
+ );
52
+ const orchestrator = new ProvingOrchestrator(
53
+ this.worldState,
54
+ facade,
55
+ this.config.proverId,
56
+ this.config.cancelJobsOnStop,
57
+ this.telemetry,
58
+ bindings,
59
+ );
50
60
  return new ServerEpochProver(facade, orchestrator);
51
61
  }
52
62
 
@@ -104,7 +114,9 @@ export class ProverClient implements EpochProverManager {
104
114
  broker: ProvingJobBroker,
105
115
  telemetry: TelemetryClient = getTelemetryClient(),
106
116
  ) {
107
- const prover = new ProverClient(config, worldState, broker, broker, telemetry);
117
+ const proofStore = await createProofStore(config.proofStore);
118
+ const failedProofStore = config.failedProofStore ? await createProofStore(config.failedProofStore) : undefined;
119
+ const prover = new ProverClient(config, worldState, broker, proofStore, failedProofStore, broker, telemetry);
108
120
  await prover.start();
109
121
  return prover;
110
122
  }
@@ -128,9 +140,11 @@ export class ProverClient implements EpochProverManager {
128
140
 
129
141
  const proofStore = new InlineProofStore();
130
142
  const prover = await buildServerCircuitProver(this.config, this.telemetry);
143
+ const bindings = this.log.getBindings();
131
144
  this.agents = times(
132
145
  this.config.proverAgentCount,
133
- () => new ProvingAgent(this.agentClient!, proofStore, prover, [], this.config.proverAgentPollIntervalMs),
146
+ () =>
147
+ new ProvingAgent(this.agentClient!, proofStore, prover, [], this.config.proverAgentPollIntervalMs, bindings),
134
148
  );
135
149
 
136
150
  await Promise.all(this.agents.map(agent => agent.start()));
@@ -149,8 +163,9 @@ export function buildServerCircuitProver(
149
163
  return BBNativeRollupProver.new(config, telemetry);
150
164
  }
151
165
 
166
+ const logger = createLogger('prover-client:acvm-native');
152
167
  const simulator = config.acvmBinaryPath
153
- ? new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath)
168
+ ? new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath, undefined, logger)
154
169
  : undefined;
155
170
 
156
171
  return Promise.resolve(new TestCircuitProver(simulator, config, telemetry));
@@ -6,7 +6,7 @@ import type {
6
6
  } from '@aztec/constants';
7
7
  import { EpochNumber } from '@aztec/foundation/branded-types';
8
8
  import { sha256 } from '@aztec/foundation/crypto/sha256';
9
- import { createLogger } from '@aztec/foundation/log';
9
+ import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
10
10
  import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
11
11
  import { truncate } from '@aztec/foundation/string';
12
12
  import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
@@ -68,14 +68,17 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
68
68
  private runningPromise?: RunningPromise;
69
69
  private timeOfLastSnapshotSync = Date.now();
70
70
  private jobsToRetrieve: Set<ProvingJobId> = new Set();
71
+ private log: Logger;
71
72
 
72
73
  constructor(
73
74
  private broker: ProvingJobProducer,
74
75
  private proofStore: ProofStore = new InlineProofStore(),
75
76
  private failedProofStore?: ProofStore,
76
77
  private pollIntervalMs = 1000,
77
- private log = createLogger('prover-client:broker-circuit-prover-facade'),
78
- ) {}
78
+ bindings?: LoggerBindings,
79
+ ) {
80
+ this.log = createLogger('prover-client:broker-circuit-prover-facade', bindings);
81
+ }
79
82
 
80
83
  /**
81
84
  * This is a critical section. This function can not be async since it writes
@@ -31,6 +31,8 @@ export const ProverBrokerConfig = z.object({
31
31
  proverBrokerBatchIntervalMs: z.number().int().nonnegative(),
32
32
  /** The maximum number of epochs to keep results for */
33
33
  proverBrokerMaxEpochsToKeepResultsFor: z.number().int().nonnegative(),
34
+ /** Enable debug replay mode for replaying proving jobs from stored inputs */
35
+ proverBrokerDebugReplayEnabled: z.boolean(),
34
36
  });
35
37
 
36
38
  export type ProverBrokerConfig = z.infer<typeof ProverBrokerConfig> &
@@ -74,6 +76,11 @@ export const proverBrokerConfigMappings: ConfigMappingsType<ProverBrokerConfig>
74
76
  parseEnv: (val: string | undefined) => (val ? +val : undefined),
75
77
  description: "The size of the prover broker's database. Will override the dataStoreMapSizeKb if set.",
76
78
  },
79
+ proverBrokerDebugReplayEnabled: {
80
+ env: 'PROVER_BROKER_DEBUG_REPLAY_ENABLED',
81
+ description: 'Enable debug replay mode for replaying proving jobs from stored inputs',
82
+ ...booleanConfigHelper(false),
83
+ },
77
84
  ...dataConfigMappings,
78
85
  ...l1ReaderConfigMappings,
79
86
  ...pickConfigMappings(chainConfigMappings, ['rollupVersion']),
@@ -100,6 +107,10 @@ export const ProverAgentConfig = z.object({
100
107
  proverTestDelayFactor: z.number(),
101
108
  /** The delay (ms) to inject during fake proof verification */
102
109
  proverTestVerificationDelayMs: z.number().optional(),
110
+ /** Whether to abort pending proving jobs when the orchestrator is cancelled */
111
+ cancelJobsOnStop: z.boolean(),
112
+ /** Where to store proving results. Must be accessible to both prover node and agents. If not set will inline-encode the parameters */
113
+ proofStore: z.string().optional(),
103
114
  });
104
115
 
105
116
  export type ProverAgentConfig = z.infer<typeof ProverAgentConfig>;
@@ -153,4 +164,15 @@ export const proverAgentConfigMappings: ConfigMappingsType<ProverAgentConfig> =
153
164
  description: 'The delay (ms) to inject during fake proof verification',
154
165
  ...numberConfigHelper(10),
155
166
  },
167
+ cancelJobsOnStop: {
168
+ env: 'PROVER_CANCEL_JOBS_ON_STOP',
169
+ description:
170
+ 'Whether to abort pending proving jobs when the orchestrator is cancelled. ' +
171
+ 'When false (default), jobs remain in the broker queue and can be reused on restart/reorg.',
172
+ ...booleanConfigHelper(false),
173
+ },
174
+ proofStore: {
175
+ env: 'PROVER_PROOF_STORE',
176
+ description: 'Optional proof input store for the prover',
177
+ },
156
178
  };
@@ -1,42 +1,20 @@
1
1
  import { createLogger } from '@aztec/foundation/log';
2
+ import { createFileStore } from '@aztec/stdlib/file-store';
2
3
 
3
- import { GoogleCloudStorageProofStore } from './gcs_proof_store.js';
4
+ import { FileStoreProofStore } from './file_store_proof_store.js';
4
5
  import { InlineProofStore } from './inline_proof_store.js';
5
6
  import type { ProofStore } from './proof_store.js';
6
7
 
7
- export function createProofStore(config: string | undefined, logger = createLogger('prover-client:proof-store')) {
8
- if (config === undefined) {
8
+ export async function createProofStore(
9
+ config: string | undefined,
10
+ logger = createLogger('prover-client:proof-store'),
11
+ ): Promise<ProofStore> {
12
+ if (!config) {
9
13
  logger.info('Creating inline proof store');
10
14
  return new InlineProofStore();
11
- } else if (config.startsWith('gs://')) {
12
- try {
13
- const url = new URL(config);
14
- const bucket = url.host;
15
- const path = url.pathname.replace(/^\/+/, '');
16
- logger.info(`Creating google cloud proof store at ${bucket}`, { bucket, path });
17
- return new GoogleCloudStorageProofStore(bucket, path);
18
- } catch {
19
- throw new Error(
20
- `Invalid google cloud proof store definition: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`,
21
- );
22
- }
23
- } else {
24
- throw new Error(`Unknown proof store config: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`);
25
15
  }
26
- }
27
16
 
28
- export function createProofStoreForUri(
29
- uri: string,
30
- logger = createLogger('prover-client:proof-store'),
31
- ): Pick<ProofStore, 'getProofInput' | 'getProofOutput'> {
32
- if (uri.startsWith('data://')) {
33
- return createProofStore(undefined, logger);
34
- } else if (uri.startsWith('gs://')) {
35
- const url = new URL(uri);
36
- const basePath = url.pathname.replace(/^\/+/, '').split('/').slice(0, -3);
37
- url.pathname = basePath.join('/');
38
- return createProofStore(uri, logger);
39
- } else {
40
- throw new Error(`Unknown proof store config: '${uri}'. Supported protocols are 'data://' and 'gs://'.`);
41
- }
17
+ const fileStore = await createFileStore(config, logger);
18
+ logger.info(`Creating file store proof store at ${config}`);
19
+ return new FileStoreProofStore(fileStore);
42
20
  }
@@ -0,0 +1,78 @@
1
+ import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
2
+ import type { FileStore } from '@aztec/stdlib/file-store';
3
+ import {
4
+ type ProofUri,
5
+ type ProvingJobId,
6
+ type ProvingJobInputs,
7
+ type ProvingJobInputsMap,
8
+ ProvingJobResult,
9
+ type ProvingJobResultsMap,
10
+ getProvingJobInputClassFor,
11
+ } from '@aztec/stdlib/interfaces/server';
12
+ import { ProvingRequestType } from '@aztec/stdlib/proofs';
13
+
14
+ import type { ProofStore } from './proof_store.js';
15
+
16
+ const INPUTS_PATH = 'inputs';
17
+ const OUTPUTS_PATH = 'outputs';
18
+
19
+ /**
20
+ * A proof store implementation backed by a generic FileStore.
21
+ * Supports any storage backend (GCS, S3, local filesystem) via the FileStore abstraction.
22
+ */
23
+ export class FileStoreProofStore implements ProofStore {
24
+ constructor(private readonly fileStore: FileStore) {}
25
+
26
+ async saveProofInput<T extends ProvingRequestType>(
27
+ id: ProvingJobId,
28
+ type: T,
29
+ inputs: ProvingJobInputsMap[T],
30
+ ): Promise<ProofUri> {
31
+ const path = `${INPUTS_PATH}/${ProvingRequestType[type]}/${id}`;
32
+ const uri = await this.fileStore.save(path, inputs.toBuffer());
33
+ return uri as ProofUri;
34
+ }
35
+
36
+ async saveProofOutput<T extends ProvingRequestType>(
37
+ id: ProvingJobId,
38
+ type: T,
39
+ result: ProvingJobResultsMap[T],
40
+ ): Promise<ProofUri> {
41
+ const jobResult = { type, result } as ProvingJobResult;
42
+ const json = jsonStringify(jobResult);
43
+ const path = `${OUTPUTS_PATH}/${ProvingRequestType[type]}/${id}.json`;
44
+ const uri = await this.fileStore.save(path, Buffer.from(json, 'utf-8'));
45
+ return uri as ProofUri;
46
+ }
47
+
48
+ async getProofInput(uri: ProofUri): Promise<ProvingJobInputs> {
49
+ try {
50
+ const buffer = await this.fileStore.read(uri);
51
+ const type = this.extractTypeFromUri(uri);
52
+ const inputs = getProvingJobInputClassFor(type).fromBuffer(buffer);
53
+ return { inputs, type } as ProvingJobInputs;
54
+ } catch (err) {
55
+ throw new Error(`Error getting proof input at ${uri}: ${err}`);
56
+ }
57
+ }
58
+
59
+ async getProofOutput(uri: ProofUri): Promise<ProvingJobResult> {
60
+ try {
61
+ const buffer = await this.fileStore.read(uri);
62
+ return jsonParseWithSchema(buffer.toString('utf-8'), ProvingJobResult);
63
+ } catch (err) {
64
+ throw new Error(`Error getting proof output at ${uri}: ${err}`);
65
+ }
66
+ }
67
+
68
+ private extractTypeFromUri(uri: string): ProvingRequestType {
69
+ const url = new URL(uri);
70
+ const pathParts = url.pathname.split('/').filter(Boolean);
71
+ const typeString = pathParts.at(-2);
72
+ const type = typeString ? ProvingRequestType[typeString as keyof typeof ProvingRequestType] : undefined;
73
+ if (type === undefined) {
74
+ throw new Error(`Unrecognized proof type ${typeString} in URI ${uri}`);
75
+ }
76
+ return type;
77
+ }
78
+ }
@@ -1,4 +1,4 @@
1
1
  export * from './proof_store.js';
2
2
  export * from './inline_proof_store.js';
3
3
  export * from './factory.js';
4
- export * from './gcs_proof_store.js';
4
+ export * from './file_store_proof_store.js';
@@ -1,5 +1,5 @@
1
1
  import { AbortError } from '@aztec/foundation/error';
2
- import { createLogger } from '@aztec/foundation/log';
2
+ import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
3
3
  import { RunningPromise } from '@aztec/foundation/running-promise';
4
4
  import { truncate } from '@aztec/foundation/string';
5
5
  import { ProvingError } from '@aztec/stdlib/errors';
@@ -23,6 +23,7 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
23
23
  export class ProvingAgent {
24
24
  private currentJobController?: ProvingJobController;
25
25
  private runningPromise: RunningPromise;
26
+ private log: Logger;
26
27
 
27
28
  constructor(
28
29
  /** The source of proving jobs */
@@ -35,8 +36,9 @@ export class ProvingAgent {
35
36
  private proofAllowList: Array<ProvingRequestType> = [],
36
37
  /** How long to wait between jobs */
37
38
  private pollIntervalMs = 1000,
38
- private log = createLogger('prover-client:proving-agent'),
39
+ bindings?: LoggerBindings,
39
40
  ) {
41
+ this.log = createLogger('prover-client:proving-agent', bindings);
40
42
  this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollIntervalMs);
41
43
  }
42
44
 
@@ -159,6 +161,7 @@ export class ProvingAgent {
159
161
  // no need to await this here. The controller will stay alive (in DONE state) until the result is send to the broker
160
162
  void this.runningPromise.trigger();
161
163
  },
164
+ this.log.getBindings(),
162
165
  );
163
166
 
164
167
  if (abortedProofJobId) {
@@ -7,6 +7,7 @@ import {
7
7
  type GetProvingJobResponse,
8
8
  type ProofUri,
9
9
  type ProvingJob,
10
+ type ProvingJobBrokerDebug,
10
11
  type ProvingJobConsumer,
11
12
  type ProvingJobFilter,
12
13
  type ProvingJobId,
@@ -36,7 +37,7 @@ type EnqueuedProvingJob = Pick<ProvingJob, 'id' | 'epochNumber'>;
36
37
  * A broker that manages proof requests and distributes them to workers based on their priority.
37
38
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
38
39
  */
39
- export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable {
40
+ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, ProvingJobBrokerDebug, Traceable {
40
41
  private queues: ProvingQueues = {
41
42
  [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
42
43
  [ProvingRequestType.PUBLIC_CHONK_VERIFIER]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
@@ -114,6 +115,8 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
114
115
 
115
116
  private started = false;
116
117
 
118
+ private debugReplayEnabled: boolean;
119
+
117
120
  public constructor(
118
121
  private database: ProvingBrokerDatabase,
119
122
  {
@@ -121,6 +124,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
121
124
  proverBrokerPollIntervalMs,
122
125
  proverBrokerJobMaxRetries,
123
126
  proverBrokerMaxEpochsToKeepResultsFor,
127
+ proverBrokerDebugReplayEnabled,
124
128
  }: Required<
125
129
  Pick<
126
130
  ProverBrokerConfig,
@@ -128,6 +132,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
128
132
  | 'proverBrokerPollIntervalMs'
129
133
  | 'proverBrokerJobMaxRetries'
130
134
  | 'proverBrokerMaxEpochsToKeepResultsFor'
135
+ | 'proverBrokerDebugReplayEnabled'
131
136
  >
132
137
  > = defaultProverBrokerConfig,
133
138
  client: TelemetryClient = getTelemetryClient(),
@@ -139,6 +144,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
139
144
  this.jobTimeoutMs = proverBrokerJobTimeoutMs!;
140
145
  this.maxRetries = proverBrokerJobMaxRetries!;
141
146
  this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor!;
147
+ this.debugReplayEnabled = proverBrokerDebugReplayEnabled ?? false;
142
148
  }
143
149
 
144
150
  private measureQueueDepth: MonitorCallback = (type: ProvingRequestType) => {
@@ -241,6 +247,29 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
241
247
  return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter));
242
248
  }
243
249
 
250
+ public async replayProvingJob(
251
+ jobId: ProvingJobId,
252
+ type: ProvingRequestType,
253
+ epochNumber: EpochNumber,
254
+ inputsUri: ProofUri,
255
+ ): Promise<ProvingJobStatus> {
256
+ if (!this.debugReplayEnabled) {
257
+ throw new Error('Debug replay not enabled. Set PROVER_BROKER_DEBUG_REPLAY_ENABLED=true');
258
+ }
259
+
260
+ this.logger.info(`Replaying proving job`, { provingJobId: jobId, epochNumber, inputsUri });
261
+
262
+ // Clear existing state and enqueue
263
+ this.cleanUpProvingJobState([jobId]);
264
+
265
+ const job: ProvingJob = { id: jobId, type, epochNumber, inputsUri };
266
+ this.jobsCache.set(jobId, job);
267
+ await this.database.addProvingJob(job);
268
+ this.enqueueJobInternal(job);
269
+
270
+ return { status: 'in-queue' };
271
+ }
272
+
244
273
  async #enqueueProvingJob(job: ProvingJob): Promise<ProvingJobStatus> {
245
274
  // We return the job status at the start of this call
246
275
  const jobStatus = this.#getProvingJobStatus(job.id);
@@ -8,6 +8,7 @@ import {
8
8
  type ObservableResult,
9
9
  type TelemetryClient,
10
10
  type UpDownCounter,
11
+ createUpDownCounterWithDefault,
11
12
  } from '@aztec/telemetry-client';
12
13
 
13
14
  export type MonitorCallback = (proofType: ProvingRequestType) => number;
@@ -31,17 +32,20 @@ export class ProvingBrokerInstrumentation {
31
32
 
32
33
  this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS);
33
34
 
34
- this.resolvedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RESOLVED_JOBS);
35
+ const provingJobTypes = Object.values(ProvingRequestType).filter(v => typeof v === 'string');
36
+ const provingJobAttrs = { [Attributes.PROVING_JOB_TYPE]: provingJobTypes };
35
37
 
36
- this.rejectedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_REJECTED_JOBS);
38
+ this.resolvedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RESOLVED_JOBS, provingJobAttrs);
37
39
 
38
- this.retriedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RETRIED_JOBS);
40
+ this.rejectedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_REJECTED_JOBS, provingJobAttrs);
39
41
 
40
- this.timedOutJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TIMED_OUT_JOBS);
42
+ this.retriedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RETRIED_JOBS, provingJobAttrs);
41
43
 
42
- this.cachedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_CACHED_JOBS);
44
+ this.timedOutJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, provingJobAttrs);
43
45
 
44
- this.totalJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TOTAL_JOBS);
46
+ this.cachedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_CACHED_JOBS, provingJobAttrs);
47
+
48
+ this.totalJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TOTAL_JOBS, provingJobAttrs);
45
49
 
46
50
  this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT);
47
51
 
@@ -1,7 +1,7 @@
1
1
  import { EpochNumber } from '@aztec/foundation/branded-types';
2
2
  import { randomBytes } from '@aztec/foundation/crypto/random';
3
3
  import { AbortError } from '@aztec/foundation/error';
4
- import { createLogger } from '@aztec/foundation/log';
4
+ import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
5
5
  import type {
6
6
  ProvingJobId,
7
7
  ProvingJobInputs,
@@ -21,6 +21,7 @@ export class ProvingJobController {
21
21
  private promise?: Promise<void>;
22
22
  private abortController = new AbortController();
23
23
  private result?: ProvingJobResultsMap[ProvingRequestType] | Error;
24
+ private log: Logger;
24
25
 
25
26
  constructor(
26
27
  private jobId: ProvingJobId,
@@ -29,8 +30,13 @@ export class ProvingJobController {
29
30
  private startedAt: number,
30
31
  private circuitProver: ServerCircuitProver,
31
32
  private onComplete: () => void,
32
- private log = createLogger('prover-client:proving-agent:job-controller-' + randomBytes(4).toString('hex')),
33
- ) {}
33
+ bindings?: LoggerBindings,
34
+ ) {
35
+ this.log = createLogger('prover-client:proving-agent:job-controller', {
36
+ instanceId: randomBytes(4).toString('hex'),
37
+ ...bindings,
38
+ });
39
+ }
34
40
 
35
41
  public start(): void {
36
42
  if (this.status !== ProvingJobControllerStatus.IDLE) {
@@ -1,9 +1,11 @@
1
+ import { EpochNumberSchema } from '@aztec/foundation/branded-types';
1
2
  import { createSafeJsonRpcClient } from '@aztec/foundation/json-rpc/client';
2
3
  import {
3
4
  type GetProvingJobResponse,
4
5
  ProofUri,
5
6
  ProvingJob,
6
7
  type ProvingJobBroker,
8
+ type ProvingJobBrokerDebug,
7
9
  type ProvingJobConsumer,
8
10
  ProvingJobId,
9
11
  type ProvingJobProducer,
@@ -53,6 +55,18 @@ export const ProvingJobBrokerSchema: ApiSchemaFor<ProvingJobBroker> = {
53
55
  ...ProvingJobProducerSchema,
54
56
  };
55
57
 
58
+ export const ProvingJobBrokerDebugSchema: ApiSchemaFor<ProvingJobBrokerDebug> = {
59
+ replayProvingJob: z
60
+ .function()
61
+ .args(ProvingJobId, z.nativeEnum(ProvingRequestType), EpochNumberSchema, ProofUri)
62
+ .returns(ProvingJobStatus),
63
+ };
64
+
65
+ export const ProvingJobBrokerSchemaWithDebug: ApiSchemaFor<ProvingJobBroker & ProvingJobBrokerDebug> = {
66
+ ...ProvingJobBrokerSchema,
67
+ ...ProvingJobBrokerDebugSchema,
68
+ };
69
+
56
70
  export function createProvingJobBrokerClient(
57
71
  url: string,
58
72
  versions: Partial<ComponentsVersions>,
@@ -1,2 +0,0 @@
1
- export * from './light.js';
2
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9ibG9jay1mYWN0b3J5L2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGNBQWMsWUFBWSxDQUFDIn0=
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/block-factory/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC"}
@@ -1 +0,0 @@
1
- export * from './light.js';
@@ -1,38 +0,0 @@
1
- import { Fr } from '@aztec/foundation/curves/bn254';
2
- import { L2Block } from '@aztec/stdlib/block';
3
- import type { IBlockFactory, MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
4
- import type { GlobalVariables, ProcessedTx } from '@aztec/stdlib/tx';
5
- import { type TelemetryClient } from '@aztec/telemetry-client';
6
- /**
7
- * Builds a block and its header from a set of processed tx without running any circuits.
8
- *
9
- * NOTE: the onus is ON THE CALLER to update the db that is passed in with the notes hashes, nullifiers, etc
10
- * PRIOR to calling `buildBlock`.
11
- *
12
- * Why? Because if you are, e.g. building a block in practice from TxObjects, you are using the
13
- * PublicProcessor which will do this for you as it processes transactions.
14
- *
15
- * If you haven't already inserted the side effects, e.g. because you are in a testing context, you can use the helper
16
- * function `buildBlockWithCleanDB`, which calls `insertSideEffects` for you.
17
- *
18
- * @deprecated Use LightweightCheckpointBuilder instead. This only works for one block per checkpoint.
19
- */
20
- export declare class LightweightBlockFactory implements IBlockFactory {
21
- private db;
22
- private telemetry;
23
- private globalVariables?;
24
- private l1ToL2Messages?;
25
- private txs;
26
- private readonly logger;
27
- constructor(db: MerkleTreeWriteOperations, telemetry?: TelemetryClient);
28
- startNewBlock(globalVariables: GlobalVariables, l1ToL2Messages: Fr[]): Promise<void>;
29
- addTxs(txs: ProcessedTx[]): Promise<void>;
30
- setBlockCompleted(): Promise<L2Block>;
31
- private buildBlock;
32
- }
33
- /**
34
- * Inserts the processed transactions into the DB, then creates a block.
35
- * @param db - A db fork to use for block building which WILL BE MODIFIED.
36
- */
37
- export declare function buildBlockWithCleanDB(txs: ProcessedTx[], globalVariables: GlobalVariables, l1ToL2Messages: Fr[], db: MerkleTreeWriteOperations, telemetry?: TelemetryClient): Promise<L2Block>;
38
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibGlnaHQuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9ibG9jay1mYWN0b3J5L2xpZ2h0LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUdBLE9BQU8sRUFBRSxFQUFFLEVBQUUsTUFBTSxnQ0FBZ0MsQ0FBQztBQUVwRCxPQUFPLEVBQUUsT0FBTyxFQUFpQixNQUFNLHFCQUFxQixDQUFDO0FBQzdELE9BQU8sS0FBSyxFQUFFLGFBQWEsRUFBRSx5QkFBeUIsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBR2hHLE9BQU8sS0FBSyxFQUFFLGVBQWUsRUFBRSxXQUFXLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUNyRSxPQUFPLEVBQUUsS0FBSyxlQUFlLEVBQXNCLE1BQU0seUJBQXlCLENBQUM7QUFRbkY7Ozs7Ozs7Ozs7Ozs7R0FhRztBQUNILHFCQUFhLHVCQUF3QixZQUFXLGFBQWE7SUFRekQsT0FBTyxDQUFDLEVBQUU7SUFDVixPQUFPLENBQUMsU0FBUztJQVJuQixPQUFPLENBQUMsZUFBZSxDQUFDLENBQWtCO0lBQzFDLE9BQU8sQ0FBQyxjQUFjLENBQUMsQ0FBTztJQUM5QixPQUFPLENBQUMsR0FBRyxDQUE0QjtJQUV2QyxPQUFPLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBNkM7SUFFcEUsWUFDVSxFQUFFLEVBQUUseUJBQXlCLEVBQzdCLFNBQVMsR0FBRSxlQUFzQyxFQUN2RDtJQUVFLGFBQWEsQ0FBQyxlQUFlLEVBQUUsZUFBZSxFQUFFLGNBQWMsRUFBRSxFQUFFLEVBQUUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBT3pGO0lBRUQsTUFBTSxDQUFDLEdBQUcsRUFBRSxXQUFXLEVBQUUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBU3hDO0lBRUQsaUJBQWlCLElBQUksT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUVwQztZQUVhLFVBQVU7Q0E2Q3pCO0FBRUQ7OztHQUdHO0FBQ0gsd0JBQXNCLHFCQUFxQixDQUN6QyxHQUFHLEVBQUUsV0FBVyxFQUFFLEVBQ2xCLGVBQWUsRUFBRSxlQUFlLEVBQ2hDLGNBQWMsRUFBRSxFQUFFLEVBQUUsRUFDcEIsRUFBRSxFQUFFLHlCQUF5QixFQUM3QixTQUFTLEdBQUUsZUFBc0Msb0JBV2xEIn0=
@@ -1 +0,0 @@
1
- {"version":3,"file":"light.d.ts","sourceRoot":"","sources":["../../src/block-factory/light.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,EAAE,EAAE,MAAM,gCAAgC,CAAC;AAEpD,OAAO,EAAE,OAAO,EAAiB,MAAM,qBAAqB,CAAC;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAC;AAGhG,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,yBAAyB,CAAC;AAQnF;;;;;;;;;;;;;GAaG;AACH,qBAAa,uBAAwB,YAAW,aAAa;IAQzD,OAAO,CAAC,EAAE;IACV,OAAO,CAAC,SAAS;IARnB,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,cAAc,CAAC,CAAO;IAC9B,OAAO,CAAC,GAAG,CAA4B;IAEvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6C;IAEpE,YACU,EAAE,EAAE,yBAAyB,EAC7B,SAAS,GAAE,eAAsC,EACvD;IAEE,aAAa,CAAC,eAAe,EAAE,eAAe,EAAE,cAAc,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAOzF;IAED,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CASxC;IAED,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC,CAEpC;YAEa,UAAU;CA6CzB;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,GAAG,EAAE,WAAW,EAAE,EAClB,eAAe,EAAE,eAAe,EAChC,cAAc,EAAE,EAAE,EAAE,EACpB,EAAE,EAAE,yBAAyB,EAC7B,SAAS,GAAE,eAAsC,oBAWlD"}