@aztec/prover-client 0.0.1-commit.e3c1de76 → 0.0.1-commit.e57c76e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/dest/config.d.ts +1 -1
  2. package/dest/config.d.ts.map +1 -1
  3. package/dest/config.js +16 -2
  4. package/dest/light/lightweight_checkpoint_builder.d.ts +10 -5
  5. package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -1
  6. package/dest/light/lightweight_checkpoint_builder.js +53 -22
  7. package/dest/mocks/test_context.d.ts +3 -1
  8. package/dest/mocks/test_context.d.ts.map +1 -1
  9. package/dest/mocks/test_context.js +18 -9
  10. package/dest/orchestrator/block-building-helpers.d.ts +4 -4
  11. package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
  12. package/dest/orchestrator/block-building-helpers.js +2 -2
  13. package/dest/orchestrator/block-proving-state.d.ts +4 -1
  14. package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
  15. package/dest/orchestrator/block-proving-state.js +7 -0
  16. package/dest/orchestrator/checkpoint-proving-state.d.ts +10 -3
  17. package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -1
  18. package/dest/orchestrator/checkpoint-proving-state.js +13 -4
  19. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts +107 -0
  20. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts.map +1 -0
  21. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.js +151 -0
  22. package/dest/orchestrator/epoch-proving-context.d.ts +51 -0
  23. package/dest/orchestrator/epoch-proving-context.d.ts.map +1 -0
  24. package/dest/orchestrator/epoch-proving-context.js +81 -0
  25. package/dest/orchestrator/epoch-proving-state.d.ts +3 -3
  26. package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -1
  27. package/dest/orchestrator/epoch-proving-state.js +5 -3
  28. package/dest/orchestrator/index.d.ts +4 -1
  29. package/dest/orchestrator/index.d.ts.map +1 -1
  30. package/dest/orchestrator/index.js +3 -0
  31. package/dest/orchestrator/orchestrator.d.ts +16 -26
  32. package/dest/orchestrator/orchestrator.d.ts.map +1 -1
  33. package/dest/orchestrator/orchestrator.js +76 -206
  34. package/dest/orchestrator/proving-scheduler.d.ts +72 -0
  35. package/dest/orchestrator/proving-scheduler.d.ts.map +1 -0
  36. package/dest/orchestrator/proving-scheduler.js +117 -0
  37. package/dest/orchestrator/top-tree-orchestrator.d.ts +83 -0
  38. package/dest/orchestrator/top-tree-orchestrator.d.ts.map +1 -0
  39. package/dest/orchestrator/top-tree-orchestrator.js +182 -0
  40. package/dest/orchestrator/top-tree-proving-scheduler.d.ts +62 -0
  41. package/dest/orchestrator/top-tree-proving-scheduler.d.ts.map +1 -0
  42. package/dest/orchestrator/top-tree-proving-scheduler.js +73 -0
  43. package/dest/orchestrator/top-tree-proving-state.d.ts +61 -0
  44. package/dest/orchestrator/top-tree-proving-state.d.ts.map +1 -0
  45. package/dest/orchestrator/top-tree-proving-state.js +185 -0
  46. package/dest/prover-client/prover-client.d.ts +64 -5
  47. package/dest/prover-client/prover-client.d.ts.map +1 -1
  48. package/dest/prover-client/prover-client.js +58 -8
  49. package/dest/proving_broker/broker_prover_facade.d.ts +1 -1
  50. package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
  51. package/dest/proving_broker/broker_prover_facade.js +13 -19
  52. package/dest/proving_broker/config.d.ts +11 -67
  53. package/dest/proving_broker/config.d.ts.map +1 -1
  54. package/dest/proving_broker/config.js +16 -5
  55. package/dest/proving_broker/index.d.ts +2 -1
  56. package/dest/proving_broker/index.d.ts.map +1 -1
  57. package/dest/proving_broker/index.js +1 -0
  58. package/dest/proving_broker/proof_store/factory.d.ts +2 -5
  59. package/dest/proving_broker/proof_store/factory.d.ts.map +1 -1
  60. package/dest/proving_broker/proof_store/factory.js +7 -30
  61. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts +18 -0
  62. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts.map +1 -0
  63. package/dest/proving_broker/proof_store/file_store_proof_store.js +60 -0
  64. package/dest/proving_broker/proof_store/index.d.ts +2 -2
  65. package/dest/proving_broker/proof_store/index.d.ts.map +1 -1
  66. package/dest/proving_broker/proof_store/index.js +1 -1
  67. package/dest/proving_broker/proving_broker.d.ts +8 -5
  68. package/dest/proving_broker/proving_broker.d.ts.map +1 -1
  69. package/dest/proving_broker/proving_broker.js +68 -11
  70. package/dest/proving_broker/proving_broker_database/persisted.js +2 -2
  71. package/dest/proving_broker/proving_broker_instrumentation.d.ts +3 -1
  72. package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
  73. package/dest/proving_broker/proving_broker_instrumentation.js +7 -0
  74. package/dest/proving_broker/rpc.d.ts +6 -2
  75. package/dest/proving_broker/rpc.d.ts.map +1 -1
  76. package/dest/proving_broker/rpc.js +87 -23
  77. package/dest/test/mock_prover.d.ts +4 -4
  78. package/package.json +18 -19
  79. package/src/config.ts +18 -2
  80. package/src/light/lightweight_checkpoint_builder.ts +56 -25
  81. package/src/mocks/test_context.ts +13 -10
  82. package/src/orchestrator/block-building-helpers.ts +2 -2
  83. package/src/orchestrator/block-proving-state.ts +9 -0
  84. package/src/orchestrator/checkpoint-proving-state.ts +18 -5
  85. package/src/orchestrator/checkpoint-sub-tree-orchestrator.ts +271 -0
  86. package/src/orchestrator/epoch-proving-context.ts +101 -0
  87. package/src/orchestrator/epoch-proving-state.ts +6 -4
  88. package/src/orchestrator/index.ts +8 -0
  89. package/src/orchestrator/orchestrator.ts +98 -268
  90. package/src/orchestrator/proving-scheduler.ts +156 -0
  91. package/src/orchestrator/top-tree-orchestrator.ts +314 -0
  92. package/src/orchestrator/top-tree-proving-scheduler.ts +154 -0
  93. package/src/orchestrator/top-tree-proving-state.ts +220 -0
  94. package/src/prover-client/prover-client.ts +132 -9
  95. package/src/proving_broker/broker_prover_facade.ts +17 -20
  96. package/src/proving_broker/config.ts +16 -2
  97. package/src/proving_broker/index.ts +1 -0
  98. package/src/proving_broker/proof_store/factory.ts +10 -32
  99. package/src/proving_broker/proof_store/file_store_proof_store.ts +78 -0
  100. package/src/proving_broker/proof_store/index.ts +1 -1
  101. package/src/proving_broker/proving_broker.ts +64 -8
  102. package/src/proving_broker/proving_broker_database/persisted.ts +2 -2
  103. package/src/proving_broker/proving_broker_instrumentation.ts +9 -0
  104. package/src/proving_broker/rpc.ts +46 -20
  105. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +0 -14
  106. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +0 -1
  107. package/dest/proving_broker/proof_store/gcs_proof_store.js +0 -52
  108. package/src/proving_broker/proof_store/gcs_proof_store.ts +0 -76
@@ -0,0 +1,78 @@
1
+ import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
2
+ import type { FileStore } from '@aztec/stdlib/file-store';
3
+ import {
4
+ type ProofUri,
5
+ type ProvingJobId,
6
+ type ProvingJobInputs,
7
+ type ProvingJobInputsMap,
8
+ ProvingJobResult,
9
+ type ProvingJobResultsMap,
10
+ getProvingJobInputClassFor,
11
+ } from '@aztec/stdlib/interfaces/server';
12
+ import { ProvingRequestType } from '@aztec/stdlib/proofs';
13
+
14
+ import type { ProofStore } from './proof_store.js';
15
+
16
+ const INPUTS_PATH = 'inputs';
17
+ const OUTPUTS_PATH = 'outputs';
18
+
19
+ /**
20
+ * A proof store implementation backed by a generic FileStore.
21
+ * Supports any storage backend (GCS, S3, local filesystem) via the FileStore abstraction.
22
+ */
23
+ export class FileStoreProofStore implements ProofStore {
24
+ constructor(private readonly fileStore: FileStore) {}
25
+
26
+ async saveProofInput<T extends ProvingRequestType>(
27
+ id: ProvingJobId,
28
+ type: T,
29
+ inputs: ProvingJobInputsMap[T],
30
+ ): Promise<ProofUri> {
31
+ const path = `${INPUTS_PATH}/${ProvingRequestType[type]}/${id}`;
32
+ const uri = await this.fileStore.save(path, inputs.toBuffer());
33
+ return uri as ProofUri;
34
+ }
35
+
36
+ async saveProofOutput<T extends ProvingRequestType>(
37
+ id: ProvingJobId,
38
+ type: T,
39
+ result: ProvingJobResultsMap[T],
40
+ ): Promise<ProofUri> {
41
+ const jobResult = { type, result } as ProvingJobResult;
42
+ const json = jsonStringify(jobResult);
43
+ const path = `${OUTPUTS_PATH}/${ProvingRequestType[type]}/${id}.json`;
44
+ const uri = await this.fileStore.save(path, Buffer.from(json, 'utf-8'));
45
+ return uri as ProofUri;
46
+ }
47
+
48
+ async getProofInput(uri: ProofUri): Promise<ProvingJobInputs> {
49
+ try {
50
+ const buffer = await this.fileStore.read(uri);
51
+ const type = this.extractTypeFromUri(uri);
52
+ const inputs = getProvingJobInputClassFor(type).fromBuffer(buffer);
53
+ return { inputs, type } as ProvingJobInputs;
54
+ } catch (err) {
55
+ throw new Error(`Error getting proof input at ${uri}: ${err}`);
56
+ }
57
+ }
58
+
59
+ async getProofOutput(uri: ProofUri): Promise<ProvingJobResult> {
60
+ try {
61
+ const buffer = await this.fileStore.read(uri);
62
+ return jsonParseWithSchema(buffer.toString('utf-8'), ProvingJobResult);
63
+ } catch (err) {
64
+ throw new Error(`Error getting proof output at ${uri}: ${err}`);
65
+ }
66
+ }
67
+
68
+ private extractTypeFromUri(uri: string): ProvingRequestType {
69
+ const url = new URL(uri);
70
+ const pathParts = url.pathname.split('/').filter(Boolean);
71
+ const typeString = pathParts.at(-2);
72
+ const type = typeString ? ProvingRequestType[typeString as keyof typeof ProvingRequestType] : undefined;
73
+ if (type === undefined) {
74
+ throw new Error(`Unrecognized proof type ${typeString} in URI ${uri}`);
75
+ }
76
+ return type;
77
+ }
78
+ }
@@ -1,4 +1,4 @@
1
1
  export * from './proof_store.js';
2
2
  export * from './inline_proof_store.js';
3
3
  export * from './factory.js';
4
- export * from './gcs_proof_store.js';
4
+ export * from './file_store_proof_store.js';
@@ -7,6 +7,7 @@ import {
7
7
  type GetProvingJobResponse,
8
8
  type ProofUri,
9
9
  type ProvingJob,
10
+ type ProvingJobBrokerDebug,
10
11
  type ProvingJobConsumer,
11
12
  type ProvingJobFilter,
12
13
  type ProvingJobId,
@@ -36,7 +37,7 @@ type EnqueuedProvingJob = Pick<ProvingJob, 'id' | 'epochNumber'>;
36
37
  * A broker that manages proof requests and distributes them to workers based on their priority.
37
38
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
38
39
  */
39
- export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable {
40
+ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, ProvingJobBrokerDebug, Traceable {
40
41
  private queues: ProvingQueues = {
41
42
  [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
42
43
  [ProvingRequestType.PUBLIC_CHONK_VERIFIER]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
@@ -114,6 +115,8 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
114
115
 
115
116
  private started = false;
116
117
 
118
+ private debugReplayEnabled: boolean;
119
+
117
120
  public constructor(
118
121
  private database: ProvingBrokerDatabase,
119
122
  {
@@ -121,6 +124,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
121
124
  proverBrokerPollIntervalMs,
122
125
  proverBrokerJobMaxRetries,
123
126
  proverBrokerMaxEpochsToKeepResultsFor,
127
+ proverBrokerDebugReplayEnabled,
124
128
  }: Required<
125
129
  Pick<
126
130
  ProverBrokerConfig,
@@ -128,6 +132,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
128
132
  | 'proverBrokerPollIntervalMs'
129
133
  | 'proverBrokerJobMaxRetries'
130
134
  | 'proverBrokerMaxEpochsToKeepResultsFor'
135
+ | 'proverBrokerDebugReplayEnabled'
131
136
  >
132
137
  > = defaultProverBrokerConfig,
133
138
  client: TelemetryClient = getTelemetryClient(),
@@ -139,6 +144,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
139
144
  this.jobTimeoutMs = proverBrokerJobTimeoutMs!;
140
145
  this.maxRetries = proverBrokerJobMaxRetries!;
141
146
  this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor!;
147
+ this.debugReplayEnabled = proverBrokerDebugReplayEnabled ?? false;
142
148
  }
143
149
 
144
150
  private measureQueueDepth: MonitorCallback = (type: ProvingRequestType) => {
@@ -241,6 +247,29 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
241
247
  return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter));
242
248
  }
243
249
 
250
+ public async replayProvingJob(
251
+ jobId: ProvingJobId,
252
+ type: ProvingRequestType,
253
+ epochNumber: EpochNumber,
254
+ inputsUri: ProofUri,
255
+ ): Promise<ProvingJobStatus> {
256
+ if (!this.debugReplayEnabled) {
257
+ throw new Error('Debug replay not enabled. Set PROVER_BROKER_DEBUG_REPLAY_ENABLED=true');
258
+ }
259
+
260
+ this.logger.info(`Replaying proving job`, { provingJobId: jobId, epochNumber, inputsUri });
261
+
262
+ // Clear existing state and enqueue
263
+ this.cleanUpProvingJobState([jobId]);
264
+
265
+ const job: ProvingJob = { id: jobId, type, epochNumber, inputsUri };
266
+ this.jobsCache.set(jobId, job);
267
+ await this.database.addProvingJob(job);
268
+ this.enqueueJobInternal(job);
269
+
270
+ return { status: 'in-queue' };
271
+ }
272
+
244
273
  async #enqueueProvingJob(job: ProvingJob): Promise<ProvingJobStatus> {
245
274
  // We return the job status at the start of this call
246
275
  const jobStatus = this.#getProvingJobStatus(job.id);
@@ -285,19 +314,25 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
285
314
  // notify listeners of the cancellation
286
315
  if (!this.resultsCache.has(id)) {
287
316
  this.logger.info(`Cancelling job id=${id}`, { provingJobId: id });
288
- await this.#reportProvingJobError(id, 'Aborted', false);
317
+ await this.#reportProvingJobError(id, 'Aborted', false, undefined, true);
289
318
  }
290
319
  }
291
320
 
292
321
  private cleanUpProvingJobState(ids: ProvingJobId[]) {
322
+ const idsToClean = new Set(ids);
293
323
  for (const id of ids) {
294
324
  this.jobsCache.delete(id);
325
+ const deferred = this.promises.get(id);
326
+ if (deferred) {
327
+ deferred.resolve({ status: 'rejected', reason: 'Proving job cleaned up' });
328
+ }
295
329
  this.promises.delete(id);
296
330
  this.resultsCache.delete(id);
297
331
  this.inProgress.delete(id);
298
332
  this.retries.delete(id);
299
333
  this.enqueuedAt.delete(id);
300
334
  }
335
+ this.completedJobNotifications = this.completedJobNotifications.filter(id => !idsToClean.has(id));
301
336
  }
302
337
 
303
338
  #getProvingJobStatus(id: ProvingJobId): ProvingJobStatus {
@@ -366,6 +401,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
366
401
  err: string,
367
402
  retry = false,
368
403
  filter?: ProvingJobFilter,
404
+ aborted = false,
369
405
  ): Promise<GetProvingJobResponse | undefined> {
370
406
  const info = this.inProgress.get(id);
371
407
  const item = this.jobsCache.get(id);
@@ -426,7 +462,11 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
426
462
  this.promises.get(id)!.resolve(result);
427
463
  this.completedJobNotifications.push(id);
428
464
 
429
- this.instrumentation.incRejectedJobs(item.type);
465
+ if (aborted) {
466
+ this.instrumentation.incAbortedJobs(item.type);
467
+ } else {
468
+ this.instrumentation.incRejectedJobs(item.type);
469
+ }
430
470
  if (info) {
431
471
  const duration = this.msTimeSource() - info.startedAt;
432
472
  this.instrumentation.recordJobDuration(item.type, duration);
@@ -560,21 +600,21 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
560
600
  }
561
601
 
562
602
  private async cleanupPass() {
563
- this.cleanupStaleJobs();
564
603
  this.reEnqueueExpiredJobs();
565
604
  const oldestEpochToKeep = this.oldestEpochToKeep();
566
605
  if (oldestEpochToKeep > 0) {
606
+ this.cleanupJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
567
607
  await this.database.deleteAllProvingJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
568
608
  this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
569
609
  }
570
610
  }
571
611
 
572
- private cleanupStaleJobs() {
612
+ private cleanupJobsOlderThanEpoch(epochNumber: EpochNumber) {
573
613
  const jobIds = Array.from(this.jobsCache.keys());
574
614
  const jobsToClean: ProvingJobId[] = [];
575
615
  for (const id of jobIds) {
576
616
  const job = this.jobsCache.get(id)!;
577
- if (this.isJobStale(job)) {
617
+ if (job.epochNumber < epochNumber) {
578
618
  jobsToClean.push(id);
579
619
  }
580
620
  }
@@ -598,10 +638,26 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
598
638
  const now = this.msTimeSource();
599
639
  const msSinceLastUpdate = now - metadata.lastUpdatedAt;
600
640
  if (msSinceLastUpdate >= this.jobTimeoutMs) {
601
- this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, { provingJobId: id });
602
641
  this.inProgress.delete(id);
603
- this.enqueueJobInternal(item);
604
642
  this.instrumentation.incTimedOutJobs(item.type);
643
+
644
+ const retries = this.retries.get(id) ?? 0;
645
+ if (retries + 1 < this.maxRetries && !this.isJobStale(item)) {
646
+ this.logger.warn(`Proving job id=${id} timed out. Re-enqueueing (retry ${retries + 1}/${this.maxRetries}).`, {
647
+ provingJobId: id,
648
+ });
649
+ this.retries.set(id, retries + 1);
650
+ this.enqueueJobInternal(item);
651
+ } else {
652
+ this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
653
+ provingJobId: id,
654
+ });
655
+ const result: ProvingJobSettledResult = { status: 'rejected', reason: 'Timed out' };
656
+ this.resultsCache.set(id, result);
657
+ this.promises.get(id)?.resolve(result);
658
+ this.completedJobNotifications.push(id);
659
+ this.instrumentation.incRejectedJobs(item.type);
660
+ }
605
661
  }
606
662
  }
607
663
  }
@@ -154,7 +154,7 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
154
154
  const db = await openVersionedStoreAt(
155
155
  fullDirectory,
156
156
  SingleEpochDatabase.SCHEMA_VERSION,
157
- config.l1Contracts.rollupAddress,
157
+ config.rollupAddress,
158
158
  config.dataStoreMapSizeKb,
159
159
  );
160
160
  const epochDb = new SingleEpochDatabase(db);
@@ -222,7 +222,7 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
222
222
  const db = await openVersionedStoreAt(
223
223
  newEpochDirectory,
224
224
  SingleEpochDatabase.SCHEMA_VERSION,
225
- this.config.l1Contracts.rollupAddress,
225
+ this.config.rollupAddress,
226
226
  this.config.dataStoreMapSizeKb,
227
227
  );
228
228
  epochDb = new SingleEpochDatabase(db);
@@ -18,6 +18,7 @@ export class ProvingBrokerInstrumentation {
18
18
  private activeJobs: ObservableGauge;
19
19
  private resolvedJobs: UpDownCounter;
20
20
  private rejectedJobs: UpDownCounter;
21
+ private abortedJobs: UpDownCounter;
21
22
  private timedOutJobs: UpDownCounter;
22
23
  private cachedJobs: UpDownCounter;
23
24
  private totalJobs: UpDownCounter;
@@ -39,6 +40,8 @@ export class ProvingBrokerInstrumentation {
39
40
 
40
41
  this.rejectedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_REJECTED_JOBS, provingJobAttrs);
41
42
 
43
+ this.abortedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_ABORTED_JOBS, provingJobAttrs);
44
+
42
45
  this.retriedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RETRIED_JOBS, provingJobAttrs);
43
46
 
44
47
  this.timedOutJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, provingJobAttrs);
@@ -72,6 +75,12 @@ export class ProvingBrokerInstrumentation {
72
75
  });
73
76
  }
74
77
 
78
+ incAbortedJobs(proofType: ProvingRequestType) {
79
+ this.abortedJobs.add(1, {
80
+ [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
81
+ });
82
+ }
83
+
75
84
  incRetriedJobs(proofType: ProvingRequestType) {
76
85
  this.retriedJobs.add(1, {
77
86
  [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
@@ -1,9 +1,11 @@
1
+ import { EpochNumberSchema } from '@aztec/foundation/branded-types';
1
2
  import { createSafeJsonRpcClient } from '@aztec/foundation/json-rpc/client';
2
3
  import {
3
4
  type GetProvingJobResponse,
4
5
  ProofUri,
5
6
  ProvingJob,
6
7
  type ProvingJobBroker,
8
+ type ProvingJobBrokerDebug,
7
9
  type ProvingJobConsumer,
8
10
  ProvingJobId,
9
11
  type ProvingJobProducer,
@@ -16,6 +18,15 @@ import { makeTracedFetch } from '@aztec/telemetry-client';
16
18
 
17
19
  import { z } from 'zod';
18
20
 
21
+ /** Indefinite backoff for broker communication: 1, 1, 1, 2, 4, 4, 4, ... seconds. */
22
+ export function* proverBrokerBackoff() {
23
+ const v = [1, 1, 1, 2, 4];
24
+ let i = 0;
25
+ while (true) {
26
+ yield v[Math.min(i++, v.length - 1)];
27
+ }
28
+ }
29
+
19
30
  const ProvingJobFilterSchema = z.object({
20
31
  allowList: z.array(z.nativeEnum(ProvingRequestType)),
21
32
  });
@@ -26,26 +37,29 @@ const GetProvingJobResponse = z.object({
26
37
  });
27
38
 
28
39
  export const ProvingJobProducerSchema: ApiSchemaFor<ProvingJobProducer> = {
29
- enqueueProvingJob: z.function().args(ProvingJob).returns(ProvingJobStatus),
30
- getProvingJobStatus: z.function().args(ProvingJobId).returns(ProvingJobStatus),
31
- cancelProvingJob: z.function().args(ProvingJobId).returns(z.void()),
32
- getCompletedJobs: z.function().args(z.array(ProvingJobId)).returns(z.array(ProvingJobId)),
40
+ enqueueProvingJob: z.function({ input: z.tuple([ProvingJob]), output: ProvingJobStatus }),
41
+ getProvingJobStatus: z.function({ input: z.tuple([ProvingJobId]), output: ProvingJobStatus }),
42
+ cancelProvingJob: z.function({ input: z.tuple([ProvingJobId]), output: z.void() }),
43
+ getCompletedJobs: z.function({ input: z.tuple([z.array(ProvingJobId)]), output: z.array(ProvingJobId) }),
33
44
  };
34
45
 
35
46
  export const ProvingJobConsumerSchema: ApiSchemaFor<ProvingJobConsumer> = {
36
- getProvingJob: z.function().args(optional(ProvingJobFilterSchema)).returns(GetProvingJobResponse.optional()),
37
- reportProvingJobError: z
38
- .function()
39
- .args(ProvingJobId, z.string(), optional(z.boolean()), optional(ProvingJobFilterSchema))
40
- .returns(GetProvingJobResponse.optional()),
41
- reportProvingJobProgress: z
42
- .function()
43
- .args(ProvingJobId, z.number(), optional(ProvingJobFilterSchema))
44
- .returns(GetProvingJobResponse.optional()),
45
- reportProvingJobSuccess: z
46
- .function()
47
- .args(ProvingJobId, ProofUri, optional(ProvingJobFilterSchema))
48
- .returns(GetProvingJobResponse.optional()),
47
+ getProvingJob: z.function({
48
+ input: z.tuple([optional(ProvingJobFilterSchema)]),
49
+ output: GetProvingJobResponse.optional(),
50
+ }),
51
+ reportProvingJobError: z.function({
52
+ input: z.tuple([ProvingJobId, z.string(), optional(z.boolean()), optional(ProvingJobFilterSchema)]),
53
+ output: GetProvingJobResponse.optional(),
54
+ }),
55
+ reportProvingJobProgress: z.function({
56
+ input: z.tuple([ProvingJobId, z.number(), optional(ProvingJobFilterSchema)]),
57
+ output: GetProvingJobResponse.optional(),
58
+ }),
59
+ reportProvingJobSuccess: z.function({
60
+ input: z.tuple([ProvingJobId, ProofUri, optional(ProvingJobFilterSchema)]),
61
+ output: GetProvingJobResponse.optional(),
62
+ }),
49
63
  };
50
64
 
51
65
  export const ProvingJobBrokerSchema: ApiSchemaFor<ProvingJobBroker> = {
@@ -53,10 +67,22 @@ export const ProvingJobBrokerSchema: ApiSchemaFor<ProvingJobBroker> = {
53
67
  ...ProvingJobProducerSchema,
54
68
  };
55
69
 
70
+ export const ProvingJobBrokerDebugSchema: ApiSchemaFor<ProvingJobBrokerDebug> = {
71
+ replayProvingJob: z.function({
72
+ input: z.tuple([ProvingJobId, z.nativeEnum(ProvingRequestType), EpochNumberSchema, ProofUri]),
73
+ output: ProvingJobStatus,
74
+ }),
75
+ };
76
+
77
+ export const ProvingJobBrokerSchemaWithDebug: ApiSchemaFor<ProvingJobBroker & ProvingJobBrokerDebug> = {
78
+ ...ProvingJobBrokerSchema,
79
+ ...ProvingJobBrokerDebugSchema,
80
+ };
81
+
56
82
  export function createProvingJobBrokerClient(
57
83
  url: string,
58
84
  versions: Partial<ComponentsVersions>,
59
- fetch = makeTracedFetch([1, 2, 3], false),
85
+ fetch = makeTracedFetch(proverBrokerBackoff, false),
60
86
  ): ProvingJobBroker {
61
87
  return createSafeJsonRpcClient(url, ProvingJobBrokerSchema, {
62
88
  namespaceMethods: 'proverBroker',
@@ -68,7 +94,7 @@ export function createProvingJobBrokerClient(
68
94
  export function createProvingJobProducerClient(
69
95
  url: string,
70
96
  versions: Partial<ComponentsVersions>,
71
- fetch = makeTracedFetch([1, 2, 3], false),
97
+ fetch = makeTracedFetch(proverBrokerBackoff, false),
72
98
  ): ProvingJobProducer {
73
99
  return createSafeJsonRpcClient(url, ProvingJobProducerSchema, {
74
100
  namespaceMethods: 'provingJobProducer',
@@ -80,7 +106,7 @@ export function createProvingJobProducerClient(
80
106
  export function createProvingJobConsumerClient(
81
107
  url: string,
82
108
  versions: Partial<ComponentsVersions>,
83
- fetch = makeTracedFetch([1, 2, 3], false),
109
+ fetch = makeTracedFetch(proverBrokerBackoff, false),
84
110
  ): ProvingJobConsumer {
85
111
  return createSafeJsonRpcClient(url, ProvingJobConsumerSchema, {
86
112
  namespaceMethods: 'provingJobConsumer',
@@ -1,14 +0,0 @@
1
- import { type ProofUri, type ProvingJobId, type ProvingJobInputs, type ProvingJobInputsMap, type ProvingJobResult, type ProvingJobResultsMap } from '@aztec/stdlib/interfaces/server';
2
- import { ProvingRequestType } from '@aztec/stdlib/proofs';
3
- import type { ProofStore } from './proof_store.js';
4
- export declare class GoogleCloudStorageProofStore implements ProofStore {
5
- private readonly bucketName;
6
- private readonly path;
7
- private readonly storage;
8
- constructor(bucketName: string, path: string);
9
- saveProofInput<T extends ProvingRequestType>(id: ProvingJobId, type: T, inputs: ProvingJobInputsMap[T]): Promise<ProofUri>;
10
- saveProofOutput<T extends ProvingRequestType>(_id: ProvingJobId, _type: T, _result: ProvingJobResultsMap[T]): Promise<ProofUri>;
11
- getProofInput(uri: ProofUri): Promise<ProvingJobInputs>;
12
- getProofOutput(_uri: ProofUri): Promise<ProvingJobResult>;
13
- }
14
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2NzX3Byb29mX3N0b3JlLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvb2Zfc3RvcmUvZ2NzX3Byb29mX3N0b3JlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFDTCxLQUFLLFFBQVEsRUFDYixLQUFLLFlBQVksRUFDakIsS0FBSyxnQkFBZ0IsRUFDckIsS0FBSyxtQkFBbUIsRUFDeEIsS0FBSyxnQkFBZ0IsRUFDckIsS0FBSyxvQkFBb0IsRUFFMUIsTUFBTSxpQ0FBaUMsQ0FBQztBQUN6QyxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUsxRCxPQUFPLEtBQUssRUFBRSxVQUFVLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUtuRCxxQkFBYSw0QkFBNkIsWUFBVyxVQUFVO0lBSTNELE9BQU8sQ0FBQyxRQUFRLENBQUMsVUFBVTtJQUMzQixPQUFPLENBQUMsUUFBUSxDQUFDLElBQUk7SUFKdkIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQVU7SUFFbEMsWUFDbUIsVUFBVSxFQUFFLE1BQU0sRUFDbEIsSUFBSSxFQUFFLE1BQU0sRUFHOUI7SUFFWSxjQUFjLENBQUMsQ0FBQyxTQUFTLGtCQUFrQixFQUN0RCxFQUFFLEVBQUUsWUFBWSxFQUNoQixJQUFJLEVBQUUsQ0FBQyxFQUNQLE1BQU0sRUFBRSxtQkFBbUIsQ0FBQyxDQUFDLENBQUMsR0FDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxDQUtuQjtJQUVELGVBQWUsQ0FBQyxDQUFDLFNBQVMsa0JBQWtCLEVBQzFDLEdBQUcsRUFBRSxZQUFZLEVBQ2pCLEtBQUssRUFBRSxDQUFDLEVBQ1IsT0FBTyxFQUFFLG9CQUFvQixDQUFDLENBQUMsQ0FBQyxHQUMvQixPQUFPLENBQUMsUUFBUSxDQUFDLENBRW5CO0lBRVksYUFBYSxDQUFDLEdBQUcsRUFBRSxRQUFRLEdBQUcsT0FBTyxDQUFDLGdCQUFnQixDQUFDLENBc0JuRTtJQUVELGNBQWMsQ0FBQyxJQUFJLEVBQUUsUUFBUSxHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxDQUV4RDtDQUNGIn0=
@@ -1 +0,0 @@
1
- {"version":3,"file":"gcs_proof_store.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proof_store/gcs_proof_store.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACrB,KAAK,oBAAoB,EAE1B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAK1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAKnD,qBAAa,4BAA6B,YAAW,UAAU;IAI3D,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAJvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAElC,YACmB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EAG9B;IAEY,cAAc,CAAC,CAAC,SAAS,kBAAkB,EACtD,EAAE,EAAE,YAAY,EAChB,IAAI,EAAE,CAAC,EACP,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAC7B,OAAO,CAAC,QAAQ,CAAC,CAKnB;IAED,eAAe,CAAC,CAAC,SAAS,kBAAkB,EAC1C,GAAG,EAAE,YAAY,EACjB,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,QAAQ,CAAC,CAEnB;IAEY,aAAa,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAsBnE;IAED,cAAc,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAExD;CACF"}
@@ -1,52 +0,0 @@
1
- import { getProvingJobInputClassFor } from '@aztec/stdlib/interfaces/server';
2
- import { ProvingRequestType } from '@aztec/stdlib/proofs';
3
- import { Storage } from '@google-cloud/storage';
4
- import { join } from 'path';
5
- const INPUTS_PATH = 'inputs';
6
- // REFACTOR(#13067): Use the stdlib/file-store instead of referencing google-cloud-storage directly.
7
- export class GoogleCloudStorageProofStore {
8
- bucketName;
9
- path;
10
- storage;
11
- constructor(bucketName, path){
12
- this.bucketName = bucketName;
13
- this.path = path;
14
- this.storage = new Storage();
15
- }
16
- async saveProofInput(id, type, inputs) {
17
- const path = join(this.path, INPUTS_PATH, ProvingRequestType[type], id);
18
- const file = this.storage.bucket(this.bucketName).file(path);
19
- await file.save(inputs.toBuffer());
20
- return file.cloudStorageURI.toString();
21
- }
22
- saveProofOutput(_id, _type, _result) {
23
- throw new Error('Not implemented');
24
- }
25
- async getProofInput(uri) {
26
- try {
27
- const url = new URL(uri);
28
- const bucket = this.storage.bucket(url.host);
29
- const path = url.pathname.replace(/^\/+/, '');
30
- const file = bucket.file(path);
31
- if (!await file.exists()) {
32
- throw new Error(`File at ${uri} does not exist`);
33
- }
34
- const typeString = path.split('/').at(-2);
35
- const type = typeString ? ProvingRequestType[typeString] : undefined;
36
- if (type === undefined) {
37
- throw new Error(`Unrecognized proof type ${type} in path ${path}`);
38
- }
39
- const contents = await file.download();
40
- const inputs = getProvingJobInputClassFor(type).fromBuffer(contents[0]);
41
- return {
42
- inputs,
43
- type
44
- };
45
- } catch (err) {
46
- throw new Error(`Error getting proof input at ${uri}: ${err}`);
47
- }
48
- }
49
- getProofOutput(_uri) {
50
- throw new Error('Not implemented');
51
- }
52
- }
@@ -1,76 +0,0 @@
1
- import {
2
- type ProofUri,
3
- type ProvingJobId,
4
- type ProvingJobInputs,
5
- type ProvingJobInputsMap,
6
- type ProvingJobResult,
7
- type ProvingJobResultsMap,
8
- getProvingJobInputClassFor,
9
- } from '@aztec/stdlib/interfaces/server';
10
- import { ProvingRequestType } from '@aztec/stdlib/proofs';
11
-
12
- import { Storage } from '@google-cloud/storage';
13
- import { join } from 'path';
14
-
15
- import type { ProofStore } from './proof_store.js';
16
-
17
- const INPUTS_PATH = 'inputs';
18
-
19
- // REFACTOR(#13067): Use the stdlib/file-store instead of referencing google-cloud-storage directly.
20
- export class GoogleCloudStorageProofStore implements ProofStore {
21
- private readonly storage: Storage;
22
-
23
- constructor(
24
- private readonly bucketName: string,
25
- private readonly path: string,
26
- ) {
27
- this.storage = new Storage();
28
- }
29
-
30
- public async saveProofInput<T extends ProvingRequestType>(
31
- id: ProvingJobId,
32
- type: T,
33
- inputs: ProvingJobInputsMap[T],
34
- ): Promise<ProofUri> {
35
- const path = join(this.path, INPUTS_PATH, ProvingRequestType[type], id);
36
- const file = this.storage.bucket(this.bucketName).file(path);
37
- await file.save(inputs.toBuffer());
38
- return file.cloudStorageURI.toString() as ProofUri;
39
- }
40
-
41
- saveProofOutput<T extends ProvingRequestType>(
42
- _id: ProvingJobId,
43
- _type: T,
44
- _result: ProvingJobResultsMap[T],
45
- ): Promise<ProofUri> {
46
- throw new Error('Not implemented');
47
- }
48
-
49
- public async getProofInput(uri: ProofUri): Promise<ProvingJobInputs> {
50
- try {
51
- const url = new URL(uri);
52
- const bucket = this.storage.bucket(url.host);
53
- const path = url.pathname.replace(/^\/+/, '');
54
- const file = bucket.file(path);
55
- if (!(await file.exists())) {
56
- throw new Error(`File at ${uri} does not exist`);
57
- }
58
-
59
- const typeString = path.split('/').at(-2);
60
- const type = typeString ? ProvingRequestType[typeString as keyof typeof ProvingRequestType] : undefined;
61
- if (type === undefined) {
62
- throw new Error(`Unrecognized proof type ${type} in path ${path}`);
63
- }
64
-
65
- const contents = await file.download();
66
- const inputs = getProvingJobInputClassFor(type).fromBuffer(contents[0]);
67
- return { inputs, type } as ProvingJobInputs;
68
- } catch (err) {
69
- throw new Error(`Error getting proof input at ${uri}: ${err}`);
70
- }
71
- }
72
-
73
- getProofOutput(_uri: ProofUri): Promise<ProvingJobResult> {
74
- throw new Error('Not implemented');
75
- }
76
- }