@aztec/prover-client 0.0.1-commit.9b94fc1 → 0.0.1-commit.9badcec54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/dest/config.d.ts +2 -2
  2. package/dest/config.d.ts.map +1 -1
  3. package/dest/config.js +13 -3
  4. package/dest/light/index.d.ts +2 -0
  5. package/dest/light/index.d.ts.map +1 -0
  6. package/dest/light/index.js +1 -0
  7. package/dest/light/lightweight_checkpoint_builder.d.ts +38 -14
  8. package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -1
  9. package/dest/light/lightweight_checkpoint_builder.js +148 -32
  10. package/dest/mocks/fixtures.d.ts +1 -1
  11. package/dest/mocks/fixtures.d.ts.map +1 -1
  12. package/dest/mocks/fixtures.js +6 -5
  13. package/dest/mocks/test_context.d.ts +5 -3
  14. package/dest/mocks/test_context.d.ts.map +1 -1
  15. package/dest/mocks/test_context.js +40 -18
  16. package/dest/orchestrator/block-building-helpers.d.ts +6 -6
  17. package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
  18. package/dest/orchestrator/block-building-helpers.js +7 -6
  19. package/dest/orchestrator/block-proving-state.d.ts +8 -4
  20. package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
  21. package/dest/orchestrator/block-proving-state.js +8 -1
  22. package/dest/orchestrator/checkpoint-proving-state.d.ts +22 -8
  23. package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -1
  24. package/dest/orchestrator/checkpoint-proving-state.js +43 -8
  25. package/dest/orchestrator/epoch-proving-state.d.ts +10 -9
  26. package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -1
  27. package/dest/orchestrator/epoch-proving-state.js +38 -2
  28. package/dest/orchestrator/orchestrator.d.ts +25 -8
  29. package/dest/orchestrator/orchestrator.d.ts.map +1 -1
  30. package/dest/orchestrator/orchestrator.js +568 -158
  31. package/dest/orchestrator/orchestrator_metrics.d.ts +1 -3
  32. package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -1
  33. package/dest/orchestrator/orchestrator_metrics.js +2 -15
  34. package/dest/orchestrator/tx-proving-state.d.ts +6 -5
  35. package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
  36. package/dest/orchestrator/tx-proving-state.js +8 -8
  37. package/dest/prover-client/factory.d.ts +3 -3
  38. package/dest/prover-client/factory.d.ts.map +1 -1
  39. package/dest/prover-client/prover-client.d.ts +6 -6
  40. package/dest/prover-client/prover-client.d.ts.map +1 -1
  41. package/dest/prover-client/prover-client.js +15 -10
  42. package/dest/prover-client/server-epoch-prover.d.ts +5 -5
  43. package/dest/prover-client/server-epoch-prover.d.ts.map +1 -1
  44. package/dest/proving_broker/broker_prover_facade.d.ts +7 -5
  45. package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
  46. package/dest/proving_broker/broker_prover_facade.js +18 -31
  47. package/dest/proving_broker/config.d.ts +15 -3
  48. package/dest/proving_broker/config.d.ts.map +1 -1
  49. package/dest/proving_broker/config.js +22 -5
  50. package/dest/proving_broker/fixtures.js +1 -1
  51. package/dest/proving_broker/proof_store/factory.d.ts +2 -5
  52. package/dest/proving_broker/proof_store/factory.d.ts.map +1 -1
  53. package/dest/proving_broker/proof_store/factory.js +7 -30
  54. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts +18 -0
  55. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts.map +1 -0
  56. package/dest/proving_broker/proof_store/file_store_proof_store.js +60 -0
  57. package/dest/proving_broker/proof_store/index.d.ts +2 -2
  58. package/dest/proving_broker/proof_store/index.d.ts.map +1 -1
  59. package/dest/proving_broker/proof_store/index.js +1 -1
  60. package/dest/proving_broker/proving_agent.d.ts +5 -9
  61. package/dest/proving_broker/proving_agent.d.ts.map +1 -1
  62. package/dest/proving_broker/proving_agent.js +4 -19
  63. package/dest/proving_broker/proving_broker.d.ts +7 -4
  64. package/dest/proving_broker/proving_broker.d.ts.map +1 -1
  65. package/dest/proving_broker/proving_broker.js +60 -18
  66. package/dest/proving_broker/proving_broker_database/persisted.d.ts +3 -2
  67. package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
  68. package/dest/proving_broker/proving_broker_database/persisted.js +389 -1
  69. package/dest/proving_broker/proving_broker_instrumentation.d.ts +3 -1
  70. package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
  71. package/dest/proving_broker/proving_broker_instrumentation.js +22 -35
  72. package/dest/proving_broker/proving_job_controller.d.ts +4 -3
  73. package/dest/proving_broker/proving_job_controller.d.ts.map +1 -1
  74. package/dest/proving_broker/proving_job_controller.js +8 -6
  75. package/dest/proving_broker/rpc.d.ts +6 -2
  76. package/dest/proving_broker/rpc.d.ts.map +1 -1
  77. package/dest/proving_broker/rpc.js +24 -15
  78. package/dest/test/mock_proof_store.d.ts +3 -3
  79. package/dest/test/mock_proof_store.d.ts.map +1 -1
  80. package/dest/test/mock_prover.d.ts +5 -5
  81. package/dest/test/mock_prover.d.ts.map +1 -1
  82. package/dest/test/mock_prover.js +4 -4
  83. package/package.json +22 -22
  84. package/src/config.ts +14 -3
  85. package/src/light/index.ts +1 -0
  86. package/src/light/lightweight_checkpoint_builder.ts +215 -45
  87. package/src/mocks/fixtures.ts +6 -5
  88. package/src/mocks/test_context.ts +36 -19
  89. package/src/orchestrator/block-building-helpers.ts +7 -6
  90. package/src/orchestrator/block-proving-state.ts +12 -2
  91. package/src/orchestrator/checkpoint-proving-state.ts +60 -12
  92. package/src/orchestrator/epoch-proving-state.ts +66 -13
  93. package/src/orchestrator/orchestrator.ts +173 -147
  94. package/src/orchestrator/orchestrator_metrics.ts +2 -25
  95. package/src/orchestrator/tx-proving-state.ts +10 -14
  96. package/src/prover-client/factory.ts +6 -2
  97. package/src/prover-client/prover-client.ts +33 -24
  98. package/src/prover-client/server-epoch-prover.ts +4 -4
  99. package/src/proving_broker/broker_prover_facade.ts +26 -36
  100. package/src/proving_broker/config.ts +25 -3
  101. package/src/proving_broker/fixtures.ts +1 -1
  102. package/src/proving_broker/proof_store/factory.ts +10 -32
  103. package/src/proving_broker/proof_store/file_store_proof_store.ts +78 -0
  104. package/src/proving_broker/proof_store/index.ts +1 -1
  105. package/src/proving_broker/proving_agent.ts +6 -19
  106. package/src/proving_broker/proving_broker.ts +59 -13
  107. package/src/proving_broker/proving_broker_database/persisted.ts +15 -1
  108. package/src/proving_broker/proving_broker_instrumentation.ts +23 -35
  109. package/src/proving_broker/proving_job_controller.ts +11 -6
  110. package/src/proving_broker/rpc.ts +26 -3
  111. package/src/test/mock_prover.ts +2 -14
  112. package/dest/block-factory/index.d.ts +0 -2
  113. package/dest/block-factory/index.d.ts.map +0 -1
  114. package/dest/block-factory/index.js +0 -1
  115. package/dest/block-factory/light.d.ts +0 -38
  116. package/dest/block-factory/light.d.ts.map +0 -1
  117. package/dest/block-factory/light.js +0 -108
  118. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +0 -14
  119. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +0 -1
  120. package/dest/proving_broker/proof_store/gcs_proof_store.js +0 -52
  121. package/dest/proving_broker/proving_agent_instrumentation.d.ts +0 -8
  122. package/dest/proving_broker/proving_agent_instrumentation.d.ts.map +0 -1
  123. package/dest/proving_broker/proving_agent_instrumentation.js +0 -16
  124. package/src/block-factory/index.ts +0 -1
  125. package/src/block-factory/light.ts +0 -137
  126. package/src/proving_broker/proof_store/gcs_proof_store.ts +0 -76
  127. package/src/proving_broker/proving_agent_instrumentation.ts +0 -21
@@ -1,5 +1,5 @@
1
1
  import { AbortError } from '@aztec/foundation/error';
2
- import { createLogger } from '@aztec/foundation/log';
2
+ import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
3
3
  import { RunningPromise } from '@aztec/foundation/running-promise';
4
4
  import { truncate } from '@aztec/foundation/string';
5
5
  import { ProvingError } from '@aztec/stdlib/errors';
@@ -13,27 +13,17 @@ import type {
13
13
  ServerCircuitProver,
14
14
  } from '@aztec/stdlib/interfaces/server';
15
15
  import { ProvingRequestType } from '@aztec/stdlib/proofs';
16
- import {
17
- type TelemetryClient,
18
- type Traceable,
19
- type Tracer,
20
- getTelemetryClient,
21
- trackSpan,
22
- } from '@aztec/telemetry-client';
23
16
 
24
17
  import type { ProofStore } from './proof_store/index.js';
25
- import { ProvingAgentInstrumentation } from './proving_agent_instrumentation.js';
26
18
  import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_controller.js';
27
19
 
28
20
  /**
29
21
  * A helper class that encapsulates a circuit prover and connects it to a job source.
30
22
  */
31
- export class ProvingAgent implements Traceable {
23
+ export class ProvingAgent {
32
24
  private currentJobController?: ProvingJobController;
33
25
  private runningPromise: RunningPromise;
34
- private instrumentation: ProvingAgentInstrumentation;
35
-
36
- public readonly tracer: Tracer;
26
+ private log: Logger;
37
27
 
38
28
  constructor(
39
29
  /** The source of proving jobs */
@@ -46,12 +36,9 @@ export class ProvingAgent implements Traceable {
46
36
  private proofAllowList: Array<ProvingRequestType> = [],
47
37
  /** How long to wait between jobs */
48
38
  private pollIntervalMs = 1000,
49
- /** A telemetry client through which to emit metrics */
50
- client: TelemetryClient = getTelemetryClient(),
51
- private log = createLogger('prover-client:proving-agent'),
39
+ bindings?: LoggerBindings,
52
40
  ) {
53
- this.tracer = client.getTracer('ProvingAgent');
54
- this.instrumentation = new ProvingAgentInstrumentation(client);
41
+ this.log = createLogger('prover-client:proving-agent', bindings);
55
42
  this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollIntervalMs);
56
43
  }
57
44
 
@@ -85,7 +72,6 @@ export class ProvingAgent implements Traceable {
85
72
  return this.runningPromise.isRunning() ? { status: 'running' } : { status: 'stopped' };
86
73
  }
87
74
 
88
- @trackSpan('ProvingAgent.safeWork')
89
75
  private async work() {
90
76
  // every tick we need to take one of the following actions:
91
77
  // 1. send a hearbeat to the broker that we're working on some job
@@ -175,6 +161,7 @@ export class ProvingAgent implements Traceable {
175
161
  // no need to await this here. The controller will stay alive (in DONE state) until the result is send to the broker
176
162
  void this.runningPromise.trigger();
177
163
  },
164
+ this.log.getBindings(),
178
165
  );
179
166
 
180
167
  if (abortedProofJobId) {
@@ -7,6 +7,7 @@ import {
7
7
  type GetProvingJobResponse,
8
8
  type ProofUri,
9
9
  type ProvingJob,
10
+ type ProvingJobBrokerDebug,
10
11
  type ProvingJobConsumer,
11
12
  type ProvingJobFilter,
12
13
  type ProvingJobId,
@@ -16,13 +17,7 @@ import {
16
17
  tryStop,
17
18
  } from '@aztec/stdlib/interfaces/server';
18
19
  import { ProvingRequestType } from '@aztec/stdlib/proofs';
19
- import {
20
- type TelemetryClient,
21
- type Traceable,
22
- type Tracer,
23
- getTelemetryClient,
24
- trackSpan,
25
- } from '@aztec/telemetry-client';
20
+ import { type TelemetryClient, type Traceable, type Tracer, getTelemetryClient } from '@aztec/telemetry-client';
26
21
 
27
22
  import assert from 'assert';
28
23
 
@@ -42,7 +37,7 @@ type EnqueuedProvingJob = Pick<ProvingJob, 'id' | 'epochNumber'>;
42
37
  * A broker that manages proof requests and distributes them to workers based on their priority.
43
38
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
44
39
  */
45
- export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable {
40
+ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, ProvingJobBrokerDebug, Traceable {
46
41
  private queues: ProvingQueues = {
47
42
  [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
48
43
  [ProvingRequestType.PUBLIC_CHONK_VERIFIER]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
@@ -120,6 +115,8 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
120
115
 
121
116
  private started = false;
122
117
 
118
+ private debugReplayEnabled: boolean;
119
+
123
120
  public constructor(
124
121
  private database: ProvingBrokerDatabase,
125
122
  {
@@ -127,6 +124,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
127
124
  proverBrokerPollIntervalMs,
128
125
  proverBrokerJobMaxRetries,
129
126
  proverBrokerMaxEpochsToKeepResultsFor,
127
+ proverBrokerDebugReplayEnabled,
130
128
  }: Required<
131
129
  Pick<
132
130
  ProverBrokerConfig,
@@ -134,6 +132,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
134
132
  | 'proverBrokerPollIntervalMs'
135
133
  | 'proverBrokerJobMaxRetries'
136
134
  | 'proverBrokerMaxEpochsToKeepResultsFor'
135
+ | 'proverBrokerDebugReplayEnabled'
137
136
  >
138
137
  > = defaultProverBrokerConfig,
139
138
  client: TelemetryClient = getTelemetryClient(),
@@ -145,6 +144,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
145
144
  this.jobTimeoutMs = proverBrokerJobTimeoutMs!;
146
145
  this.maxRetries = proverBrokerJobMaxRetries!;
147
146
  this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor!;
147
+ this.debugReplayEnabled = proverBrokerDebugReplayEnabled ?? false;
148
148
  }
149
149
 
150
150
  private measureQueueDepth: MonitorCallback = (type: ProvingRequestType) => {
@@ -247,6 +247,29 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
247
247
  return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter));
248
248
  }
249
249
 
250
+ public async replayProvingJob(
251
+ jobId: ProvingJobId,
252
+ type: ProvingRequestType,
253
+ epochNumber: EpochNumber,
254
+ inputsUri: ProofUri,
255
+ ): Promise<ProvingJobStatus> {
256
+ if (!this.debugReplayEnabled) {
257
+ throw new Error('Debug replay not enabled. Set PROVER_BROKER_DEBUG_REPLAY_ENABLED=true');
258
+ }
259
+
260
+ this.logger.info(`Replaying proving job`, { provingJobId: jobId, epochNumber, inputsUri });
261
+
262
+ // Clear existing state and enqueue
263
+ this.cleanUpProvingJobState([jobId]);
264
+
265
+ const job: ProvingJob = { id: jobId, type, epochNumber, inputsUri };
266
+ this.jobsCache.set(jobId, job);
267
+ await this.database.addProvingJob(job);
268
+ this.enqueueJobInternal(job);
269
+
270
+ return { status: 'in-queue' };
271
+ }
272
+
250
273
  async #enqueueProvingJob(job: ProvingJob): Promise<ProvingJobStatus> {
251
274
  // We return the job status at the start of this call
252
275
  const jobStatus = this.#getProvingJobStatus(job.id);
@@ -291,7 +314,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
291
314
  // notify listeners of the cancellation
292
315
  if (!this.resultsCache.has(id)) {
293
316
  this.logger.info(`Cancelling job id=${id}`, { provingJobId: id });
294
- await this.#reportProvingJobError(id, 'Aborted', false);
317
+ await this.#reportProvingJobError(id, 'Aborted', false, undefined, true);
295
318
  }
296
319
  }
297
320
 
@@ -302,6 +325,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
302
325
  this.resultsCache.delete(id);
303
326
  this.inProgress.delete(id);
304
327
  this.retries.delete(id);
328
+ this.enqueuedAt.delete(id);
305
329
  }
306
330
  }
307
331
 
@@ -354,6 +378,8 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
354
378
  const enqueuedAt = this.enqueuedAt.get(job.id);
355
379
  if (enqueuedAt) {
356
380
  this.instrumentation.recordJobWait(job.type, enqueuedAt);
381
+ // we can clear this flag now.
382
+ this.enqueuedAt.delete(job.id);
357
383
  }
358
384
 
359
385
  return { job, time };
@@ -369,6 +395,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
369
395
  err: string,
370
396
  retry = false,
371
397
  filter?: ProvingJobFilter,
398
+ aborted = false,
372
399
  ): Promise<GetProvingJobResponse | undefined> {
373
400
  const info = this.inProgress.get(id);
374
401
  const item = this.jobsCache.get(id);
@@ -429,7 +456,11 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
429
456
  this.promises.get(id)!.resolve(result);
430
457
  this.completedJobNotifications.push(id);
431
458
 
432
- this.instrumentation.incRejectedJobs(item.type);
459
+ if (aborted) {
460
+ this.instrumentation.incAbortedJobs(item.type);
461
+ } else {
462
+ this.instrumentation.incRejectedJobs(item.type);
463
+ }
433
464
  if (info) {
434
465
  const duration = this.msTimeSource() - info.startedAt;
435
466
  this.instrumentation.recordJobDuration(item.type, duration);
@@ -562,7 +593,6 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
562
593
  return this.#getProvingJob(filter);
563
594
  }
564
595
 
565
- @trackSpan('ProvingBroker.cleanupPass')
566
596
  private async cleanupPass() {
567
597
  this.cleanupStaleJobs();
568
598
  this.reEnqueueExpiredJobs();
@@ -602,10 +632,26 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
602
632
  const now = this.msTimeSource();
603
633
  const msSinceLastUpdate = now - metadata.lastUpdatedAt;
604
634
  if (msSinceLastUpdate >= this.jobTimeoutMs) {
605
- this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, { provingJobId: id });
606
635
  this.inProgress.delete(id);
607
- this.enqueueJobInternal(item);
608
636
  this.instrumentation.incTimedOutJobs(item.type);
637
+
638
+ const retries = this.retries.get(id) ?? 0;
639
+ if (retries + 1 < this.maxRetries && !this.isJobStale(item)) {
640
+ this.logger.warn(`Proving job id=${id} timed out. Re-enqueueing (retry ${retries + 1}/${this.maxRetries}).`, {
641
+ provingJobId: id,
642
+ });
643
+ this.retries.set(id, retries + 1);
644
+ this.enqueueJobInternal(item);
645
+ } else {
646
+ this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
647
+ provingJobId: id,
648
+ });
649
+ const result: ProvingJobSettledResult = { status: 'rejected', reason: 'Timed out' };
650
+ this.resultsCache.set(id, result);
651
+ this.promises.get(id)?.resolve(result);
652
+ this.completedJobNotifications.push(id);
653
+ this.instrumentation.incRejectedJobs(item.type);
654
+ }
609
655
  }
610
656
  }
611
657
  }
@@ -11,7 +11,14 @@ import {
11
11
  ProvingJobSettledResult,
12
12
  getEpochFromProvingJobId,
13
13
  } from '@aztec/stdlib/interfaces/server';
14
- import { Attributes, LmdbMetrics, type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
14
+ import {
15
+ Attributes,
16
+ LmdbMetrics,
17
+ type TelemetryClient,
18
+ type Tracer,
19
+ getTelemetryClient,
20
+ trackSpan,
21
+ } from '@aztec/telemetry-client';
15
22
 
16
23
  import { mkdir, readdir } from 'fs/promises';
17
24
  import { join } from 'path';
@@ -78,6 +85,8 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
78
85
 
79
86
  private batchQueue: BatchQueue<ProvingJob | [ProvingJobId, ProvingJobSettledResult], number>;
80
87
 
88
+ public readonly tracer: Tracer;
89
+
81
90
  private constructor(
82
91
  private epochs: Map<number, SingleEpochDatabase>,
83
92
  private config: ProverBrokerConfig,
@@ -92,6 +101,8 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
92
101
  () => this.estimateSize(),
93
102
  );
94
103
 
104
+ this.tracer = client.getTracer('KVBrokerDatabase');
105
+
95
106
  this.batchQueue = new BatchQueue(
96
107
  (items, key) => this.commitWrites(items, key),
97
108
  config.proverBrokerBatchSize,
@@ -165,6 +176,9 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
165
176
  }
166
177
  }
167
178
 
179
+ @trackSpan('KVBrokerDatabase.deleteAllProvingJobsOlderThanEpoch', epochNumber => ({
180
+ [Attributes.EPOCH_NUMBER]: epochNumber,
181
+ }))
168
182
  async deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void> {
169
183
  const oldEpochs = Array.from(this.epochs.keys()).filter(e => e < Number(epochNumber));
170
184
  for (const old of oldEpochs) {
@@ -8,7 +8,7 @@ import {
8
8
  type ObservableResult,
9
9
  type TelemetryClient,
10
10
  type UpDownCounter,
11
- ValueType,
11
+ createUpDownCounterWithDefault,
12
12
  } from '@aztec/telemetry-client';
13
13
 
14
14
  export type MonitorCallback = (proofType: ProvingRequestType) => number;
@@ -18,6 +18,7 @@ export class ProvingBrokerInstrumentation {
18
18
  private activeJobs: ObservableGauge;
19
19
  private resolvedJobs: UpDownCounter;
20
20
  private rejectedJobs: UpDownCounter;
21
+ private abortedJobs: UpDownCounter;
21
22
  private timedOutJobs: UpDownCounter;
22
23
  private cachedJobs: UpDownCounter;
23
24
  private totalJobs: UpDownCounter;
@@ -28,49 +29,30 @@ export class ProvingBrokerInstrumentation {
28
29
  constructor(client: TelemetryClient, name = 'ProvingBroker') {
29
30
  const meter = client.getMeter(name);
30
31
 
31
- this.queueSize = meter.createObservableGauge(Metrics.PROVING_QUEUE_SIZE, {
32
- valueType: ValueType.INT,
33
- });
32
+ this.queueSize = meter.createObservableGauge(Metrics.PROVING_QUEUE_SIZE);
34
33
 
35
- this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS, {
36
- valueType: ValueType.INT,
37
- });
34
+ this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS);
38
35
 
39
- this.resolvedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RESOLVED_JOBS, {
40
- valueType: ValueType.INT,
41
- });
36
+ const provingJobTypes = Object.values(ProvingRequestType).filter(v => typeof v === 'string');
37
+ const provingJobAttrs = { [Attributes.PROVING_JOB_TYPE]: provingJobTypes };
42
38
 
43
- this.rejectedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_REJECTED_JOBS, {
44
- valueType: ValueType.INT,
45
- });
39
+ this.resolvedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RESOLVED_JOBS, provingJobAttrs);
46
40
 
47
- this.retriedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RETRIED_JOBS, {
48
- valueType: ValueType.INT,
49
- });
41
+ this.rejectedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_REJECTED_JOBS, provingJobAttrs);
50
42
 
51
- this.timedOutJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, {
52
- valueType: ValueType.INT,
53
- });
43
+ this.abortedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_ABORTED_JOBS, provingJobAttrs);
54
44
 
55
- this.cachedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_CACHED_JOBS, {
56
- valueType: ValueType.INT,
57
- });
45
+ this.retriedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RETRIED_JOBS, provingJobAttrs);
58
46
 
59
- this.totalJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TOTAL_JOBS, {
60
- valueType: ValueType.INT,
61
- });
47
+ this.timedOutJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, provingJobAttrs);
62
48
 
63
- this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT, {
64
- description: 'Records how long a job sits in the queue',
65
- unit: 'ms',
66
- valueType: ValueType.INT,
67
- });
49
+ this.cachedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_CACHED_JOBS, provingJobAttrs);
68
50
 
69
- this.jobDuration = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_DURATION, {
70
- description: 'Records how long a job takes to complete',
71
- unit: 'ms',
72
- valueType: ValueType.INT,
73
- });
51
+ this.totalJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TOTAL_JOBS, provingJobAttrs);
52
+
53
+ this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT);
54
+
55
+ this.jobDuration = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_DURATION);
74
56
  }
75
57
 
76
58
  monitorQueueDepth(fn: MonitorCallback) {
@@ -93,6 +75,12 @@ export class ProvingBrokerInstrumentation {
93
75
  });
94
76
  }
95
77
 
78
+ incAbortedJobs(proofType: ProvingRequestType) {
79
+ this.abortedJobs.add(1, {
80
+ [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
81
+ });
82
+ }
83
+
96
84
  incRetriedJobs(proofType: ProvingRequestType) {
97
85
  this.retriedJobs.add(1, {
98
86
  [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
@@ -1,7 +1,7 @@
1
1
  import { EpochNumber } from '@aztec/foundation/branded-types';
2
- import { randomBytes } from '@aztec/foundation/crypto';
2
+ import { randomBytes } from '@aztec/foundation/crypto/random';
3
3
  import { AbortError } from '@aztec/foundation/error';
4
- import { createLogger } from '@aztec/foundation/log';
4
+ import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
5
5
  import type {
6
6
  ProvingJobId,
7
7
  ProvingJobInputs,
@@ -21,6 +21,7 @@ export class ProvingJobController {
21
21
  private promise?: Promise<void>;
22
22
  private abortController = new AbortController();
23
23
  private result?: ProvingJobResultsMap[ProvingRequestType] | Error;
24
+ private log: Logger;
24
25
 
25
26
  constructor(
26
27
  private jobId: ProvingJobId,
@@ -29,8 +30,13 @@ export class ProvingJobController {
29
30
  private startedAt: number,
30
31
  private circuitProver: ServerCircuitProver,
31
32
  private onComplete: () => void,
32
- private log = createLogger('prover-client:proving-agent:job-controller-' + randomBytes(4).toString('hex')),
33
- ) {}
33
+ bindings?: LoggerBindings,
34
+ ) {
35
+ this.log = createLogger('prover-client:proving-agent:job-controller', {
36
+ instanceId: randomBytes(4).toString('hex'),
37
+ ...bindings,
38
+ });
39
+ }
34
40
 
35
41
  public start(): void {
36
42
  if (this.status !== ProvingJobControllerStatus.IDLE) {
@@ -125,8 +131,7 @@ export class ProvingJobController {
125
131
  const signal = this.abortController.signal;
126
132
  switch (type) {
127
133
  case ProvingRequestType.PUBLIC_VM: {
128
- // TODO(#14234)[Unconditional PIs validation]: Remove argument "undefined".
129
- return await this.circuitProver.getAvmProof(inputs, undefined, signal, this.epochNumber);
134
+ return await this.circuitProver.getAvmProof(inputs, signal, this.epochNumber);
130
135
  }
131
136
 
132
137
  case ProvingRequestType.PUBLIC_CHONK_VERIFIER: {
@@ -1,9 +1,11 @@
1
+ import { EpochNumberSchema } from '@aztec/foundation/branded-types';
1
2
  import { createSafeJsonRpcClient } from '@aztec/foundation/json-rpc/client';
2
3
  import {
3
4
  type GetProvingJobResponse,
4
5
  ProofUri,
5
6
  ProvingJob,
6
7
  type ProvingJobBroker,
8
+ type ProvingJobBrokerDebug,
7
9
  type ProvingJobConsumer,
8
10
  ProvingJobId,
9
11
  type ProvingJobProducer,
@@ -16,6 +18,15 @@ import { makeTracedFetch } from '@aztec/telemetry-client';
16
18
 
17
19
  import { z } from 'zod';
18
20
 
21
+ /** Indefinite backoff for broker communication: 1, 1, 1, 2, 4, 4, 4, ... seconds. */
22
+ export function* proverBrokerBackoff() {
23
+ const v = [1, 1, 1, 2, 4];
24
+ let i = 0;
25
+ while (true) {
26
+ yield v[Math.min(i++, v.length - 1)];
27
+ }
28
+ }
29
+
19
30
  const ProvingJobFilterSchema = z.object({
20
31
  allowList: z.array(z.nativeEnum(ProvingRequestType)),
21
32
  });
@@ -53,10 +64,22 @@ export const ProvingJobBrokerSchema: ApiSchemaFor<ProvingJobBroker> = {
53
64
  ...ProvingJobProducerSchema,
54
65
  };
55
66
 
67
+ export const ProvingJobBrokerDebugSchema: ApiSchemaFor<ProvingJobBrokerDebug> = {
68
+ replayProvingJob: z
69
+ .function()
70
+ .args(ProvingJobId, z.nativeEnum(ProvingRequestType), EpochNumberSchema, ProofUri)
71
+ .returns(ProvingJobStatus),
72
+ };
73
+
74
+ export const ProvingJobBrokerSchemaWithDebug: ApiSchemaFor<ProvingJobBroker & ProvingJobBrokerDebug> = {
75
+ ...ProvingJobBrokerSchema,
76
+ ...ProvingJobBrokerDebugSchema,
77
+ };
78
+
56
79
  export function createProvingJobBrokerClient(
57
80
  url: string,
58
81
  versions: Partial<ComponentsVersions>,
59
- fetch = makeTracedFetch([1, 2, 3], false),
82
+ fetch = makeTracedFetch(proverBrokerBackoff, false),
60
83
  ): ProvingJobBroker {
61
84
  return createSafeJsonRpcClient(url, ProvingJobBrokerSchema, {
62
85
  namespaceMethods: 'proverBroker',
@@ -68,7 +91,7 @@ export function createProvingJobBrokerClient(
68
91
  export function createProvingJobProducerClient(
69
92
  url: string,
70
93
  versions: Partial<ComponentsVersions>,
71
- fetch = makeTracedFetch([1, 2, 3], false),
94
+ fetch = makeTracedFetch(proverBrokerBackoff, false),
72
95
  ): ProvingJobProducer {
73
96
  return createSafeJsonRpcClient(url, ProvingJobProducerSchema, {
74
97
  namespaceMethods: 'provingJobProducer',
@@ -80,7 +103,7 @@ export function createProvingJobProducerClient(
80
103
  export function createProvingJobConsumerClient(
81
104
  url: string,
82
105
  versions: Partial<ComponentsVersions>,
83
- fetch = makeTracedFetch([1, 2, 3], false),
106
+ fetch = makeTracedFetch(proverBrokerBackoff, false),
84
107
  ): ProvingJobConsumer {
85
108
  return createSafeJsonRpcClient(url, ProvingJobConsumerSchema, {
86
109
  namespaceMethods: 'provingJobConsumer',
@@ -1,6 +1,5 @@
1
1
  import {
2
2
  AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED,
3
- AVM_V2_VERIFICATION_KEY_LENGTH_IN_FIELDS_PADDED,
4
3
  NESTED_RECURSIVE_PROOF_LENGTH,
5
4
  NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
6
5
  RECURSIVE_PROOF_LENGTH,
@@ -14,7 +13,6 @@ import {
14
13
  type ProvingJobStatus,
15
14
  type PublicInputsAndRecursiveProof,
16
15
  type ServerCircuitProver,
17
- makeProofAndVerificationKey,
18
16
  makePublicInputsAndRecursiveProof,
19
17
  } from '@aztec/stdlib/interfaces/server';
20
18
  import type { ParityBasePrivateInputs, ParityRootPrivateInputs } from '@aztec/stdlib/parity';
@@ -105,18 +103,8 @@ export class TestBroker implements ProvingJobProducer {
105
103
  export class MockProver implements ServerCircuitProver {
106
104
  constructor() {}
107
105
 
108
- getAvmProof(
109
- _inputs: AvmCircuitInputs,
110
- _skipPublicInputsValidation?: boolean, // TODO(#14234)[Unconditional PIs validation]: Remove.
111
- _signal?: AbortSignal,
112
- _epochNumber?: number,
113
- ) {
114
- return Promise.resolve(
115
- makeProofAndVerificationKey(
116
- makeEmptyRecursiveProof(AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED),
117
- VerificationKeyData.makeFake(AVM_V2_VERIFICATION_KEY_LENGTH_IN_FIELDS_PADDED),
118
- ),
119
- );
106
+ getAvmProof(_inputs: AvmCircuitInputs, _signal?: AbortSignal, _epochNumber?: number) {
107
+ return Promise.resolve(makeEmptyRecursiveProof(AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED));
120
108
  }
121
109
 
122
110
  getBaseParityProof(_inputs: ParityBasePrivateInputs, _signal?: AbortSignal, _epochNumber?: number) {
@@ -1,2 +0,0 @@
1
- export * from './light.js';
2
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9ibG9jay1mYWN0b3J5L2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGNBQWMsWUFBWSxDQUFDIn0=
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/block-factory/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC"}
@@ -1 +0,0 @@
1
- export * from './light.js';
@@ -1,38 +0,0 @@
1
- import { Fr } from '@aztec/foundation/fields';
2
- import { L2Block } from '@aztec/stdlib/block';
3
- import type { IBlockFactory, MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
4
- import { type GlobalVariables, type ProcessedTx } from '@aztec/stdlib/tx';
5
- import { type TelemetryClient } from '@aztec/telemetry-client';
6
- /**
7
- * Builds a block and its header from a set of processed tx without running any circuits.
8
- *
9
- * NOTE: the onus is ON THE CALLER to update the db that is passed in with the notes hashes, nullifiers, etc
10
- * PRIOR to calling `buildBlock`.
11
- *
12
- * Why? Because if you are, e.g. building a block in practice from TxObjects, you are using the
13
- * PublicProcessor which will do this for you as it processes transactions.
14
- *
15
- * If you haven't already inserted the side effects, e.g. because you are in a testing context, you can use the helper
16
- * function `buildBlockWithCleanDB`, which calls `insertSideEffects` for you.
17
- *
18
- * @deprecated Use LightweightCheckpointBuilder instead. This only works for one block per checkpoint.
19
- */
20
- export declare class LightweightBlockFactory implements IBlockFactory {
21
- private db;
22
- private telemetry;
23
- private globalVariables?;
24
- private l1ToL2Messages?;
25
- private txs;
26
- private readonly logger;
27
- constructor(db: MerkleTreeWriteOperations, telemetry?: TelemetryClient);
28
- startNewBlock(globalVariables: GlobalVariables, l1ToL2Messages: Fr[]): Promise<void>;
29
- addTxs(txs: ProcessedTx[]): Promise<void>;
30
- setBlockCompleted(): Promise<L2Block>;
31
- private buildBlock;
32
- }
33
- /**
34
- * Inserts the processed transactions into the DB, then creates a block.
35
- * @param db - A db fork to use for block building which WILL BE MODIFIED.
36
- */
37
- export declare function buildBlockWithCleanDB(txs: ProcessedTx[], globalVariables: GlobalVariables, l1ToL2Messages: Fr[], db: MerkleTreeWriteOperations, telemetry?: TelemetryClient): Promise<L2Block>;
38
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibGlnaHQuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9ibG9jay1mYWN0b3J5L2xpZ2h0LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUdBLE9BQU8sRUFBRSxFQUFFLEVBQUUsTUFBTSwwQkFBMEIsQ0FBQztBQUU5QyxPQUFPLEVBQUUsT0FBTyxFQUFpQixNQUFNLHFCQUFxQixDQUFDO0FBQzdELE9BQU8sS0FBSyxFQUFFLGFBQWEsRUFBRSx5QkFBeUIsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBR2hHLE9BQU8sRUFBcUIsS0FBSyxlQUFlLEVBQUUsS0FBSyxXQUFXLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUM3RixPQUFPLEVBQUUsS0FBSyxlQUFlLEVBQXNCLE1BQU0seUJBQXlCLENBQUM7QUFRbkY7Ozs7Ozs7Ozs7Ozs7R0FhRztBQUNILHFCQUFhLHVCQUF3QixZQUFXLGFBQWE7SUFRekQsT0FBTyxDQUFDLEVBQUU7SUFDVixPQUFPLENBQUMsU0FBUztJQVJuQixPQUFPLENBQUMsZUFBZSxDQUFDLENBQWtCO0lBQzFDLE9BQU8sQ0FBQyxjQUFjLENBQUMsQ0FBTztJQUM5QixPQUFPLENBQUMsR0FBRyxDQUE0QjtJQUV2QyxPQUFPLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBNkM7SUFFcEUsWUFDVSxFQUFFLEVBQUUseUJBQXlCLEVBQzdCLFNBQVMsR0FBRSxlQUFzQyxFQUN2RDtJQUVFLGFBQWEsQ0FBQyxlQUFlLEVBQUUsZUFBZSxFQUFFLGNBQWMsRUFBRSxFQUFFLEVBQUUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBT3pGO0lBRUQsTUFBTSxDQUFDLEdBQUcsRUFBRSxXQUFXLEVBQUUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBU3hDO0lBRUQsaUJBQWlCLElBQUksT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUVwQztZQUVhLFVBQVU7Q0E4Q3pCO0FBRUQ7OztHQUdHO0FBQ0gsd0JBQXNCLHFCQUFxQixDQUN6QyxHQUFHLEVBQUUsV0FBVyxFQUFFLEVBQ2xCLGVBQWUsRUFBRSxlQUFlLEVBQ2hDLGNBQWMsRUFBRSxFQUFFLEVBQUUsRUFDcEIsRUFBRSxFQUFFLHlCQUF5QixFQUM3QixTQUFTLEdBQUUsZUFBc0Msb0JBV2xEIn0=
@@ -1 +0,0 @@
1
- {"version":3,"file":"light.d.ts","sourceRoot":"","sources":["../../src/block-factory/light.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,EAAE,EAAE,MAAM,0BAA0B,CAAC;AAE9C,OAAO,EAAE,OAAO,EAAiB,MAAM,qBAAqB,CAAC;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAC;AAGhG,OAAO,EAAqB,KAAK,eAAe,EAAE,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC7F,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,yBAAyB,CAAC;AAQnF;;;;;;;;;;;;;GAaG;AACH,qBAAa,uBAAwB,YAAW,aAAa;IAQzD,OAAO,CAAC,EAAE;IACV,OAAO,CAAC,SAAS;IARnB,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,cAAc,CAAC,CAAO;IAC9B,OAAO,CAAC,GAAG,CAA4B;IAEvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6C;IAEpE,YACU,EAAE,EAAE,yBAAyB,EAC7B,SAAS,GAAE,eAAsC,EACvD;IAEE,aAAa,CAAC,eAAe,EAAE,eAAe,EAAE,cAAc,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAOzF;IAED,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CASxC;IAED,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC,CAEpC;YAEa,UAAU;CA8CzB;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,GAAG,EAAE,WAAW,EAAE,EAClB,eAAe,EAAE,eAAe,EAChC,cAAc,EAAE,EAAE,EAAE,EACpB,EAAE,EAAE,yBAAyB,EAC7B,SAAS,GAAE,eAAsC,oBAWlD"}