@aztec/prover-client 0.65.1 → 0.66.0

package/src/proving_broker/proving_broker.ts

@@ -1,29 +1,34 @@
 import {
+  type ProofUri,
+  type ProvingJob,
+  type ProvingJobConsumer,
+  type ProvingJobFilter,
+  type ProvingJobId,
+  type ProvingJobProducer,
+  type ProvingJobSettledResult,
+  type ProvingJobStatus,
   ProvingRequestType,
-  type V2ProofOutput,
-  type V2ProvingJob,
-  type V2ProvingJobId,
-  type V2ProvingJobResult,
-  type V2ProvingJobStatus,
 } from '@aztec/circuit-types';
 import { createDebugLogger } from '@aztec/foundation/log';
-import { RunningPromise } from '@aztec/foundation/promise';
+import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { PriorityMemoryQueue } from '@aztec/foundation/queue';
+import { Timer } from '@aztec/foundation/timer';
+import { type TelemetryClient } from '@aztec/telemetry-client';
 
 import assert from 'assert';
 
-import type { ProvingJobConsumer, ProvingJobFilter, ProvingJobProducer } from './proving_broker_interface.js';
-import { type ProvingJobDatabase } from './proving_job_database.js';
+import { type ProvingBrokerDatabase } from './proving_broker_database.js';
+import { type MonitorCallback, ProvingBrokerInstrumentation } from './proving_broker_instrumentation.js';
 
 type InProgressMetadata = {
-  id: V2ProvingJobId;
+  id: ProvingJobId;
   startedAt: number;
   lastUpdatedAt: number;
 };
 
 type ProofRequestBrokerConfig = {
-  timeoutIntervalSec?: number;
-  jobTimeoutSec?: number;
+  timeoutIntervalMs?: number;
+  jobTimeoutMs?: number;
   maxRetries?: number;
 };
 
@@ -33,60 +38,88 @@ type ProofRequestBrokerConfig = {
  */
 export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
   private queues: ProvingQueues = {
-    [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
-    [ProvingRequestType.TUBE_PROOF]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
-    [ProvingRequestType.PRIVATE_KERNEL_EMPTY]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
+    [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.TUBE_PROOF]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.PRIVATE_KERNEL_EMPTY]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
 
-    [ProvingRequestType.PRIVATE_BASE_ROLLUP]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
-    [ProvingRequestType.PUBLIC_BASE_ROLLUP]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
-    [ProvingRequestType.MERGE_ROLLUP]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
-    [ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
+    [ProvingRequestType.PRIVATE_BASE_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.PUBLIC_BASE_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.MERGE_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
 
-    [ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
-    [ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
-    [ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
+    [ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
 
-    [ProvingRequestType.BASE_PARITY]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
-    [ProvingRequestType.ROOT_PARITY]: new PriorityMemoryQueue<V2ProvingJob>(provingJobComparator),
+    [ProvingRequestType.BASE_PARITY]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.ROOT_PARITY]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
   };
 
   // holds a copy of the database in memory in order to quickly fulfill requests
   // this is fine because this broker is the only one that can modify the database
-  private jobsCache = new Map<V2ProvingJobId, V2ProvingJob>();
+  private jobsCache = new Map<ProvingJobId, ProvingJob>();
   // as above, but for results
-  private resultsCache = new Map<V2ProvingJobId, V2ProvingJobResult>();
+  private resultsCache = new Map<ProvingJobId, ProvingJobSettledResult>();
+
+  // tracks when each job was enqueued
+  private enqueuedAt = new Map<ProvingJobId, Timer>();
 
   // keeps track of which jobs are currently being processed
   // in the event of a crash this information is lost, but that's ok
   // the next time the broker starts it will recreate jobsCache and still
   // accept results from the workers
-  private inProgress = new Map<V2ProvingJobId, InProgressMetadata>();
+  private inProgress = new Map<ProvingJobId, InProgressMetadata>();
 
   // keep track of which proving job has been retried
-  private retries = new Map<V2ProvingJobId, number>();
+  private retries = new Map<ProvingJobId, number>();
+
+  // a map of promises that will be resolved when a job is settled
+  private promises = new Map<ProvingJobId, PromiseWithResolvers<ProvingJobSettledResult>>();
 
   private timeoutPromise: RunningPromise;
   private timeSource = () => Math.floor(Date.now() / 1000);
-  private jobTimeoutSec: number;
+  private jobTimeoutMs: number;
   private maxRetries: number;
 
+  private instrumentation: ProvingBrokerInstrumentation;
+
   public constructor(
-    private database: ProvingJobDatabase,
-    { jobTimeoutSec = 30, timeoutIntervalSec = 10, maxRetries = 3 }: ProofRequestBrokerConfig = {},
-    private logger = createDebugLogger('aztec:prover-client:proof-request-broker'),
+    private database: ProvingBrokerDatabase,
+    client: TelemetryClient,
+    { jobTimeoutMs = 30_000, timeoutIntervalMs = 10_000, maxRetries = 3 }: ProofRequestBrokerConfig = {},
+    private logger = createDebugLogger('aztec:prover-client:proving-broker'),
   ) {
-    this.timeoutPromise = new RunningPromise(this.timeoutCheck, timeoutIntervalSec * 1000);
-    this.jobTimeoutSec = jobTimeoutSec;
+    this.instrumentation = new ProvingBrokerInstrumentation(client);
+    this.timeoutPromise = new RunningPromise(this.timeoutCheck, timeoutIntervalMs);
+    this.jobTimeoutMs = jobTimeoutMs;
     this.maxRetries = maxRetries;
   }
 
-  // eslint-disable-next-line require-await
-  public async start(): Promise<void> {
+  private measureQueueDepth: MonitorCallback = (type: ProvingRequestType) => {
+    return this.queues[type].length();
+  };
+
+  private countActiveJobs: MonitorCallback = (type: ProvingRequestType) => {
+    let count = 0;
+    for (const { id } of this.inProgress.values()) {
+      const job = this.jobsCache.get(id);
+      if (job?.type === type) {
+        count++;
+      }
+    }
+
+    return count;
+  };
+
+  public start(): Promise<void> {
     for (const [item, result] of this.database.allProvingJobs()) {
       this.logger.info(`Restoring proving job id=${item.id} settled=${!!result}`);
 
       this.jobsCache.set(item.id, item);
+      this.promises.set(item.id, promiseWithResolvers());
+
       if (result) {
+        this.promises.get(item.id)!.resolve(result);
         this.resultsCache.set(item.id, result);
       } else {
         this.logger.debug(`Re-enqueuing proving job id=${item.id}`);
@@ -95,13 +128,18 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     }
 
     this.timeoutPromise.start();
+
+    this.instrumentation.monitorQueueDepth(this.measureQueueDepth);
+    this.instrumentation.monitorActiveJobs(this.countActiveJobs);
+
+    return Promise.resolve();
   }
 
   public stop(): Promise<void> {
     return this.timeoutPromise.stop();
   }
 
-  public async enqueueProvingJob(job: V2ProvingJob): Promise<void> {
+  public async enqueueProvingJob(job: ProvingJob): Promise<void> {
     if (this.jobsCache.has(job.id)) {
       const existing = this.jobsCache.get(job.id);
       assert.deepStrictEqual(job, existing, 'Duplicate proving job ID');
@@ -113,20 +151,35 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     this.enqueueJobInternal(job);
   }
 
-  public async removeAndCancelProvingJob(id: V2ProvingJobId): Promise<void> {
+  public waitForJobToSettle(id: ProvingJobId): Promise<ProvingJobSettledResult> {
+    const promiseWithResolvers = this.promises.get(id);
+    if (!promiseWithResolvers) {
+      return Promise.resolve({ status: 'rejected', reason: `Job ${id} not found` });
+    }
+    return promiseWithResolvers.promise;
+  }
+
+  public async removeAndCancelProvingJob(id: ProvingJobId): Promise<void> {
     this.logger.info(`Cancelling job id=${id}`);
     await this.database.deleteProvingJobAndResult(id);
 
+    // notify listeners of the cancellation
+    if (!this.resultsCache.has(id)) {
+      this.promises.get(id)?.resolve({ status: 'rejected', reason: 'Aborted' });
+    }
+
     this.jobsCache.delete(id);
+    this.promises.delete(id);
     this.resultsCache.delete(id);
     this.inProgress.delete(id);
     this.retries.delete(id);
   }
 
-  // eslint-disable-next-line require-await
-  public async getProvingJobStatus(id: V2ProvingJobId): Promise<V2ProvingJobStatus> {
+  public getProvingJobStatus(id: ProvingJobId): Promise<ProvingJobStatus> {
     const result = this.resultsCache.get(id);
-    if (!result) {
+    if (result) {
+      return Promise.resolve(result);
+    } else {
       // no result yet, check if we know the item
       const item = this.jobsCache.get(id);
 
@@ -136,29 +189,26 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       }
 
       return Promise.resolve({ status: this.inProgress.has(id) ? 'in-progress' : 'in-queue' });
-    } else if ('value' in result) {
-      return Promise.resolve({ status: 'resolved', value: result.value });
-    } else {
-      return Promise.resolve({ status: 'rejected', error: result.error });
     }
   }
 
   // eslint-disable-next-line require-await
-  async getProvingJob<T extends ProvingRequestType[]>(
-    filter: ProvingJobFilter<T> = {},
-  ): Promise<{ job: V2ProvingJob; time: number } | undefined> {
-    const allowedProofs: ProvingRequestType[] = filter.allowList
-      ? [...filter.allowList]
-      : Object.values(ProvingRequestType).filter((x): x is ProvingRequestType => typeof x === 'number');
+  async getProvingJob(
+    filter: ProvingJobFilter = { allowList: [] },
+  ): Promise<{ job: ProvingJob; time: number } | undefined> {
+    const allowedProofs: ProvingRequestType[] =
+      Array.isArray(filter.allowList) && filter.allowList.length > 0
+        ? [...filter.allowList]
+        : Object.values(ProvingRequestType).filter((x): x is ProvingRequestType => typeof x === 'number');
     allowedProofs.sort(proofTypeComparator);
 
     for (const proofType of allowedProofs) {
       const queue = this.queues[proofType];
-      let job: V2ProvingJob | undefined;
+      let job: ProvingJob | undefined;
       // exhaust the queue and make sure we're not sending a job that's already in progress
       // or has already been completed
       // this can happen if the broker crashes and restarts
-      // it's possible agents will report progress or results for jobs that are no longer in the queue
+      // it's possible agents will report progress or results for jobs that are in the queue (after the restart)
       while ((job = queue.getImmediate())) {
         if (!this.inProgress.has(job.id) && !this.resultsCache.has(job.id)) {
           const time = this.timeSource();
@@ -167,6 +217,10 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
             startedAt: time,
             lastUpdatedAt: time,
           });
+          const enqueuedAt = this.enqueuedAt.get(job.id);
+          if (enqueuedAt) {
+            this.instrumentation.recordJobWait(job.type, enqueuedAt);
+          }
 
           return { job, time };
         }
@@ -176,7 +230,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     return undefined;
   }
 
-  async reportProvingJobError(id: V2ProvingJobId, err: Error, retry = false): Promise<void> {
+  async reportProvingJobError(id: ProvingJobId, err: string, retry = false): Promise<void> {
     const info = this.inProgress.get(id);
     const item = this.jobsCache.get(id);
     const retries = this.retries.get(id) ?? 0;
@@ -196,21 +250,31 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       this.logger.info(`Retrying proving job id=${id} type=${ProvingRequestType[item.type]} retry=${retries + 1}`);
       this.retries.set(id, retries + 1);
       this.enqueueJobInternal(item);
+      this.instrumentation.incRetriedJobs(item.type);
       return;
     }
 
     this.logger.debug(
       `Marking proving job id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1} as failed`,
     );
+
     await this.database.setProvingJobError(id, err);
-    this.resultsCache.set(id, { error: String(err) });
+
+    const result: ProvingJobSettledResult = { status: 'rejected', reason: String(err) };
+    this.resultsCache.set(id, result);
+    this.promises.get(id)!.resolve(result);
+    this.instrumentation.incRejectedJobs(item.type);
+    if (info) {
+      const duration = this.timeSource() - info.startedAt;
+      this.instrumentation.recordJobDuration(item.type, duration * 1000);
+    }
   }
 
-  reportProvingJobProgress<F extends ProvingRequestType[]>(
-    id: V2ProvingJobId,
+  reportProvingJobProgress(
+    id: ProvingJobId,
     startedAt: number,
-    filter?: ProvingJobFilter<F>,
-  ): Promise<{ job: V2ProvingJob; time: number } | undefined> {
+    filter?: ProvingJobFilter,
+  ): Promise<{ job: ProvingJob; time: number } | undefined> {
     const job = this.jobsCache.get(id);
     if (!job) {
       this.logger.warn(`Proving job id=${id} does not exist`);
@@ -255,7 +319,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     }
   }
 
-  async reportProvingJobSuccess(id: V2ProvingJobId, value: V2ProofOutput): Promise<void> {
+  async reportProvingJobSuccess(id: ProvingJobId, value: ProofUri): Promise<void> {
     const info = this.inProgress.get(id);
     const item = this.jobsCache.get(id);
     const retries = this.retries.get(id) ?? 0;
@@ -273,8 +337,13 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     this.logger.debug(
       `Proving job complete id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1}`,
     );
+
     await this.database.setProvingJobResult(id, value);
-    this.resultsCache.set(id, { value });
+
+    const result: ProvingJobSettledResult = { status: 'fulfilled', value };
+    this.resultsCache.set(id, result);
+    this.promises.get(id)!.resolve(result);
+    this.instrumentation.incResolvedJobs(item.type);
   }
 
   private timeoutCheck = () => {
@@ -287,23 +356,28 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
         continue;
       }
 
-      const secondsSinceLastUpdate = this.timeSource() - metadata.lastUpdatedAt;
-      if (secondsSinceLastUpdate >= this.jobTimeoutSec) {
+      const msSinceLastUpdate = (this.timeSource() - metadata.lastUpdatedAt) * 1000;
+      if (msSinceLastUpdate >= this.jobTimeoutMs) {
         this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`);
         this.inProgress.delete(id);
         this.enqueueJobInternal(item);
+        this.instrumentation.incTimedOutJobs(item.type);
       }
     }
   };
 
-  private enqueueJobInternal(job: V2ProvingJob): void {
+  private enqueueJobInternal(job: ProvingJob): void {
+    if (!this.promises.has(job.id)) {
+      this.promises.set(job.id, promiseWithResolvers());
+    }
     this.queues[job.type].put(job);
+    this.enqueuedAt.set(job.id, new Timer());
     this.logger.debug(`Enqueued new proving job id=${job.id}`);
   }
 }
 
 type ProvingQueues = {
-  [K in ProvingRequestType]: PriorityMemoryQueue<V2ProvingJob>;
+  [K in ProvingRequestType]: PriorityMemoryQueue<ProvingJob>;
 };
 
 /**
@@ -312,10 +386,12 @@ type ProvingQueues = {
  * @param b - Another proving job
  * @returns A number indicating the relative priority of the two proving jobs
  */
-function provingJobComparator(a: V2ProvingJob, b: V2ProvingJob): -1 | 0 | 1 {
-  if (a.blockNumber < b.blockNumber) {
+function provingJobComparator(a: ProvingJob, b: ProvingJob): -1 | 0 | 1 {
+  const aBlockNumber = a.blockNumber ?? 0;
+  const bBlockNumber = b.blockNumber ?? 0;
+  if (aBlockNumber < bBlockNumber) {
     return -1;
-  } else if (a.blockNumber > b.blockNumber) {
+  } else if (aBlockNumber > bBlockNumber) {
     return 1;
   } else {
     return 0;

package/src/proving_broker/proving_broker_database/memory.ts

@@ -0,0 +1,43 @@
+import type { ProofUri, ProvingJob, ProvingJobId, ProvingJobSettledResult } from '@aztec/circuit-types';
+
+import { type ProvingBrokerDatabase } from '../proving_broker_database.js';
+
+export class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
+  private jobs = new Map<ProvingJobId, ProvingJob>();
+  private results = new Map<ProvingJobId, ProvingJobSettledResult>();
+
+  getProvingJob(id: ProvingJobId): ProvingJob | undefined {
+    return this.jobs.get(id);
+  }
+
+  getProvingJobResult(id: ProvingJobId): ProvingJobSettledResult | undefined {
+    return this.results.get(id);
+  }
+
+  addProvingJob(request: ProvingJob): Promise<void> {
+    this.jobs.set(request.id, request);
+    return Promise.resolve();
+  }
+
+  setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void> {
+    this.results.set(id, { status: 'fulfilled', value });
+    return Promise.resolve();
+  }
+
+  setProvingJobError(id: ProvingJobId, reason: string): Promise<void> {
+    this.results.set(id, { status: 'rejected', reason });
+    return Promise.resolve();
+  }
+
+  deleteProvingJobAndResult(id: ProvingJobId): Promise<void> {
+    this.jobs.delete(id);
+    this.results.delete(id);
+    return Promise.resolve();
+  }
+
+  *allProvingJobs(): Iterable<[ProvingJob, ProvingJobSettledResult | undefined]> {
+    for (const item of this.jobs.values()) {
+      yield [item, this.results.get(item.id)] as const;
+    }
+  }
+}

package/src/proving_broker/proving_broker_database/persisted.ts

@@ -0,0 +1,60 @@
+import { type ProofUri, ProvingJob, type ProvingJobId, ProvingJobSettledResult } from '@aztec/circuit-types';
+import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
+import { type AztecKVStore, type AztecMap } from '@aztec/kv-store';
+import { LmdbMetrics, Metrics, type TelemetryClient } from '@aztec/telemetry-client';
+
+import { type ProvingBrokerDatabase } from '../proving_broker_database.js';
+
+export class KVBrokerDatabase implements ProvingBrokerDatabase {
+  private jobs: AztecMap<ProvingJobId, string>;
+  private jobResults: AztecMap<ProvingJobId, string>;
+  private metrics: LmdbMetrics;
+
+  constructor(private store: AztecKVStore, client: TelemetryClient) {
+    this.metrics = new LmdbMetrics(
+      client.getMeter('KVBrokerDatabase'),
+      {
+        name: Metrics.PROVING_QUEUE_DB_MAP_SIZE,
+        description: 'Database map size for the proving broker',
+      },
+      {
+        name: Metrics.PROVING_QUEUE_DB_USED_SIZE,
+        description: 'Database used size for the proving broker',
+      },
+      { name: Metrics.PROVING_QUEUE_DB_NUM_ITEMS, description: 'Number of items in the broker database' },
+      () => store.estimateSize(),
+    );
+    this.jobs = store.openMap('proving_jobs');
+    this.jobResults = store.openMap('proving_job_results');
+  }
+
+  async addProvingJob(job: ProvingJob): Promise<void> {
+    await this.jobs.set(job.id, jsonStringify(job));
+  }
+
+  *allProvingJobs(): Iterable<[ProvingJob, ProvingJobSettledResult | undefined]> {
+    for (const jobStr of this.jobs.values()) {
+      const job = jsonParseWithSchema(jobStr, ProvingJob);
+      const resultStr = this.jobResults.get(job.id);
+      const result = resultStr ? jsonParseWithSchema(resultStr, ProvingJobSettledResult) : undefined;
+      yield [job, result];
+    }
+  }
+
+  deleteProvingJobAndResult(id: ProvingJobId): Promise<void> {
+    return this.store.transaction(() => {
+      void this.jobs.delete(id);
+      void this.jobResults.delete(id);
+    });
+  }
+
+  async setProvingJobError(id: ProvingJobId, reason: string): Promise<void> {
+    const result: ProvingJobSettledResult = { status: 'rejected', reason };
+    await this.jobResults.set(id, jsonStringify(result));
+  }
+
+  async setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void> {
+    const result: ProvingJobSettledResult = { status: 'fulfilled', value };
+    await this.jobResults.set(id, jsonStringify(result));
+  }
+}

package/src/proving_broker/{proving_job_database.ts → proving_broker_database.ts}

@@ -1,30 +1,25 @@
-import {
-  type V2ProofOutput,
-  type V2ProvingJob,
-  type V2ProvingJobId,
-  type V2ProvingJobResult,
-} from '@aztec/circuit-types';
+import { type ProofUri, type ProvingJob, type ProvingJobId, type ProvingJobSettledResult } from '@aztec/circuit-types';
 
 /**
  * A database for storing proof requests and their results
  */
-export interface ProvingJobDatabase {
+export interface ProvingBrokerDatabase {
   /**
    * Saves a proof request so it can be retrieved later
    * @param request - The proof request to save
    */
-  addProvingJob(request: V2ProvingJob): Promise<void>;
+  addProvingJob(request: ProvingJob): Promise<void>;
 
   /**
    * Removes a proof request from the backend
    * @param id - The ID of the proof request to remove
    */
-  deleteProvingJobAndResult(id: V2ProvingJobId): Promise<void>;
+  deleteProvingJobAndResult(id: ProvingJobId): Promise<void>;
 
   /**
    * Returns an iterator over all saved proving jobs
    */
-  allProvingJobs(): Iterable<[V2ProvingJob, V2ProvingJobResult | undefined]>;
+  allProvingJobs(): Iterable<[ProvingJob, ProvingJobSettledResult | undefined]>;
 
   /**
    * Saves the result of a proof request
@@ -32,7 +27,7 @@ export interface ProvingJobDatabase {
    * @param ProvingRequestType - The type of proof that was requested
    * @param value - The result of the proof request
    */
-  setProvingJobResult(id: V2ProvingJobId, value: V2ProofOutput): Promise<void>;
+  setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
 
   /**
    * Saves an error that occurred while processing a proof request
@@ -40,5 +35,5 @@ export interface ProvingJobDatabase {
    * @param ProvingRequestType - The type of proof that was requested
    * @param err - The error that occurred while processing the proof request
    */
-  setProvingJobError(id: V2ProvingJobId, err: Error): Promise<void>;
+  setProvingJobError(id: ProvingJobId, err: string): Promise<void>;
 }

package/src/proving_broker/proving_broker_instrumentation.ts

@@ -0,0 +1,130 @@
+import { ProvingRequestType } from '@aztec/circuit-types';
+import { type Timer } from '@aztec/foundation/timer';
+import {
+  Attributes,
+  type Histogram,
+  Metrics,
+  type ObservableGauge,
+  type ObservableResult,
+  type TelemetryClient,
+  type UpDownCounter,
+  ValueType,
+  millisecondBuckets,
+} from '@aztec/telemetry-client';
+
+export type MonitorCallback = (proofType: ProvingRequestType) => number;
+
+export class ProvingBrokerInstrumentation {
+  private queueSize: ObservableGauge;
+  private activeJobs: ObservableGauge;
+  private resolvedJobs: UpDownCounter;
+  private rejectedJobs: UpDownCounter;
+  private timedOutJobs: UpDownCounter;
+  private jobWait: Histogram;
+  private jobDuration: Histogram;
+  private retriedJobs: UpDownCounter;
+
+  constructor(client: TelemetryClient, name = 'ProvingBroker') {
+    const meter = client.getMeter(name);
+
+    this.queueSize = meter.createObservableGauge(Metrics.PROVING_QUEUE_SIZE, {
+      valueType: ValueType.INT,
+    });
+
+    this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.resolvedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RESOLVED_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.rejectedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_REJECTED_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.retriedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RETRIED_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.timedOutJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT, {
+      description: 'Records how long a job sits in the queue',
+      unit: 'ms',
+      valueType: ValueType.INT,
+      advice: {
+        explicitBucketBoundaries: millisecondBuckets(1), // 10ms -> ~327s
+      },
+    });
+
+    this.jobDuration = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_DURATION, {
+      description: 'Records how long a job takes to complete',
+      unit: 'ms',
+      valueType: ValueType.INT,
+      advice: {
+        explicitBucketBoundaries: millisecondBuckets(1), // 10ms -> ~327s
+      },
+    });
+  }
+
+  monitorQueueDepth(fn: MonitorCallback) {
+    this.queueSize.addCallback(obs => this.observe(obs, fn));
+  }
+
+  monitorActiveJobs(fn: MonitorCallback) {
+    this.activeJobs.addCallback(obs => this.observe(obs, fn));
+  }
+
+  incResolvedJobs(proofType: ProvingRequestType) {
+    this.resolvedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incRejectedJobs(proofType: ProvingRequestType) {
+    this.rejectedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incRetriedJobs(proofType: ProvingRequestType) {
+    this.retriedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incTimedOutJobs(proofType: ProvingRequestType) {
+    this.timedOutJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  recordJobWait(proofType: ProvingRequestType, msOrTimer: Timer | number) {
+    const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
+    this.jobWait.record(duration, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  recordJobDuration(proofType: ProvingRequestType, msOrTimer: Timer | number) {
+    const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
+    this.jobDuration.record(duration, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  private observe(obs: ObservableResult, fn: MonitorCallback) {
+    for (const proofType of Object.values(ProvingRequestType)) {
+      // a type predicate for TypeScript to recognize that we're only iterating over enum values
+      if (typeof proofType !== 'number') {
+        continue;
+      }
+      obs.observe(fn(proofType), {
+        [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+      });
+    }
+  }
+}