@aztec/prover-client 0.65.2 → 0.67.0

package/src/proving_broker/proving_broker.ts

@@ -9,13 +9,17 @@ import {
   type ProvingJobStatus,
   ProvingRequestType,
 } from '@aztec/circuit-types';
-import { createDebugLogger } from '@aztec/foundation/log';
+import { asyncPool } from '@aztec/foundation/async-pool';
+import { createLogger } from '@aztec/foundation/log';
 import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { PriorityMemoryQueue } from '@aztec/foundation/queue';
+import { Timer } from '@aztec/foundation/timer';
+import { type TelemetryClient } from '@aztec/telemetry-client';
 
 import assert from 'assert';
 
 import { type ProvingBrokerDatabase } from './proving_broker_database.js';
+import { type MonitorCallback, ProvingBrokerInstrumentation } from './proving_broker_instrumentation.js';
 
 type InProgressMetadata = {
   id: ProvingJobId;
@@ -27,29 +31,33 @@ type ProofRequestBrokerConfig = {
   timeoutIntervalMs?: number;
   jobTimeoutMs?: number;
   maxRetries?: number;
+  maxEpochsToKeepResultsFor?: number;
+  maxParallelCleanUps?: number;
 };
 
+type EnqueuedProvingJob = Pick<ProvingJob, 'id' | 'epochNumber'>;
+
 /**
  * A broker that manages proof requests and distributes them to workers based on their priority.
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
  */
 export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
   private queues: ProvingQueues = {
-    [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
-    [ProvingRequestType.TUBE_PROOF]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
-    [ProvingRequestType.PRIVATE_KERNEL_EMPTY]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
+    [ProvingRequestType.TUBE_PROOF]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
+    [ProvingRequestType.PRIVATE_KERNEL_EMPTY]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
 
-    [ProvingRequestType.PRIVATE_BASE_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
-    [ProvingRequestType.PUBLIC_BASE_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
-    [ProvingRequestType.MERGE_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
-    [ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.PRIVATE_BASE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
+    [ProvingRequestType.PUBLIC_BASE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
+    [ProvingRequestType.MERGE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
+    [ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
 
-    [ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
-    [ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
-    [ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
+    [ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
+    [ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
 
-    [ProvingRequestType.BASE_PARITY]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
-    [ProvingRequestType.ROOT_PARITY]: new PriorityMemoryQueue<ProvingJob>(provingJobComparator),
+    [ProvingRequestType.BASE_PARITY]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
+    [ProvingRequestType.ROOT_PARITY]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
   };
 
   // holds a copy of the database in memory in order to quickly fulfill requests
@@ -58,6 +66,9 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
   // as above, but for results
   private resultsCache = new Map<ProvingJobId, ProvingJobSettledResult>();
 
+  // tracks when each job was enqueued
+  private enqueuedAt = new Map<ProvingJobId, Timer>();
+
   // keeps track of which jobs are currently being processed
   // in the event of a crash this information is lost, but that's ok
   // the next time the broker starts it will recreate jobsCache and still
@@ -70,23 +81,65 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
   // a map of promises that will be resolved when a job is settled
   private promises = new Map<ProvingJobId, PromiseWithResolvers<ProvingJobSettledResult>>();
 
-  private timeoutPromise: RunningPromise;
-  private timeSource = () => Math.floor(Date.now() / 1000);
+  private cleanupPromise: RunningPromise;
+  private msTimeSource = () => Date.now();
   private jobTimeoutMs: number;
   private maxRetries: number;
 
+  private instrumentation: ProvingBrokerInstrumentation;
+
+  private maxParallelCleanUps: number;
+
+  /**
+   * The broker keeps track of the highest epoch its seen.
+   * This information is used for garbage collection: once it reaches the next epoch, it can start pruning the database of old state.
+   * This clean up pass is only done against _settled_ jobs. This pass will not cancel jobs that are in-progress or in-queue.
+   * It is a client responsibility to cancel jobs if they are no longer necessary.
+   * Example:
+   * proving epoch 11 - the broker will wipe all setlled jobs for epochs 9 and lower
+   * finished proving epoch 11 and got first job for epoch 12 -> the broker will wipe all setlled jobs for epochs 10 and lower
+   * reorged back to end of epoch 10 -> epoch 11 is skipped and epoch 12 starts -> the broker will wipe all setlled jobs for epochs 10 and lower
+   */
+  private epochHeight = 0;
+  private maxEpochsToKeepResultsFor = 1;
+
   public constructor(
     private database: ProvingBrokerDatabase,
-    { jobTimeoutMs = 30, timeoutIntervalMs = 10, maxRetries = 3 }: ProofRequestBrokerConfig = {},
-    private logger = createDebugLogger('aztec:prover-client:proving-broker'),
+    client: TelemetryClient,
+    {
+      jobTimeoutMs = 30_000,
+      timeoutIntervalMs = 10_000,
+      maxRetries = 3,
+      maxEpochsToKeepResultsFor = 1,
+      maxParallelCleanUps = 20,
+    }: ProofRequestBrokerConfig = {},
+    private logger = createLogger('prover-client:proving-broker'),
   ) {
-    this.timeoutPromise = new RunningPromise(this.timeoutCheck, timeoutIntervalMs);
+    this.instrumentation = new ProvingBrokerInstrumentation(client);
+    this.cleanupPromise = new RunningPromise(this.cleanupPass, timeoutIntervalMs);
     this.jobTimeoutMs = jobTimeoutMs;
     this.maxRetries = maxRetries;
+    this.maxEpochsToKeepResultsFor = maxEpochsToKeepResultsFor;
+    this.maxParallelCleanUps = maxParallelCleanUps;
   }
 
-  // eslint-disable-next-line require-await
-  public async start(): Promise<void> {
+  private measureQueueDepth: MonitorCallback = (type: ProvingRequestType) => {
+    return this.queues[type].length();
+  };
+
+  private countActiveJobs: MonitorCallback = (type: ProvingRequestType) => {
+    let count = 0;
+    for (const { id } of this.inProgress.values()) {
+      const job = this.jobsCache.get(id);
+      if (job?.type === type) {
+        count++;
+      }
+    }
+
+    return count;
+  };
+
+  public start(): Promise<void> {
     for (const [item, result] of this.database.allProvingJobs()) {
       this.logger.info(`Restoring proving job id=${item.id} settled=${!!result}`);
 
@@ -102,11 +155,16 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       }
     }
 
-    this.timeoutPromise.start();
+    this.cleanupPromise.start();
+
+    this.instrumentation.monitorQueueDepth(this.measureQueueDepth);
+    this.instrumentation.monitorActiveJobs(this.countActiveJobs);
+
+    return Promise.resolve();
   }
 
-  public stop(): Promise<void> {
-    return this.timeoutPromise.stop();
+  public async stop(): Promise<void> {
+    await this.cleanupPromise.stop();
   }
 
   public async enqueueProvingJob(job: ProvingJob): Promise<void> {
@@ -129,15 +187,22 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     return promiseWithResolvers.promise;
   }
 
-  public async removeAndCancelProvingJob(id: ProvingJobId): Promise<void> {
-    this.logger.info(`Cancelling job id=${id}`);
-    await this.database.deleteProvingJobAndResult(id);
-
+  public async cancelProvingJob(id: ProvingJobId): Promise<void> {
     // notify listeners of the cancellation
     if (!this.resultsCache.has(id)) {
-      this.promises.get(id)?.resolve({ status: 'rejected', reason: 'Aborted' });
+      this.logger.info(`Cancelling job id=${id}`);
+      await this.reportProvingJobError(id, 'Aborted', false);
+    }
+  }
+
+  public async cleanUpProvingJobState(id: ProvingJobId): Promise<void> {
+    if (!this.resultsCache.has(id)) {
+      this.logger.warn(`Can't cleanup busy proving job: id=${id}`);
+      return;
     }
 
+    this.logger.debug(`Cleaning up state for job id=${id}`);
+    await this.database.deleteProvingJobAndResult(id);
     this.jobsCache.delete(id);
     this.promises.delete(id);
     this.resultsCache.delete(id);
@@ -174,19 +239,24 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
 
     for (const proofType of allowedProofs) {
       const queue = this.queues[proofType];
-      let job: ProvingJob | undefined;
+      let enqueuedJob: EnqueuedProvingJob | undefined;
       // exhaust the queue and make sure we're not sending a job that's already in progress
       // or has already been completed
       // this can happen if the broker crashes and restarts
       // it's possible agents will report progress or results for jobs that are in the queue (after the restart)
-      while ((job = queue.getImmediate())) {
-        if (!this.inProgress.has(job.id) && !this.resultsCache.has(job.id)) {
-          const time = this.timeSource();
+      while ((enqueuedJob = queue.getImmediate())) {
+        const job = this.jobsCache.get(enqueuedJob.id);
+        if (job && !this.inProgress.has(enqueuedJob.id) && !this.resultsCache.has(enqueuedJob.id)) {
+          const time = this.msTimeSource();
           this.inProgress.set(job.id, {
             id: job.id,
             startedAt: time,
             lastUpdatedAt: time,
           });
+          const enqueuedAt = this.enqueuedAt.get(job.id);
+          if (enqueuedAt) {
+            this.instrumentation.recordJobWait(job.type, enqueuedAt);
+          }
 
           return { job, time };
         }
@@ -212,15 +282,23 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       this.inProgress.delete(id);
     }
 
-    if (retry && retries + 1 < this.maxRetries) {
+    if (this.resultsCache.has(id)) {
+      this.logger.warn(`Proving job id=${id} already is already settled, ignoring error`);
+      return;
+    }
+
+    if (retry && retries + 1 < this.maxRetries && !this.isJobStale(item)) {
       this.logger.info(`Retrying proving job id=${id} type=${ProvingRequestType[item.type]} retry=${retries + 1}`);
       this.retries.set(id, retries + 1);
       this.enqueueJobInternal(item);
+      this.instrumentation.incRetriedJobs(item.type);
       return;
     }
 
-    this.logger.debug(
-      `Marking proving job id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1} as failed`,
+    this.logger.warn(
+      `Marking proving job as failed id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${
+        retries + 1
+      } err=${err}`,
     );
 
     await this.database.setProvingJobError(id, err);
@@ -228,6 +306,11 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     const result: ProvingJobSettledResult = { status: 'rejected', reason: String(err) };
     this.resultsCache.set(id, result);
     this.promises.get(id)!.resolve(result);
+    this.instrumentation.incRejectedJobs(item.type);
+    if (info) {
+      const duration = this.msTimeSource() - info.startedAt;
+      this.instrumentation.recordJobDuration(item.type, duration);
+    }
   }
 
   reportProvingJobProgress(
@@ -241,8 +324,13 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       return filter ? this.getProvingJob(filter) : Promise.resolve(undefined);
     }
 
+    if (this.resultsCache.has(id)) {
+      this.logger.warn(`Proving job id=${id} has already been completed`);
+      return filter ? this.getProvingJob(filter) : Promise.resolve(undefined);
+    }
+
     const metadata = this.inProgress.get(id);
-    const now = this.timeSource();
+    const now = this.msTimeSource();
     if (!metadata) {
       this.logger.warn(
         `Proving job id=${id} type=${ProvingRequestType[job.type]} not found in the in-progress cache, adding it`,
@@ -253,7 +341,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       this.inProgress.set(id, {
         id,
         startedAt,
-        lastUpdatedAt: this.timeSource(),
+        lastUpdatedAt: this.msTimeSource(),
       });
       return Promise.resolve(undefined);
     } else if (startedAt <= metadata.startedAt) {
@@ -294,6 +382,11 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       this.inProgress.delete(id);
     }
 
+    if (this.resultsCache.has(id)) {
+      this.logger.warn(`Proving job id=${id} already settled, ignoring result`);
+      return;
+    }
+
     this.logger.debug(
       `Proving job complete id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1}`,
     );
@@ -303,9 +396,34 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     const result: ProvingJobSettledResult = { status: 'fulfilled', value };
     this.resultsCache.set(id, result);
     this.promises.get(id)!.resolve(result);
+    this.instrumentation.incResolvedJobs(item.type);
   }
 
-  private timeoutCheck = () => {
+  private cleanupPass = async () => {
+    await this.cleanupStaleJobs();
+    await this.reEnqueueExpiredJobs();
+  };
+
+  private async cleanupStaleJobs() {
+    const jobIds = Array.from(this.jobsCache.keys());
+    const jobsToClean: ProvingJobId[] = [];
+    for (const id of jobIds) {
+      const job = this.jobsCache.get(id)!;
+      const isComplete = this.resultsCache.has(id);
+      if (isComplete && this.isJobStale(job)) {
+        jobsToClean.push(id);
+      }
+    }
+
+    if (jobsToClean.length > 0) {
+      this.logger.info(`Cleaning up [${jobsToClean.join(',')}]`);
+      await asyncPool(this.maxParallelCleanUps, jobsToClean, async jobId => {
+        await this.cleanUpProvingJobState(jobId);
+      });
+    }
+  }
+
+  private async reEnqueueExpiredJobs() {
     const inProgressEntries = Array.from(this.inProgress.entries());
     for (const [id, metadata] of inProgressEntries) {
       const item = this.jobsCache.get(id);
@@ -315,26 +433,42 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
         continue;
       }
 
-      const msSinceLastUpdate = (this.timeSource() - metadata.lastUpdatedAt) * 1000;
+      const now = this.msTimeSource();
+      const msSinceLastUpdate = now - metadata.lastUpdatedAt;
       if (msSinceLastUpdate >= this.jobTimeoutMs) {
-        this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`);
-        this.inProgress.delete(id);
-        this.enqueueJobInternal(item);
+        if (this.isJobStale(item)) {
+          // the job has timed out and it's also old, just cancel and move on
+          await this.cancelProvingJob(item.id);
+        } else {
+          this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`);
+          this.inProgress.delete(id);
+          this.enqueueJobInternal(item);
+          this.instrumentation.incTimedOutJobs(item.type);
+        }
       }
     }
-  };
+  }
 
   private enqueueJobInternal(job: ProvingJob): void {
     if (!this.promises.has(job.id)) {
       this.promises.set(job.id, promiseWithResolvers());
     }
-    this.queues[job.type].put(job);
+    this.queues[job.type].put({
+      epochNumber: job.epochNumber,
+      id: job.id,
+    });
+    this.enqueuedAt.set(job.id, new Timer());
+    this.epochHeight = Math.max(this.epochHeight, job.epochNumber);
     this.logger.debug(`Enqueued new proving job id=${job.id}`);
   }
+
+  private isJobStale(job: ProvingJob) {
+    return job.epochNumber < this.epochHeight - this.maxEpochsToKeepResultsFor;
+  }
 }
 
 type ProvingQueues = {
-  [K in ProvingRequestType]: PriorityMemoryQueue<ProvingJob>;
+  [K in ProvingRequestType]: PriorityMemoryQueue<EnqueuedProvingJob>;
 };
 
 /**
@@ -343,12 +477,10 @@ type ProvingQueues = {
  * @param b - Another proving job
  * @returns A number indicating the relative priority of the two proving jobs
  */
-function provingJobComparator(a: ProvingJob, b: ProvingJob): -1 | 0 | 1 {
-  const aBlockNumber = a.blockNumber ?? 0;
-  const bBlockNumber = b.blockNumber ?? 0;
-  if (aBlockNumber < bBlockNumber) {
+function provingJobComparator(a: EnqueuedProvingJob, b: EnqueuedProvingJob): -1 | 0 | 1 {
+  if (a.epochNumber < b.epochNumber) {
     return -1;
-  } else if (aBlockNumber > bBlockNumber) {
+  } else if (a.epochNumber > b.epochNumber) {
     return 1;
   } else {
     return 0;

package/src/proving_broker/proving_broker_database/persisted.ts

@@ -1,14 +1,23 @@
 import { type ProofUri, ProvingJob, type ProvingJobId, ProvingJobSettledResult } from '@aztec/circuit-types';
 import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
 import { type AztecKVStore, type AztecMap } from '@aztec/kv-store';
+import { Attributes, LmdbMetrics, type TelemetryClient } from '@aztec/telemetry-client';
 
 import { type ProvingBrokerDatabase } from '../proving_broker_database.js';
 
 export class KVBrokerDatabase implements ProvingBrokerDatabase {
   private jobs: AztecMap<ProvingJobId, string>;
   private jobResults: AztecMap<ProvingJobId, string>;
-
-  constructor(private store: AztecKVStore) {
+  private metrics: LmdbMetrics;
+
+  constructor(private store: AztecKVStore, client: TelemetryClient) {
+    this.metrics = new LmdbMetrics(
+      client.getMeter('KVBrokerDatabase'),
+      {
+        [Attributes.DB_DATA_TYPE]: 'prover-broker',
+      },
+      () => store.estimateSize(),
+    );
     this.jobs = store.openMap('proving_jobs');
     this.jobResults = store.openMap('proving_job_results');
   }

package/src/proving_broker/proving_broker_instrumentation.ts

@@ -0,0 +1,123 @@
+import { ProvingRequestType } from '@aztec/circuit-types';
+import { type Timer } from '@aztec/foundation/timer';
+import {
+  Attributes,
+  type Histogram,
+  Metrics,
+  type ObservableGauge,
+  type ObservableResult,
+  type TelemetryClient,
+  type UpDownCounter,
+  ValueType,
+} from '@aztec/telemetry-client';
+
+export type MonitorCallback = (proofType: ProvingRequestType) => number;
+
+export class ProvingBrokerInstrumentation {
+  private queueSize: ObservableGauge;
+  private activeJobs: ObservableGauge;
+  private resolvedJobs: UpDownCounter;
+  private rejectedJobs: UpDownCounter;
+  private timedOutJobs: UpDownCounter;
+  private jobWait: Histogram;
+  private jobDuration: Histogram;
+  private retriedJobs: UpDownCounter;
+
+  constructor(client: TelemetryClient, name = 'ProvingBroker') {
+    const meter = client.getMeter(name);
+
+    this.queueSize = meter.createObservableGauge(Metrics.PROVING_QUEUE_SIZE, {
+      valueType: ValueType.INT,
+    });
+
+    this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.resolvedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RESOLVED_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.rejectedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_REJECTED_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.retriedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RETRIED_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.timedOutJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT, {
+      description: 'Records how long a job sits in the queue',
+      unit: 'ms',
+      valueType: ValueType.INT,
+    });
+
+    this.jobDuration = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_DURATION, {
+      description: 'Records how long a job takes to complete',
+      unit: 'ms',
+      valueType: ValueType.INT,
+    });
+  }
+
+  monitorQueueDepth(fn: MonitorCallback) {
+    this.queueSize.addCallback(obs => this.observe(obs, fn));
+  }
+
+  monitorActiveJobs(fn: MonitorCallback) {
+    this.activeJobs.addCallback(obs => this.observe(obs, fn));
+  }
+
+  incResolvedJobs(proofType: ProvingRequestType) {
+    this.resolvedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incRejectedJobs(proofType: ProvingRequestType) {
+    this.rejectedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incRetriedJobs(proofType: ProvingRequestType) {
+    this.retriedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incTimedOutJobs(proofType: ProvingRequestType) {
+    this.timedOutJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  recordJobWait(proofType: ProvingRequestType, msOrTimer: Timer | number) {
+    const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
+    this.jobWait.record(duration, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  recordJobDuration(proofType: ProvingRequestType, msOrTimer: Timer | number) {
+    const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
+    this.jobDuration.record(duration, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  private observe(obs: ObservableResult, fn: MonitorCallback) {
+    for (const proofType of Object.values(ProvingRequestType)) {
+      // a type predicate for TypeScript to recognize that we're only iterating over enum values
+      if (typeof proofType !== 'number') {
+        continue;
+      }
+      obs.observe(fn(proofType), {
+        [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+      });
+    }
+  }
+}

package/src/proving_broker/rpc.ts

@@ -28,7 +28,8 @@ const GetProvingJobResponse = z.object({
 export const ProvingJobProducerSchema: ApiSchemaFor<ProvingJobProducer> = {
   enqueueProvingJob: z.function().args(ProvingJob).returns(z.void()),
   getProvingJobStatus: z.function().args(ProvingJobId).returns(ProvingJobStatus),
-  removeAndCancelProvingJob: z.function().args(ProvingJobId).returns(z.void()),
+  cleanUpProvingJobState: z.function().args(ProvingJobId).returns(z.void()),
+  cancelProvingJob: z.function().args(ProvingJobId).returns(z.void()),
   waitForJobToSettle: z.function().args(ProvingJobId).returns(ProvingJobSettledResult),
 };
 

package/src/test/mock_prover.ts

@@ -43,6 +43,7 @@ import {
   makeRootRollupPublicInputs,
 } from '@aztec/circuits.js/testing';
 import { times } from '@aztec/foundation/collection';
+import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 
 import { InlineProofStore, type ProofStore } from '../proving_broker/proof_store.js';
 import { ProvingAgent } from '../proving_broker/proving_agent.js';
@@ -50,7 +51,7 @@ import { ProvingBroker } from '../proving_broker/proving_broker.js';
 import { InMemoryBrokerDatabase } from '../proving_broker/proving_broker_database/memory.js';
 
 export class TestBroker implements ProvingJobProducer {
-  private broker = new ProvingBroker(new InMemoryBrokerDatabase());
+  private broker = new ProvingBroker(new InMemoryBrokerDatabase(), new NoopTelemetryClient());
   private agents: ProvingAgent[];
 
   constructor(
@@ -58,7 +59,7 @@ export class TestBroker implements ProvingJobProducer {
     prover: ServerCircuitProver,
     private proofStore: ProofStore = new InlineProofStore(),
   ) {
-    this.agents = times(agentCount, () => new ProvingAgent(this.broker, proofStore, prover));
+    this.agents = times(agentCount, () => new ProvingAgent(this.broker, proofStore, prover, new NoopTelemetryClient()));
   }
 
   public async start() {
@@ -81,8 +82,11 @@ export class TestBroker implements ProvingJobProducer {
   getProvingJobStatus(id: ProvingJobId): Promise<ProvingJobStatus> {
     return this.broker.getProvingJobStatus(id);
   }
-  removeAndCancelProvingJob(id: ProvingJobId): Promise<void> {
-    return this.broker.removeAndCancelProvingJob(id);
+  cleanUpProvingJobState(id: ProvingJobId): Promise<void> {
+    return this.broker.cleanUpProvingJobState(id);
+  }
+  cancelProvingJob(id: string): Promise<void> {
+    return this.broker.cancelProvingJob(id);
   }
   waitForJobToSettle(id: ProvingJobId): Promise<ProvingJobSettledResult> {
     return this.broker.waitForJobToSettle(id);

package/dest/tx-prover/factory.d.ts

@@ -1,6 +0,0 @@
-import { type ProvingJobBroker } from '@aztec/circuit-types';
-import { type TelemetryClient } from '@aztec/telemetry-client';
-import { type ProverClientConfig } from '../config.js';
-import { TxProver } from './tx-prover.js';
-export declare function createProverClient(config: ProverClientConfig, broker: ProvingJobBroker, telemetry?: TelemetryClient): Promise<TxProver>;
-//# sourceMappingURL=factory.d.ts.map

package/dest/tx-prover/factory.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/tx-prover/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAG/D,OAAO,EAAE,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,kBAAkB,EAC1B,MAAM,EAAE,gBAAgB,EACxB,SAAS,GAAE,eAA2C,qBAGvD"}

package/dest/tx-prover/factory.js

@@ -1,6 +0,0 @@
-import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
-import { TxProver } from './tx-prover.js';
-export function createProverClient(config, broker, telemetry = new NoopTelemetryClient()) {
-    return TxProver.new(config, broker, telemetry);
-}
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFjdG9yeS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy90eC1wcm92ZXIvZmFjdG9yeS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFFQSxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSw4QkFBOEIsQ0FBQztBQUduRSxPQUFPLEVBQUUsUUFBUSxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFMUMsTUFBTSxVQUFVLGtCQUFrQixDQUNoQyxNQUEwQixFQUMxQixNQUF3QixFQUN4QixZQUE2QixJQUFJLG1CQUFtQixFQUFFO0lBRXRELE9BQU8sUUFBUSxDQUFDLEdBQUcsQ0FBQyxNQUFNLEVBQUUsTUFBTSxFQUFFLFNBQVMsQ0FBQyxDQUFDO0FBQ2pELENBQUMifQ==

package/dest/tx-prover/tx-prover.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"tx-prover.d.ts","sourceRoot":"","sources":["../../src/tx-prover/tx-prover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,QAAQ,EAA2C,MAAM,kBAAkB,CAAC;AAC3G,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EAEvB,KAAK,mBAAmB,EACzB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,EAAE,EAAE,MAAM,oBAAoB,CAAC;AAIxC,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAI/D,OAAO,EAAE,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAOvD;;;GAGG;AACH,qBAAa,QAAS,YAAW,kBAAkB;IAO/C,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,kBAAkB;IAC1B,OAAO,CAAC,WAAW,CAAC;IACpB,OAAO,CAAC,GAAG;IAVb,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAsB;IAEpC,OAAO,CAAC,QAAQ,CAAC,CAAS;IAE1B,OAAO;IAYA,iBAAiB,CAAC,EAAE,EAAE,yBAAyB,EAAE,KAAK,GAAE,WAAuC,GAAG,WAAW;IAS7G,WAAW,IAAI,EAAE;IAIlB,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,kBAAkB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB5E;;OAEG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IASnC;;OAEG;IACU,IAAI;IAQjB;;;;;;OAMG;WACiB,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE,SAAS,EAAE,eAAe;IAMjG,mBAAmB,IAAI,kBAAkB;YAQlC,oBAAoB;YAmBpB,UAAU;CAGzB;AAED,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,kBAAkB,GAAG,UAAU,GAAG,QAAQ,EAClD,SAAS,EAAE,eAAe,GACzB,OAAO,CAAC,mBAAmB,CAAC,CAU9B"}