@aztec/prover-client 0.65.2 → 0.66.0

package/src/orchestrator/tx-proving-state.ts

@@ -1,18 +1,10 @@
-import {
-  EncryptedNoteTxL2Logs,
-  EncryptedTxL2Logs,
-  type MerkleTreeId,
-  type ProcessedTx,
-  type ProofAndVerificationKey,
-  UnencryptedTxL2Logs,
-} from '@aztec/circuit-types';
+import { type MerkleTreeId, type ProcessedTx, type ProofAndVerificationKey } from '@aztec/circuit-types';
 import {
   type AVM_PROOF_LENGTH_IN_FIELDS,
   AVM_VK_INDEX,
   type AppendOnlyTreeSnapshot,
   AvmProofData,
   type BaseRollupHints,
-  Fr,
   PrivateBaseRollupHints,
   PrivateBaseRollupInputs,
   PrivateTubeData,
@@ -112,53 +104,6 @@ export class TxProvingState {
     this.avm = avmProofAndVk;
   }
 
-  public verifyStateOrReject(): string | undefined {
-    const txEffect = this.processedTx.txEffect;
-    const fromPrivate = this.processedTx.data;
-
-    const noteEncryptedLogsHashes = [
-      fromPrivate.forRollup?.end.noteEncryptedLogsHashes || [],
-      fromPrivate.forPublic?.nonRevertibleAccumulatedData.noteEncryptedLogsHashes || [],
-      fromPrivate.forPublic?.revertibleAccumulatedData.noteEncryptedLogsHashes || [],
-    ].flat();
-    const txNoteEncryptedLogsHash = EncryptedNoteTxL2Logs.hashNoteLogs(
-      noteEncryptedLogsHashes.filter(log => !log.isEmpty()).map(log => log.value.toBuffer()),
-    );
-    if (!txNoteEncryptedLogsHash.equals(txEffect.noteEncryptedLogs.hash())) {
-      return `Note encrypted logs hash mismatch: ${Fr.fromBuffer(txNoteEncryptedLogsHash)} === ${Fr.fromBuffer(
-        txEffect.noteEncryptedLogs.hash(),
-      )}`;
-    }
-
-    const encryptedLogsHashes = [
-      fromPrivate.forRollup?.end.encryptedLogsHashes || [],
-      fromPrivate.forPublic?.nonRevertibleAccumulatedData.encryptedLogsHashes || [],
-      fromPrivate.forPublic?.revertibleAccumulatedData.encryptedLogsHashes || [],
-    ].flat();
-    const txEncryptedLogsHash = EncryptedTxL2Logs.hashSiloedLogs(
-      encryptedLogsHashes.filter(log => !log.isEmpty()).map(log => log.getSiloedHash()),
-    );
-    if (!txEncryptedLogsHash.equals(txEffect.encryptedLogs.hash())) {
-      // @todo This rejection messages is never seen. Never making it out to the logs
-      return `Encrypted logs hash mismatch: ${Fr.fromBuffer(txEncryptedLogsHash)} === ${Fr.fromBuffer(
-        txEffect.encryptedLogs.hash(),
-      )}`;
-    }
-
-    const avmOutput = this.processedTx.avmProvingRequest?.inputs.output;
-    const unencryptedLogsHashes = avmOutput
-      ? avmOutput.accumulatedData.unencryptedLogsHashes
-      : fromPrivate.forRollup!.end.unencryptedLogsHashes;
-    const txUnencryptedLogsHash = UnencryptedTxL2Logs.hashSiloedLogs(
-      unencryptedLogsHashes.filter(log => !log.isEmpty()).map(log => log.getSiloedHash()),
-    );
-    if (!txUnencryptedLogsHash.equals(txEffect.unencryptedLogs.hash())) {
-      return `Unencrypted logs hash mismatch: ${Fr.fromBuffer(txUnencryptedLogsHash)} === ${Fr.fromBuffer(
-        txEffect.unencryptedLogs.hash(),
-      )}`;
-    }
-  }
-
   private getTubeVkData() {
     let vkIndex = TUBE_VK_INDEX;
     try {

package/src/{tx-prover → prover-client}/factory.ts

@@ -1,14 +1,15 @@
-import { type ProvingJobBroker } from '@aztec/circuit-types';
+import { type ForkMerkleTreeOperations, type ProvingJobBroker } from '@aztec/circuit-types';
 import { type TelemetryClient } from '@aztec/telemetry-client';
 import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 
 import { type ProverClientConfig } from '../config.js';
-import { TxProver } from './tx-prover.js';
+import { ProverClient } from './prover-client.js';
 
 export function createProverClient(
   config: ProverClientConfig,
+  worldState: ForkMerkleTreeOperations,
   broker: ProvingJobBroker,
   telemetry: TelemetryClient = new NoopTelemetryClient(),
 ) {
-  return TxProver.new(config, broker, telemetry);
+  return ProverClient.new(config, worldState, broker, telemetry);
 }

package/src/prover-client/index.ts

@@ -0,0 +1,2 @@
+export * from './factory.js';
+export * from './prover-client.js';

package/src/{tx-prover/tx-prover.ts → prover-client/prover-client.ts}

@@ -3,7 +3,7 @@ import {
   type ActualProverConfig,
   type EpochProver,
   type EpochProverManager,
-  type MerkleTreeWriteOperations,
+  type ForkMerkleTreeOperations,
   type ProverCache,
   type ProvingJobBroker,
   type ProvingJobConsumer,
@@ -25,11 +25,8 @@ import { InlineProofStore } from '../proving_broker/proof_store.js';
 import { InMemoryProverCache } from '../proving_broker/prover_cache/memory.js';
 import { ProvingAgent } from '../proving_broker/proving_agent.js';
 
-/**
- * A prover factory.
- * TODO(palla/prover-node): Rename this class
- */
-export class TxProver implements EpochProverManager {
+/** Manages proving of epochs by orchestrating the proving of individual blocks relying on a pool of prover agents. */
+export class ProverClient implements EpochProverManager {
   private running = false;
   private agents: ProvingAgent[] = [];
 
@@ -37,6 +34,7 @@ export class TxProver implements EpochProverManager {
 
   private constructor(
     private config: ProverClientConfig,
+    private worldState: ForkMerkleTreeOperations,
     private telemetry: TelemetryClient,
     private orchestratorClient: ProvingJobProducer,
     private agentClient?: ProvingJobConsumer,
@@ -47,9 +45,9 @@ export class TxProver implements EpochProverManager {
     this.cacheDir = this.config.cacheDir ? join(this.config.cacheDir, `tx_prover_${this.config.proverId}`) : undefined;
   }
 
-  public createEpochProver(db: MerkleTreeWriteOperations, cache: ProverCache = new InMemoryProverCache()): EpochProver {
+  public createEpochProver(cache: ProverCache = new InMemoryProverCache()): EpochProver {
     return new ProvingOrchestrator(
-      db,
+      this.worldState,
       new CachingBrokerFacade(this.orchestratorClient, cache),
       this.telemetry,
       this.config.proverId,
@@ -104,12 +102,16 @@ export class TxProver implements EpochProverManager {
   /**
    * Creates a new prover client and starts it
    * @param config - The prover configuration.
-   * @param vks - The verification keys for the prover
-   * @param worldStateSynchronizer - An instance of the world state
+   * @param worldState - An instance of the world state
    * @returns An instance of the prover, constructed and started.
    */
-  public static async new(config: ProverClientConfig, broker: ProvingJobBroker, telemetry: TelemetryClient) {
-    const prover = new TxProver(config, telemetry, broker, broker);
+  public static async new(
+    config: ProverClientConfig,
+    worldState: ForkMerkleTreeOperations,
+    broker: ProvingJobBroker,
+    telemetry: TelemetryClient,
+  ) {
+    const prover = new ProverClient(config, worldState, telemetry, broker, broker);
     await prover.start();
     return prover;
   }
@@ -135,7 +137,15 @@ export class TxProver implements EpochProverManager {
     const prover = await buildServerCircuitProver(this.config, this.telemetry);
     this.agents = times(
       this.config.proverAgentCount,
-      () => new ProvingAgent(this.agentClient!, proofStore, prover, [], this.config.proverAgentPollIntervalMs),
+      () =>
+        new ProvingAgent(
+          this.agentClient!,
+          proofStore,
+          prover,
+          this.telemetry,
+          [],
+          this.config.proverAgentPollIntervalMs,
+        ),
     );
 
     await Promise.all(this.agents.map(agent => agent.start()));

package/src/proving_broker/factory.ts

@@ -1,16 +1,20 @@
 import { type ProverBrokerConfig } from '@aztec/circuit-types';
 import { AztecLmdbStore } from '@aztec/kv-store/lmdb';
+import { type TelemetryClient } from '@aztec/telemetry-client';
 
 import { ProvingBroker } from './proving_broker.js';
 import { InMemoryBrokerDatabase } from './proving_broker_database/memory.js';
 import { KVBrokerDatabase } from './proving_broker_database/persisted.js';
 
-export async function createAndStartProvingBroker(config: ProverBrokerConfig): Promise<ProvingBroker> {
+export async function createAndStartProvingBroker(
+  config: ProverBrokerConfig,
+  client: TelemetryClient,
+): Promise<ProvingBroker> {
   const database = config.proverBrokerDataDirectory
-    ? new KVBrokerDatabase(AztecLmdbStore.open(config.proverBrokerDataDirectory))
+    ? new KVBrokerDatabase(AztecLmdbStore.open(config.proverBrokerDataDirectory), client)
     : new InMemoryBrokerDatabase();
 
-  const broker = new ProvingBroker(database, {
+  const broker = new ProvingBroker(database, client, {
     jobTimeoutMs: config.proverBrokerJobTimeoutMs,
     maxRetries: config.proverBrokerJobMaxRetries,
     timeoutIntervalMs: config.proverBrokerPollIntervalMs,

package/src/proving_broker/proving_agent.ts

@@ -10,8 +10,11 @@ import {
 } from '@aztec/circuit-types';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
+import { Timer } from '@aztec/foundation/timer';
+import { type TelemetryClient } from '@aztec/telemetry-client';
 
 import { type ProofStore } from './proof_store.js';
+import { ProvingAgentInstrumentation } from './proving_agent_instrumentation.js';
 import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_controller.js';
 
 /**
@@ -20,6 +23,8 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
 export class ProvingAgent {
   private currentJobController?: ProvingJobController;
   private runningPromise: RunningPromise;
+  private instrumentation: ProvingAgentInstrumentation;
+  private idleTimer: Timer | undefined;
 
   constructor(
     /** The source of proving jobs */
@@ -28,12 +33,15 @@ export class ProvingAgent {
     private proofStore: ProofStore,
     /** The prover implementation to defer jobs to */
     private circuitProver: ServerCircuitProver,
+    /** A telemetry client through which to emit metrics */
+    client: TelemetryClient,
     /** Optional list of allowed proof types to build */
     private proofAllowList: Array<ProvingRequestType> = [],
     /** How long to wait between jobs */
     private pollIntervalMs = 1000,
     private log = createDebugLogger('aztec:prover-client:proving-agent'),
   ) {
+    this.instrumentation = new ProvingAgentInstrumentation(client);
     this.runningPromise = new RunningPromise(this.safeWork, this.pollIntervalMs);
   }
 
@@ -46,6 +54,7 @@ export class ProvingAgent {
   }
 
   public start(): void {
+    this.idleTimer = new Timer();
     this.runningPromise.start();
   }
 
@@ -114,6 +123,11 @@ export class ProvingAgent {
         );
       }
 
+      if (this.idleTimer) {
+        this.instrumentation.recordIdleTime(this.idleTimer);
+      }
+      this.idleTimer = undefined;
+
       this.currentJobController.start();
     } catch (err) {
       this.log.error(`Error in ProvingAgent: ${String(err)}`);
@@ -126,9 +140,10 @@ export class ProvingAgent {
     err: Error | undefined,
     result: ProvingJobResultsMap[T] | undefined,
   ) => {
+    this.idleTimer = new Timer();
     if (err) {
       const retry = err.name === ProvingError.NAME ? (err as ProvingError).retry : false;
-      this.log.info(`Job id=${jobId} type=${ProvingRequestType[type]} failed err=${err.message} retry=${retry}`);
+      this.log.error(`Job id=${jobId} type=${ProvingRequestType[type]} failed err=${err.message} retry=${retry}`, err);
       return this.broker.reportProvingJobError(jobId, err.message, retry);
     } else if (result) {
       const outputUri = await this.proofStore.saveProofOutput(jobId, type, result);

package/src/proving_broker/proving_agent_instrumentation.ts

@@ -0,0 +1,21 @@
+import { type Timer } from '@aztec/foundation/timer';
+import { type Histogram, Metrics, type TelemetryClient, ValueType } from '@aztec/telemetry-client';
+
+export class ProvingAgentInstrumentation {
+  private idleTime: Histogram;
+
+  constructor(client: TelemetryClient, name = 'ProvingAgent') {
+    const meter = client.getMeter(name);
+
+    this.idleTime = meter.createHistogram(Metrics.PROVING_AGENT_IDLE, {
+      description: 'Records how long an agent was idle',
+      unit: 'ms',
+      valueType: ValueType.INT,
+    });
+  }
+
+  recordIdleTime(msOrTimer: Timer | number) {
+    const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
+    this.idleTime.record(duration);
+  }
+}

package/src/proving_broker/proving_broker.ts

@@ -12,10 +12,13 @@ import {
 import { createDebugLogger } from '@aztec/foundation/log';
 import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { PriorityMemoryQueue } from '@aztec/foundation/queue';
+import { Timer } from '@aztec/foundation/timer';
+import { type TelemetryClient } from '@aztec/telemetry-client';
 
 import assert from 'assert';
 
 import { type ProvingBrokerDatabase } from './proving_broker_database.js';
+import { type MonitorCallback, ProvingBrokerInstrumentation } from './proving_broker_instrumentation.js';
 
 type InProgressMetadata = {
   id: ProvingJobId;
@@ -58,6 +61,9 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
   // as above, but for results
   private resultsCache = new Map<ProvingJobId, ProvingJobSettledResult>();
 
+  // tracks when each job was enqueued
+  private enqueuedAt = new Map<ProvingJobId, Timer>();
+
   // keeps track of which jobs are currently being processed
   // in the event of a crash this information is lost, but that's ok
   // the next time the broker starts it will recreate jobsCache and still
@@ -75,18 +81,37 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
   private jobTimeoutMs: number;
   private maxRetries: number;
 
+  private instrumentation: ProvingBrokerInstrumentation;
+
   public constructor(
     private database: ProvingBrokerDatabase,
-    { jobTimeoutMs = 30, timeoutIntervalMs = 10, maxRetries = 3 }: ProofRequestBrokerConfig = {},
+    client: TelemetryClient,
+    { jobTimeoutMs = 30_000, timeoutIntervalMs = 10_000, maxRetries = 3 }: ProofRequestBrokerConfig = {},
     private logger = createDebugLogger('aztec:prover-client:proving-broker'),
   ) {
+    this.instrumentation = new ProvingBrokerInstrumentation(client);
     this.timeoutPromise = new RunningPromise(this.timeoutCheck, timeoutIntervalMs);
     this.jobTimeoutMs = jobTimeoutMs;
     this.maxRetries = maxRetries;
   }
 
-  // eslint-disable-next-line require-await
-  public async start(): Promise<void> {
+  private measureQueueDepth: MonitorCallback = (type: ProvingRequestType) => {
+    return this.queues[type].length();
+  };
+
+  private countActiveJobs: MonitorCallback = (type: ProvingRequestType) => {
+    let count = 0;
+    for (const { id } of this.inProgress.values()) {
+      const job = this.jobsCache.get(id);
+      if (job?.type === type) {
+        count++;
+      }
+    }
+
+    return count;
+  };
+
+  public start(): Promise<void> {
     for (const [item, result] of this.database.allProvingJobs()) {
       this.logger.info(`Restoring proving job id=${item.id} settled=${!!result}`);
 
@@ -103,6 +128,11 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     }
 
     this.timeoutPromise.start();
+
+    this.instrumentation.monitorQueueDepth(this.measureQueueDepth);
+    this.instrumentation.monitorActiveJobs(this.countActiveJobs);
+
+    return Promise.resolve();
   }
 
   public stop(): Promise<void> {
@@ -187,6 +217,10 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
             startedAt: time,
             lastUpdatedAt: time,
           });
+          const enqueuedAt = this.enqueuedAt.get(job.id);
+          if (enqueuedAt) {
+            this.instrumentation.recordJobWait(job.type, enqueuedAt);
+          }
 
           return { job, time };
         }
@@ -216,6 +250,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       this.logger.info(`Retrying proving job id=${id} type=${ProvingRequestType[item.type]} retry=${retries + 1}`);
       this.retries.set(id, retries + 1);
       this.enqueueJobInternal(item);
+      this.instrumentation.incRetriedJobs(item.type);
       return;
     }
 
@@ -228,6 +263,11 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     const result: ProvingJobSettledResult = { status: 'rejected', reason: String(err) };
     this.resultsCache.set(id, result);
     this.promises.get(id)!.resolve(result);
+    this.instrumentation.incRejectedJobs(item.type);
+    if (info) {
+      const duration = this.timeSource() - info.startedAt;
+      this.instrumentation.recordJobDuration(item.type, duration * 1000);
+    }
   }
 
   reportProvingJobProgress(
@@ -303,6 +343,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     const result: ProvingJobSettledResult = { status: 'fulfilled', value };
     this.resultsCache.set(id, result);
     this.promises.get(id)!.resolve(result);
+    this.instrumentation.incResolvedJobs(item.type);
   }
 
   private timeoutCheck = () => {
@@ -320,6 +361,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
         this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`);
         this.inProgress.delete(id);
         this.enqueueJobInternal(item);
+        this.instrumentation.incTimedOutJobs(item.type);
       }
     }
   };
@@ -329,6 +371,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       this.promises.set(job.id, promiseWithResolvers());
     }
     this.queues[job.type].put(job);
+    this.enqueuedAt.set(job.id, new Timer());
     this.logger.debug(`Enqueued new proving job id=${job.id}`);
   }
 }

package/src/proving_broker/proving_broker_database/persisted.ts

@@ -1,14 +1,29 @@
 import { type ProofUri, ProvingJob, type ProvingJobId, ProvingJobSettledResult } from '@aztec/circuit-types';
 import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
 import { type AztecKVStore, type AztecMap } from '@aztec/kv-store';
+import { LmdbMetrics, Metrics, type TelemetryClient } from '@aztec/telemetry-client';
 
 import { type ProvingBrokerDatabase } from '../proving_broker_database.js';
 
 export class KVBrokerDatabase implements ProvingBrokerDatabase {
   private jobs: AztecMap<ProvingJobId, string>;
   private jobResults: AztecMap<ProvingJobId, string>;
-
-  constructor(private store: AztecKVStore) {
+  private metrics: LmdbMetrics;
+
+  constructor(private store: AztecKVStore, client: TelemetryClient) {
+    this.metrics = new LmdbMetrics(
+      client.getMeter('KVBrokerDatabase'),
+      {
+        name: Metrics.PROVING_QUEUE_DB_MAP_SIZE,
+        description: 'Database map size for the proving broker',
+      },
+      {
+        name: Metrics.PROVING_QUEUE_DB_USED_SIZE,
+        description: 'Database used size for the proving broker',
+      },
+      { name: Metrics.PROVING_QUEUE_DB_NUM_ITEMS, description: 'Number of items in the broker database' },
+      () => store.estimateSize(),
+    );
     this.jobs = store.openMap('proving_jobs');
     this.jobResults = store.openMap('proving_job_results');
   }

package/src/proving_broker/proving_broker_instrumentation.ts

@@ -0,0 +1,130 @@
+import { ProvingRequestType } from '@aztec/circuit-types';
+import { type Timer } from '@aztec/foundation/timer';
+import {
+  Attributes,
+  type Histogram,
+  Metrics,
+  type ObservableGauge,
+  type ObservableResult,
+  type TelemetryClient,
+  type UpDownCounter,
+  ValueType,
+  millisecondBuckets,
+} from '@aztec/telemetry-client';
+
+export type MonitorCallback = (proofType: ProvingRequestType) => number;
+
+export class ProvingBrokerInstrumentation {
+  private queueSize: ObservableGauge;
+  private activeJobs: ObservableGauge;
+  private resolvedJobs: UpDownCounter;
+  private rejectedJobs: UpDownCounter;
+  private timedOutJobs: UpDownCounter;
+  private jobWait: Histogram;
+  private jobDuration: Histogram;
+  private retriedJobs: UpDownCounter;
+
+  constructor(client: TelemetryClient, name = 'ProvingBroker') {
+    const meter = client.getMeter(name);
+
+    this.queueSize = meter.createObservableGauge(Metrics.PROVING_QUEUE_SIZE, {
+      valueType: ValueType.INT,
+    });
+
+    this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.resolvedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RESOLVED_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.rejectedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_REJECTED_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.retriedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RETRIED_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.timedOutJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, {
+      valueType: ValueType.INT,
+    });
+
+    this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT, {
+      description: 'Records how long a job sits in the queue',
+      unit: 'ms',
+      valueType: ValueType.INT,
+      advice: {
+        explicitBucketBoundaries: millisecondBuckets(1), // 10ms -> ~327s
+      },
+    });
+
+    this.jobDuration = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_DURATION, {
+      description: 'Records how long a job takes to complete',
+      unit: 'ms',
+      valueType: ValueType.INT,
+      advice: {
+        explicitBucketBoundaries: millisecondBuckets(1), // 10ms -> ~327s
+      },
+    });
+  }
+
+  monitorQueueDepth(fn: MonitorCallback) {
+    this.queueSize.addCallback(obs => this.observe(obs, fn));
+  }
+
+  monitorActiveJobs(fn: MonitorCallback) {
+    this.activeJobs.addCallback(obs => this.observe(obs, fn));
+  }
+
+  incResolvedJobs(proofType: ProvingRequestType) {
+    this.resolvedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incRejectedJobs(proofType: ProvingRequestType) {
+    this.rejectedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incRetriedJobs(proofType: ProvingRequestType) {
+    this.retriedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incTimedOutJobs(proofType: ProvingRequestType) {
+    this.timedOutJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  recordJobWait(proofType: ProvingRequestType, msOrTimer: Timer | number) {
+    const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
+    this.jobWait.record(duration, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  recordJobDuration(proofType: ProvingRequestType, msOrTimer: Timer | number) {
+    const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
+    this.jobDuration.record(duration, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  private observe(obs: ObservableResult, fn: MonitorCallback) {
+    for (const proofType of Object.values(ProvingRequestType)) {
+      // a type predicate for TypeScript to recognize that we're only iterating over enum values
+      if (typeof proofType !== 'number') {
+        continue;
+      }
+      obs.observe(fn(proofType), {
+        [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+      });
+    }
+  }
+}

package/src/test/mock_prover.ts

@@ -43,6 +43,7 @@ import {
   makeRootRollupPublicInputs,
 } from '@aztec/circuits.js/testing';
 import { times } from '@aztec/foundation/collection';
+import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 
 import { InlineProofStore, type ProofStore } from '../proving_broker/proof_store.js';
 import { ProvingAgent } from '../proving_broker/proving_agent.js';
@@ -50,7 +51,7 @@ import { ProvingBroker } from '../proving_broker/proving_broker.js';
 import { InMemoryBrokerDatabase } from '../proving_broker/proving_broker_database/memory.js';
 
 export class TestBroker implements ProvingJobProducer {
-  private broker = new ProvingBroker(new InMemoryBrokerDatabase());
+  private broker = new ProvingBroker(new InMemoryBrokerDatabase(), new NoopTelemetryClient());
   private agents: ProvingAgent[];
 
   constructor(
@@ -58,7 +59,7 @@ export class TestBroker implements ProvingJobProducer {
     prover: ServerCircuitProver,
     private proofStore: ProofStore = new InlineProofStore(),
   ) {
-    this.agents = times(agentCount, () => new ProvingAgent(this.broker, proofStore, prover));
+    this.agents = times(agentCount, () => new ProvingAgent(this.broker, proofStore, prover, new NoopTelemetryClient()));
   }
 
   public async start() {

package/dest/tx-prover/factory.d.ts

@@ -1,6 +0,0 @@
-import { type ProvingJobBroker } from '@aztec/circuit-types';
-import { type TelemetryClient } from '@aztec/telemetry-client';
-import { type ProverClientConfig } from '../config.js';
-import { TxProver } from './tx-prover.js';
-export declare function createProverClient(config: ProverClientConfig, broker: ProvingJobBroker, telemetry?: TelemetryClient): Promise<TxProver>;
-//# sourceMappingURL=factory.d.ts.map

package/dest/tx-prover/factory.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/tx-prover/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAG/D,OAAO,EAAE,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,kBAAkB,EAC1B,MAAM,EAAE,gBAAgB,EACxB,SAAS,GAAE,eAA2C,qBAGvD"}

package/dest/tx-prover/factory.js

@@ -1,6 +0,0 @@
-import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
-import { TxProver } from './tx-prover.js';
-export function createProverClient(config, broker, telemetry = new NoopTelemetryClient()) {
-    return TxProver.new(config, broker, telemetry);
-}
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFjdG9yeS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy90eC1wcm92ZXIvZmFjdG9yeS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFFQSxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSw4QkFBOEIsQ0FBQztBQUduRSxPQUFPLEVBQUUsUUFBUSxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFMUMsTUFBTSxVQUFVLGtCQUFrQixDQUNoQyxNQUEwQixFQUMxQixNQUF3QixFQUN4QixZQUE2QixJQUFJLG1CQUFtQixFQUFFO0lBRXRELE9BQU8sUUFBUSxDQUFDLEdBQUcsQ0FBQyxNQUFNLEVBQUUsTUFBTSxFQUFFLFNBQVMsQ0FBQyxDQUFDO0FBQ2pELENBQUMifQ==

package/dest/tx-prover/tx-prover.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"tx-prover.d.ts","sourceRoot":"","sources":["../../src/tx-prover/tx-prover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,QAAQ,EAA2C,MAAM,kBAAkB,CAAC;AAC3G,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EAEvB,KAAK,mBAAmB,EACzB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,EAAE,EAAE,MAAM,oBAAoB,CAAC;AAIxC,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAI/D,OAAO,EAAE,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAOvD;;;GAGG;AACH,qBAAa,QAAS,YAAW,kBAAkB;IAO/C,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,kBAAkB;IAC1B,OAAO,CAAC,WAAW,CAAC;IACpB,OAAO,CAAC,GAAG;IAVb,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAsB;IAEpC,OAAO,CAAC,QAAQ,CAAC,CAAS;IAE1B,OAAO;IAYA,iBAAiB,CAAC,EAAE,EAAE,yBAAyB,EAAE,KAAK,GAAE,WAAuC,GAAG,WAAW;IAS7G,WAAW,IAAI,EAAE;IAIlB,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,kBAAkB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB5E;;OAEG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IASnC;;OAEG;IACU,IAAI;IAQjB;;;;;;OAMG;WACiB,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE,SAAS,EAAE,eAAe;IAMjG,mBAAmB,IAAI,kBAAkB;YAQlC,oBAAoB;YAmBpB,UAAU;CAGzB;AAED,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,kBAAkB,GAAG,UAAU,GAAG,QAAQ,EAClD,SAAS,EAAE,eAAe,GACzB,OAAO,CAAC,mBAAmB,CAAC,CAU9B"}