@aztec-labs/prover-client 6.0.0-nightly.20260829

package/src/proving_broker/proving_broker_database/memory.ts

@@ -0,0 +1,78 @@
+import { EpochNumber } from '@aztec-labs/foundation/branded-types';
+import {
+  type ProofUri,
+  type ProvingJob,
+  type ProvingJobId,
+  type ProvingJobSettledResult,
+  getEpochFromProvingJobId,
+} from '@aztec-labs/stdlib/interfaces/server';
+
+import type { ProvingBrokerDatabase } from '../proving_broker_database.js';
+
+export class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
+  private jobs = new Map<ProvingJobId, ProvingJob>();
+  private results = new Map<ProvingJobId, ProvingJobSettledResult>();
+
+  getProvingJob(id: ProvingJobId): ProvingJob | undefined {
+    return this.jobs.get(id);
+  }
+
+  getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined> {
+    return Promise.resolve(this.jobs.get(id)?.inputsUri);
+  }
+
+  getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined> {
+    return Promise.resolve(this.results.get(id));
+  }
+
+  addProvingJob(job: ProvingJob): Promise<void> {
+    this.jobs.set(job.id, job);
+    return Promise.resolve();
+  }
+
+  setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void> {
+    this.results.set(id, { status: 'fulfilled', value });
+    return Promise.resolve();
+  }
+
+  setProvingJobError(id: ProvingJobId, reason: string): Promise<void> {
+    this.results.set(id, { status: 'rejected', reason });
+    return Promise.resolve();
+  }
+
+  setProvingJobAborted(id: ProvingJobId): Promise<void> {
+    this.results.set(id, { status: 'aborted' });
+    return Promise.resolve();
+  }
+
+  deleteProvingJobResult(id: ProvingJobId): Promise<void> {
+    this.results.delete(id);
+    return Promise.resolve();
+  }
+
+  deleteProvingJobs(ids: ProvingJobId[]): Promise<void> {
+    for (const id of ids) {
+      this.jobs.delete(id);
+      this.results.delete(id);
+    }
+    return Promise.resolve();
+  }
+
+  deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void> {
+    const toDelete = [
+      ...Array.from(this.jobs.keys()).filter(x => getEpochFromProvingJobId(x) < epochNumber),
+      ...Array.from(this.results.keys()).filter(x => getEpochFromProvingJobId(x) < epochNumber),
+    ];
+    return this.deleteProvingJobs(toDelete);
+  }
+
+  async *allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]> {
+    for (const item of this.jobs.values()) {
+      yield [item, this.results.get(item.id)] as const;
+    }
+  }
+
+  close(): Promise<void> {
+    return Promise.resolve();
+  }
+}

package/src/proving_broker/proving_broker_database/persisted.ts

@@ -0,0 +1,296 @@
+import { EpochNumber } from '@aztec-labs/foundation/branded-types';
+import { jsonParseWithSchema, jsonStringify } from '@aztec-labs/foundation/json-rpc';
+import { type Logger, createLogger } from '@aztec-labs/foundation/log';
+import { BatchQueue } from '@aztec-labs/foundation/queue';
+import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec-labs/kv-store';
+import { openVersionedStoreAt } from '@aztec-labs/kv-store/lmdb-v2';
+import {
+  type ProofUri,
+  ProvingJob,
+  type ProvingJobId,
+  ProvingJobSettledResult,
+  getEpochFromProvingJobId,
+} from '@aztec-labs/stdlib/interfaces/server';
+import {
+  Attributes,
+  LmdbMetrics,
+  type TelemetryClient,
+  type Tracer,
+  getTelemetryClient,
+  trackSpan,
+} from '@aztec-labs/telemetry-client';
+import { mkdir, readdir } from 'fs/promises';
+import { join } from 'path';
+
+import type { ProverBrokerConfig } from '../config.js';
+import type { ProvingBrokerDatabase } from '../proving_broker_database.js';
+
+class SingleEpochDatabase {
+  public static readonly SCHEMA_VERSION = 2;
+
+  private jobs: AztecAsyncMap<ProvingJobId, string>;
+  private jobResults: AztecAsyncMap<ProvingJobId, string>;
+
+  constructor(public readonly store: AztecAsyncKVStore) {
+    this.jobs = store.openMap('proving_jobs');
+    this.jobResults = store.openMap('proving_job_results');
+  }
+
+  estimateSize() {
+    return this.store.estimateSize();
+  }
+
+  async batchWrite(
+    jobs: ProvingJob[],
+    results: Array<[ProvingJobId, ProvingJobSettledResult]>,
+    deletedResults: ProvingJobId[] = [],
+  ) {
+    await this.store.transactionAsync(async () => {
+      for (const job of jobs) {
+        await this.jobs.set(job.id, jsonStringify(job));
+      }
+      for (const [id, result] of results) {
+        await this.jobResults.set(id, jsonStringify(result));
+      }
+      // Deletes are applied after sets so that, if a set and a delete for the same id land in one
+      // batch (e.g. an abort immediately followed by a revive's result-delete), the delete wins.
+      for (const id of deletedResults) {
+        await this.jobResults.delete(id);
+      }
+    });
+  }
+
+  async *allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]> {
+    for await (const jobStr of this.jobs.valuesAsync()) {
+      const job = jsonParseWithSchema(jobStr, ProvingJob);
+      const resultStr = await this.jobResults.getAsync(job.id);
+      const result = resultStr ? jsonParseWithSchema(resultStr, ProvingJobSettledResult) : undefined;
+      yield [job, result];
+    }
+  }
+
+  async getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined> {
+    const jobStr = await this.jobs.getAsync(id);
+    return jobStr ? jsonParseWithSchema(jobStr, ProvingJob).inputsUri : undefined;
+  }
+
+  async getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined> {
+    const resultStr = await this.jobResults.getAsync(id);
+    return resultStr ? jsonParseWithSchema(resultStr, ProvingJobSettledResult) : undefined;
+  }
+
+  async setProvingJobError(id: ProvingJobId, reason: string): Promise<void> {
+    const result: ProvingJobSettledResult = { status: 'rejected', reason };
+    await this.jobResults.set(id, jsonStringify(result));
+  }
+
+  async setProvingJobAborted(id: ProvingJobId): Promise<void> {
+    const result: ProvingJobSettledResult = { status: 'aborted' };
+    await this.jobResults.set(id, jsonStringify(result));
+  }
+
+  async setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void> {
+    const result: ProvingJobSettledResult = { status: 'fulfilled', value };
+    await this.jobResults.set(id, jsonStringify(result));
+  }
+
+  delete() {
+    return this.store.delete();
+  }
+
+  close() {
+    return this.store.close();
+  }
+}
+
+/**
+ * An item queued for a batched write. A {@link ProvingJob} adds/updates a job; a `[id, result]`
+ * tuple sets a job's settled result; a `[id, undefined]` tuple deletes a job's result. Deletes ride
+ * the same queue as writes so they stay FIFO-ordered against them: a revive's result-delete must
+ * land after the abort that preceded it, otherwise a still-queued abort could flush later and
+ * resurrect the cancelled state on disk.
+ */
+type BrokerWrite = ProvingJob | [ProvingJobId, ProvingJobSettledResult | undefined];
+
+export class KVBrokerDatabase implements ProvingBrokerDatabase {
+  private metrics: LmdbMetrics;
+
+  private batchQueue: BatchQueue<BrokerWrite, number>;
+
+  public readonly tracer: Tracer;
+
+  private constructor(
+    private epochs: Map<number, SingleEpochDatabase>,
+    private config: ProverBrokerConfig,
+    client: TelemetryClient = getTelemetryClient(),
+    private logger: Logger,
+  ) {
+    this.metrics = new LmdbMetrics(
+      client.getMeter('KVBrokerDatabase'),
+      {
+        [Attributes.DB_DATA_TYPE]: 'prover-broker',
+      },
+      () => this.estimateSize(),
+    );
+
+    this.tracer = client.getTracer('KVBrokerDatabase');
+
+    this.batchQueue = new BatchQueue(
+      (items, key) => this.commitWrites(items, key),
+      config.proverBrokerBatchSize,
+      config.proverBrokerBatchIntervalMs,
+      createLogger('proving-client:proving-broker-database:batch-queue'),
+    );
+  }
+
+  // exposed for testing
+  public async commitWrites(items: Array<BrokerWrite>, epochNumber: number) {
+    const jobsToAdd = items.filter((item): item is ProvingJob => 'id' in item);
+    const resultOps = items.filter((item): item is [ProvingJobId, ProvingJobSettledResult | undefined] =>
+      Array.isArray(item),
+    );
+    const resultsToSet = resultOps.filter((op): op is [ProvingJobId, ProvingJobSettledResult] => op[1] !== undefined);
+    const resultsToDelete = resultOps.filter(op => op[1] === undefined).map(([id]) => id);
+
+    const db = await this.getEpochDatabase(EpochNumber(epochNumber));
+    await db.batchWrite(jobsToAdd, resultsToSet, resultsToDelete);
+  }
+
+  private async estimateSize() {
+    const sizes = await Promise.all(Array.from(this.epochs.values()).map(x => x.estimateSize()));
+    return {
+      mappingSize: this.config.dataStoreMapSizeKb,
+      physicalFileSize: sizes.reduce((prev, curr) => prev + curr.physicalFileSize, 0),
+      numItems: sizes.reduce((prev, curr) => prev + curr.numItems, 0),
+      actualSize: sizes.reduce((prev, curr) => prev + curr.actualSize, 0),
+    };
+  }
+
+  public static async new(
+    config: ProverBrokerConfig,
+    client: TelemetryClient = getTelemetryClient(),
+    logger = createLogger('prover-client:proving-broker-database'),
+  ) {
+    const epochs: Map<number, SingleEpochDatabase> = new Map<number, SingleEpochDatabase>();
+    const files = await readdir(config.dataDirectory!, { recursive: false, withFileTypes: true });
+    for (const file of files) {
+      if (!file.isDirectory()) {
+        continue;
+      }
+      const fullDirectory = join(config.dataDirectory!, file.name);
+      const epochDirectory = file.name;
+      const epochNumber = parseInt(epochDirectory, 10);
+      if (!Number.isSafeInteger(epochNumber) || epochNumber < 0) {
+        logger.warn(`Found invalid epoch directory ${fullDirectory} when loading epoch databases, ignoring`);
+        continue;
+      }
+      logger.info(
+        `Loading broker database for epoch ${epochNumber} from ${fullDirectory} with map size ${config.dataStoreMapSizeKb}KB`,
+      );
+      const db = await openVersionedStoreAt(
+        fullDirectory,
+        SingleEpochDatabase.SCHEMA_VERSION,
+        config.rollupAddress,
+        config.dataStoreMapSizeKb,
+      );
+      const epochDb = new SingleEpochDatabase(db);
+      epochs.set(epochNumber, epochDb);
+    }
+    const db = new KVBrokerDatabase(epochs, config, client, logger);
+    db.start();
+    return db;
+  }
+
+  private start(): void {
+    this.batchQueue.start();
+  }
+
+  async close(): Promise<void> {
+    await this.batchQueue.stop();
+    for (const [_, v] of this.epochs) {
+      await v.close();
+    }
+  }
+
+  @trackSpan('KVBrokerDatabase.deleteAllProvingJobsOlderThanEpoch', epochNumber => ({
+    [Attributes.EPOCH_NUMBER]: epochNumber,
+  }))
+  async deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void> {
+    const oldEpochs = Array.from(this.epochs.keys()).filter(e => e < Number(epochNumber));
+    for (const old of oldEpochs) {
+      const db = this.epochs.get(old);
+      if (!db) {
+        continue;
+      }
+      this.logger.verbose(`Deleting broker database for epoch ${old}`);
+      await db.delete();
+      this.epochs.delete(old);
+    }
+  }
+
+  // Key jobs by the id-derived epoch, the same epoch results are written under (setProvingJobResult et
+  // al.). This keeps a job's inputs and its result co-located in one epoch database so both can be read
+  // back with a single keyed lookup (getProvingJobInputs/getProvingJobResult) rather than a scan. The id
+  // embeds the epoch and equals `job.epochNumber` at every construction site, so this matches the prior
+  // storage epoch while making the id the single source of truth for placement.
+  addProvingJob(job: ProvingJob): Promise<void> {
+    return this.batchQueue.put(job, getEpochFromProvingJobId(job.id));
+  }
+
+  async *allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]> {
+    const iterators = Array.from(this.epochs.values()).map(x => x.allProvingJobs());
+    for (const it of iterators) {
+      yield* it;
+    }
+  }
+
+  // A job id is `${epochNumber}:${type}:${inputsHash}`, so its epoch is recoverable directly and maps to
+  // exactly one epoch database. Results are already written under this same id-derived epoch (see
+  // setProvingJobResult et al.), and a job's `epochNumber` equals its id-epoch at every construction site,
+  // so inputs live there too. Use `epochs.get` (not `getEpochDatabase`, which would create the store on a
+  // miss) so a read for an unknown id has no side effects.
+  getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined> {
+    return this.epochs.get(getEpochFromProvingJobId(id))?.getProvingJobInputs(id) ?? Promise.resolve(undefined);
+  }
+
+  getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined> {
+    return this.epochs.get(getEpochFromProvingJobId(id))?.getProvingJobResult(id) ?? Promise.resolve(undefined);
+  }
+
+  setProvingJobError(id: ProvingJobId, reason: string): Promise<void> {
+    return this.batchQueue.put([id, { status: 'rejected', reason }], getEpochFromProvingJobId(id));
+  }
+
+  setProvingJobAborted(id: ProvingJobId): Promise<void> {
+    return this.batchQueue.put([id, { status: 'aborted' }], getEpochFromProvingJobId(id));
+  }
+
+  deleteProvingJobResult(id: ProvingJobId): Promise<void> {
+    return this.batchQueue.put([id, undefined], getEpochFromProvingJobId(id));
+  }
+
+  setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void> {
+    return this.batchQueue.put([id, { status: 'fulfilled', value }], getEpochFromProvingJobId(id));
+  }
+
+  private async getEpochDatabase(epochNumber: EpochNumber): Promise<SingleEpochDatabase> {
+    let epochDb = this.epochs.get(epochNumber);
+    if (!epochDb) {
+      const newEpochDirectory = join(this.config.dataDirectory!, epochNumber.toString());
+      await mkdir(newEpochDirectory, { recursive: true });
+      this.logger.info(
+        `Creating broker database for epoch ${epochNumber} at ${newEpochDirectory} with map size ${this.config.dataStoreMapSizeKb}`,
+      );
+      const db = await openVersionedStoreAt(
+        newEpochDirectory,
+        SingleEpochDatabase.SCHEMA_VERSION,
+        this.config.rollupAddress,
+        this.config.dataStoreMapSizeKb,
+      );
+      epochDb = new SingleEpochDatabase(db);
+      this.epochs.set(epochNumber, epochDb);
+    }
+
+    return epochDb;
+  }
+}

package/src/proving_broker/proving_broker_database.ts

@@ -0,0 +1,75 @@
+import { EpochNumber } from '@aztec-labs/foundation/branded-types';
+import type { ProofUri, ProvingJob, ProvingJobId, ProvingJobSettledResult } from '@aztec-labs/stdlib/interfaces/server';
+
+/**
+ * A database for storing proof requests and their results
+ */
+export interface ProvingBrokerDatabase {
+  /**
+   * Saves a proof request so it can be retrieved later
+   * @param job - The proof request to save
+   */
+  addProvingJob(job: ProvingJob): Promise<void>;
+
+  /**
+   * Deletes all proving jobs belonging to epochs older than the given epoch
+   * @param epochNumber - The epoch number beyond which jobs should be deleted
+   */
+  deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void>;
+
+  /**
+   * Returns an iterator over all saved proving jobs
+   */
+  allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
+
+  /**
+   * Reads a single job's inputs URI by id, or undefined if the job is unknown. The broker keeps only
+   * job metadata in memory and reads the (large) inputs from here on demand when dispatching to an agent.
+   * @param id - The ID of the proof request
+   */
+  getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined>;
+
+  /**
+   * Reads a single job's settled result by id, or undefined if not settled. The broker keeps only the
+   * settled status in memory and reads the (large) fulfilled proof from here on demand.
+   * @param id - The ID of the proof request
+   */
+  getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined>;
+
+  /**
+   * Saves the result of a proof request
+   * @param id - The ID of the proof request to save the result for
+   * @param ProvingRequestType - The type of proof that was requested
+   * @param value - The result of the proof request
+   */
+  setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
+
+  /**
+   * Saves an error that occurred while processing a proof request
+   * @param id - The ID of the proof request to save the error for
+   * @param ProvingRequestType - The type of proof that was requested
+   * @param err - The error that occurred while processing the proof request
+   */
+  setProvingJobError(id: ProvingJobId, err: string): Promise<void>;
+
+  /**
+   * Records that a proof request was cancelled. Unlike a result or error this is not terminal:
+   * re-enqueuing the same job id revives it, so the aborted state can survive a restart without
+   * permanently blocking the proof.
+   * @param id - The ID of the cancelled proof request
+   */
+  setProvingJobAborted(id: ProvingJobId): Promise<void>;
+
+  /**
+   * Clears any stored result for a proof request, returning it to the pending state while keeping
+   * the job itself. Used when reviving an aborted job so the revival is persisted and a restart
+   * cannot resurrect the stale aborted state.
+   * @param id - The ID of the proof request whose result should be cleared
+   */
+  deleteProvingJobResult(id: ProvingJobId): Promise<void>;
+
+  /**
+   * Closes the database
+   */
+  close(): Promise<void>;
+}

package/src/proving_broker/proving_broker_instrumentation.ts

@@ -0,0 +1,133 @@
+import type { Timer } from '@aztec-labs/foundation/timer';
+import { ProvingRequestType } from '@aztec-labs/stdlib/proofs';
+import {
+  Attributes,
+  type Histogram,
+  Metrics,
+  type ObservableGauge,
+  type ObservableResult,
+  type TelemetryClient,
+  type UpDownCounter,
+  createUpDownCounterWithDefault,
+} from '@aztec-labs/telemetry-client';
+
+export type MonitorCallback = (proofType: ProvingRequestType) => number;
+
+export class ProvingBrokerInstrumentation {
+  private queueSize: ObservableGauge;
+  private activeJobs: ObservableGauge;
+  private resolvedJobs: UpDownCounter;
+  private rejectedJobs: UpDownCounter;
+  private abortedJobs: UpDownCounter;
+  private timedOutJobs: UpDownCounter;
+  private cachedJobs: UpDownCounter;
+  private totalJobs: UpDownCounter;
+  private jobWait: Histogram;
+  private jobDuration: Histogram;
+  private retriedJobs: UpDownCounter;
+
+  constructor(client: TelemetryClient, name = 'ProvingBroker') {
+    const meter = client.getMeter(name);
+
+    this.queueSize = meter.createObservableGauge(Metrics.PROVING_QUEUE_SIZE);
+
+    this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS);
+
+    const provingJobTypes = Object.values(ProvingRequestType).filter(v => typeof v === 'string');
+    const provingJobAttrs = { [Attributes.PROVING_JOB_TYPE]: provingJobTypes };
+
+    this.resolvedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RESOLVED_JOBS, provingJobAttrs);
+
+    this.rejectedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_REJECTED_JOBS, provingJobAttrs);
+
+    this.abortedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_ABORTED_JOBS, provingJobAttrs);
+
+    this.retriedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RETRIED_JOBS, provingJobAttrs);
+
+    this.timedOutJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, provingJobAttrs);
+
+    this.cachedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_CACHED_JOBS, provingJobAttrs);
+
+    this.totalJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TOTAL_JOBS, provingJobAttrs);
+
+    this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT);
+
+    this.jobDuration = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_DURATION);
+  }
+
+  monitorQueueDepth(fn: MonitorCallback) {
+    this.queueSize.addCallback(obs => this.observe(obs, fn));
+  }
+
+  monitorActiveJobs(fn: MonitorCallback) {
+    this.activeJobs.addCallback(obs => this.observe(obs, fn));
+  }
+
+  incResolvedJobs(proofType: ProvingRequestType) {
+    this.resolvedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incRejectedJobs(proofType: ProvingRequestType) {
+    this.rejectedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incAbortedJobs(proofType: ProvingRequestType) {
+    this.abortedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incRetriedJobs(proofType: ProvingRequestType) {
+    this.retriedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incTimedOutJobs(proofType: ProvingRequestType) {
+    this.timedOutJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incCachedJobs(proofType: ProvingRequestType) {
+    this.cachedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  incTotalJobs(proofType: ProvingRequestType) {
+    this.totalJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  recordJobWait(proofType: ProvingRequestType, msOrTimer: Timer | number) {
+    const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
+    this.jobWait.record(duration, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  recordJobDuration(proofType: ProvingRequestType, msOrTimer: Timer | number) {
+    const duration = typeof msOrTimer === 'number' ? msOrTimer : Math.floor(msOrTimer.ms());
+    this.jobDuration.record(duration, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
+  private observe(obs: ObservableResult, fn: MonitorCallback) {
+    for (const proofType of Object.values(ProvingRequestType)) {
+      // a type predicate for TypeScript to recognize that we're only iterating over enum values
+      if (typeof proofType !== 'number') {
+        continue;
+      }
+      obs.observe(fn(proofType), {
+        [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+      });
+    }
+  }
+}