@aztec/prover-client 0.0.1-commit.fffb133c → 0.0.1-dev

package/src/prover-client/prover-client.ts

@@ -1,7 +1,7 @@
 import { type ACVMConfig, type BBConfig, BBNativeRollupProver, TestCircuitProver } from '@aztec/bb-prover';
 import { times } from '@aztec/foundation/collection';
 import type { EthAddress } from '@aztec/foundation/eth-address';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, createLogger } from '@aztec/foundation/log';
 import { NativeACVMSimulator } from '@aztec/simulator/server';
 import {
   type ActualProverConfig,
@@ -29,29 +29,34 @@ export class ProverClient implements EpochProverManager {
   private running = false;
   private agents: ProvingAgent[] = [];
 
-  private proofStore: ProofStore;
-  private failedProofStore: ProofStore | undefined;
-
   private constructor(
     private config: ProverClientConfig,
     private worldState: ForkMerkleTreeOperations & ReadonlyWorldStateAccess,
     private orchestratorClient: ProvingJobProducer,
+    private proofStore: ProofStore,
+    private failedProofStore: ProofStore | undefined,
     private agentClient?: ProvingJobConsumer,
     private telemetry: TelemetryClient = getTelemetryClient(),
-    private log = createLogger('prover-client:tx-prover'),
-  ) {
-    this.proofStore = new InlineProofStore();
-    this.failedProofStore = this.config.failedProofStore ? createProofStore(this.config.failedProofStore) : undefined;
-  }
+    private log: Logger = createLogger('prover-client:tx-prover'),
+  ) {}
 
   public createEpochProver(): EpochProver {
-    const facade = new BrokerCircuitProverFacade(this.orchestratorClient, this.proofStore, this.failedProofStore);
+    const bindings = this.log.getBindings();
+    const facade = new BrokerCircuitProverFacade(
+      this.orchestratorClient,
+      this.proofStore,
+      this.failedProofStore,
+      undefined,
+      bindings,
+    );
     const orchestrator = new ProvingOrchestrator(
       this.worldState,
       facade,
       this.config.proverId,
       this.config.cancelJobsOnStop,
+      this.config.enqueueConcurrency,
       this.telemetry,
+      bindings,
     );
     return new ServerEpochProver(facade, orchestrator);
   }
@@ -110,7 +115,9 @@ export class ProverClient implements EpochProverManager {
     broker: ProvingJobBroker,
     telemetry: TelemetryClient = getTelemetryClient(),
   ) {
-    const prover = new ProverClient(config, worldState, broker, broker, telemetry);
+    const proofStore = await createProofStore(config.proofStore);
+    const failedProofStore = config.failedProofStore ? await createProofStore(config.failedProofStore) : undefined;
+    const prover = new ProverClient(config, worldState, broker, proofStore, failedProofStore, broker, telemetry);
     await prover.start();
     return prover;
   }
@@ -134,9 +141,11 @@ export class ProverClient implements EpochProverManager {
 
     const proofStore = new InlineProofStore();
     const prover = await buildServerCircuitProver(this.config, this.telemetry);
+    const bindings = this.log.getBindings();
     this.agents = times(
       this.config.proverAgentCount,
-      () => new ProvingAgent(this.agentClient!, proofStore, prover, [], this.config.proverAgentPollIntervalMs),
+      () =>
+        new ProvingAgent(this.agentClient!, proofStore, prover, [], this.config.proverAgentPollIntervalMs, bindings),
     );
 
     await Promise.all(this.agents.map(agent => agent.start()));
@@ -148,15 +157,16 @@ export class ProverClient implements EpochProverManager {
 }
 
 export function buildServerCircuitProver(
-  config: ActualProverConfig & ACVMConfig & BBConfig,
+  config: Omit<ActualProverConfig, 'enqueueConcurrency'> & ACVMConfig & BBConfig,
   telemetry: TelemetryClient,
 ): Promise<ServerCircuitProver> {
   if (config.realProofs) {
     return BBNativeRollupProver.new(config, telemetry);
   }
 
+  const logger = createLogger('prover-client:acvm-native');
   const simulator = config.acvmBinaryPath
-    ? new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath)
+    ? new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath, undefined, logger)
     : undefined;
 
   return Promise.resolve(new TestCircuitProver(simulator, config, telemetry));

package/src/proving_broker/broker_prover_facade.ts

@@ -5,8 +5,7 @@ import type {
   RECURSIVE_PROOF_LENGTH,
 } from '@aztec/constants';
 import { EpochNumber } from '@aztec/foundation/branded-types';
-import { sha256 } from '@aztec/foundation/crypto/sha256';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { truncate } from '@aztec/foundation/string';
 import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
@@ -46,6 +45,8 @@ import type {
   TxRollupPublicInputs,
 } from '@aztec/stdlib/rollup';
 
+import { createHash } from 'node:crypto';
+
 import { InlineProofStore, type ProofStore } from './proof_store/index.js';
 
 // Perform a snapshot sync every 30 seconds
@@ -68,14 +69,17 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
   private runningPromise?: RunningPromise;
   private timeOfLastSnapshotSync = Date.now();
   private jobsToRetrieve: Set<ProvingJobId> = new Set();
+  private log: Logger;
 
   constructor(
     private broker: ProvingJobProducer,
     private proofStore: ProofStore = new InlineProofStore(),
     private failedProofStore?: ProofStore,
     private pollIntervalMs = 1000,
-    private log = createLogger('prover-client:broker-circuit-prover-facade'),
-  ) {}
+    bindings?: LoggerBindings,
+  ) {
+    this.log = createLogger('prover-client:broker-circuit-prover-facade', bindings);
+  }
 
   /**
    * This is a critical section. This function can not be async since it writes
@@ -656,8 +660,12 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
     );
   }
 
-  private generateId(type: ProvingRequestType, inputs: { toBuffer(): Buffer }, epochNumber = EpochNumber.ZERO) {
-    const inputsHash = sha256(inputs.toBuffer());
-    return makeProvingJobId(epochNumber, type, inputsHash.toString('hex'));
+  private generateId(
+    type: ProvingRequestType,
+    inputs: { toBuffer(): Buffer },
+    epochNumber = EpochNumber.ZERO,
+  ): ProvingJobId {
+    const inputsHash = createHash('sha256').update(inputs.toBuffer()).digest('hex');
+    return makeProvingJobId(epochNumber, type, inputsHash);
   }
 }

package/src/proving_broker/config.ts

@@ -31,6 +31,8 @@ export const ProverBrokerConfig = z.object({
   proverBrokerBatchIntervalMs: z.number().int().nonnegative(),
   /** The maximum number of epochs to keep results for */
   proverBrokerMaxEpochsToKeepResultsFor: z.number().int().nonnegative(),
+  /** Enable debug replay mode for replaying proving jobs from stored inputs */
+  proverBrokerDebugReplayEnabled: z.boolean(),
 });
 
 export type ProverBrokerConfig = z.infer<typeof ProverBrokerConfig> &
@@ -74,6 +76,11 @@ export const proverBrokerConfigMappings: ConfigMappingsType<ProverBrokerConfig>
     parseEnv: (val: string | undefined) => (val ? +val : undefined),
     description: "The size of the prover broker's database. Will override the dataStoreMapSizeKb if set.",
   },
+  proverBrokerDebugReplayEnabled: {
+    env: 'PROVER_BROKER_DEBUG_REPLAY_ENABLED',
+    description: 'Enable debug replay mode for replaying proving jobs from stored inputs',
+    ...booleanConfigHelper(false),
+  },
   ...dataConfigMappings,
   ...l1ReaderConfigMappings,
   ...pickConfigMappings(chainConfigMappings, ['rollupVersion']),
@@ -102,6 +109,8 @@ export const ProverAgentConfig = z.object({
   proverTestVerificationDelayMs: z.number().optional(),
   /** Whether to abort pending proving jobs when the orchestrator is cancelled */
   cancelJobsOnStop: z.boolean(),
+  /** Where to store proving results. Must be accessible to both prover node and agents. If not set will inline-encode the parameters */
+  proofStore: z.string().optional(),
 });
 
 export type ProverAgentConfig = z.infer<typeof ProverAgentConfig>;
@@ -162,4 +171,8 @@ export const proverAgentConfigMappings: ConfigMappingsType<ProverAgentConfig> =
       'When false (default), jobs remain in the broker queue and can be reused on restart/reorg.',
     ...booleanConfigHelper(false),
   },
+  proofStore: {
+    env: 'PROVER_PROOF_STORE',
+    description: 'Optional proof input store for the prover',
+  },
 };

package/src/proving_broker/proof_store/factory.ts

@@ -1,42 +1,20 @@
 import { createLogger } from '@aztec/foundation/log';
+import { createFileStore } from '@aztec/stdlib/file-store';
 
-import { GoogleCloudStorageProofStore } from './gcs_proof_store.js';
+import { FileStoreProofStore } from './file_store_proof_store.js';
 import { InlineProofStore } from './inline_proof_store.js';
 import type { ProofStore } from './proof_store.js';
 
-export function createProofStore(config: string | undefined, logger = createLogger('prover-client:proof-store')) {
-  if (config === undefined) {
+export async function createProofStore(
+  config: string | undefined,
+  logger = createLogger('prover-client:proof-store'),
+): Promise<ProofStore> {
+  if (!config) {
     logger.info('Creating inline proof store');
     return new InlineProofStore();
-  } else if (config.startsWith('gs://')) {
-    try {
-      const url = new URL(config);
-      const bucket = url.host;
-      const path = url.pathname.replace(/^\/+/, '');
-      logger.info(`Creating google cloud proof store at ${bucket}`, { bucket, path });
-      return new GoogleCloudStorageProofStore(bucket, path);
-    } catch {
-      throw new Error(
-        `Invalid google cloud proof store definition: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`,
-      );
-    }
-  } else {
-    throw new Error(`Unknown proof store config: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`);
   }
-}
 
-export function createProofStoreForUri(
-  uri: string,
-  logger = createLogger('prover-client:proof-store'),
-): Pick<ProofStore, 'getProofInput' | 'getProofOutput'> {
-  if (uri.startsWith('data://')) {
-    return createProofStore(undefined, logger);
-  } else if (uri.startsWith('gs://')) {
-    const url = new URL(uri);
-    const basePath = url.pathname.replace(/^\/+/, '').split('/').slice(0, -3);
-    url.pathname = basePath.join('/');
-    return createProofStore(uri, logger);
-  } else {
-    throw new Error(`Unknown proof store config: '${uri}'. Supported protocols are 'data://' and 'gs://'.`);
-  }
+  const fileStore = await createFileStore(config, logger);
+  logger.info(`Creating file store proof store at ${config}`);
+  return new FileStoreProofStore(fileStore);
 }

package/src/proving_broker/proof_store/file_store_proof_store.ts

@@ -0,0 +1,78 @@
+import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
+import type { FileStore } from '@aztec/stdlib/file-store';
+import {
+  type ProofUri,
+  type ProvingJobId,
+  type ProvingJobInputs,
+  type ProvingJobInputsMap,
+  ProvingJobResult,
+  type ProvingJobResultsMap,
+  getProvingJobInputClassFor,
+} from '@aztec/stdlib/interfaces/server';
+import { ProvingRequestType } from '@aztec/stdlib/proofs';
+
+import type { ProofStore } from './proof_store.js';
+
+const INPUTS_PATH = 'inputs';
+const OUTPUTS_PATH = 'outputs';
+
+/**
+ * A proof store implementation backed by a generic FileStore.
+ * Supports any storage backend (GCS, S3, local filesystem) via the FileStore abstraction.
+ */
+export class FileStoreProofStore implements ProofStore {
+  constructor(private readonly fileStore: FileStore) {}
+
+  async saveProofInput<T extends ProvingRequestType>(
+    id: ProvingJobId,
+    type: T,
+    inputs: ProvingJobInputsMap[T],
+  ): Promise<ProofUri> {
+    const path = `${INPUTS_PATH}/${ProvingRequestType[type]}/${id}`;
+    const uri = await this.fileStore.save(path, inputs.toBuffer());
+    return uri as ProofUri;
+  }
+
+  async saveProofOutput<T extends ProvingRequestType>(
+    id: ProvingJobId,
+    type: T,
+    result: ProvingJobResultsMap[T],
+  ): Promise<ProofUri> {
+    const jobResult = { type, result } as ProvingJobResult;
+    const json = jsonStringify(jobResult);
+    const path = `${OUTPUTS_PATH}/${ProvingRequestType[type]}/${id}.json`;
+    const uri = await this.fileStore.save(path, Buffer.from(json, 'utf-8'));
+    return uri as ProofUri;
+  }
+
+  async getProofInput(uri: ProofUri): Promise<ProvingJobInputs> {
+    try {
+      const buffer = await this.fileStore.read(uri);
+      const type = this.extractTypeFromUri(uri);
+      const inputs = getProvingJobInputClassFor(type).fromBuffer(buffer);
+      return { inputs, type } as ProvingJobInputs;
+    } catch (err) {
+      throw new Error(`Error getting proof input at ${uri}: ${err}`);
+    }
+  }
+
+  async getProofOutput(uri: ProofUri): Promise<ProvingJobResult> {
+    try {
+      const buffer = await this.fileStore.read(uri);
+      return jsonParseWithSchema(buffer.toString('utf-8'), ProvingJobResult);
+    } catch (err) {
+      throw new Error(`Error getting proof output at ${uri}: ${err}`);
+    }
+  }
+
+  private extractTypeFromUri(uri: string): ProvingRequestType {
+    const url = new URL(uri);
+    const pathParts = url.pathname.split('/').filter(Boolean);
+    const typeString = pathParts.at(-2);
+    const type = typeString ? ProvingRequestType[typeString as keyof typeof ProvingRequestType] : undefined;
+    if (type === undefined) {
+      throw new Error(`Unrecognized proof type ${typeString} in URI ${uri}`);
+    }
+    return type;
+  }
+}

package/src/proving_broker/proof_store/index.ts

@@ -1,4 +1,4 @@
 export * from './proof_store.js';
 export * from './inline_proof_store.js';
 export * from './factory.js';
-export * from './gcs_proof_store.js';
+export * from './file_store_proof_store.js';

package/src/proving_broker/proving_agent.ts

@@ -1,5 +1,5 @@
 import { AbortError } from '@aztec/foundation/error';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 import { truncate } from '@aztec/foundation/string';
 import { ProvingError } from '@aztec/stdlib/errors';
@@ -23,6 +23,7 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
 export class ProvingAgent {
   private currentJobController?: ProvingJobController;
   private runningPromise: RunningPromise;
+  private log: Logger;
 
   constructor(
     /** The source of proving jobs */
@@ -35,8 +36,9 @@ export class ProvingAgent {
     private proofAllowList: Array<ProvingRequestType> = [],
     /** How long to wait between jobs */
     private pollIntervalMs = 1000,
-    private log = createLogger('prover-client:proving-agent'),
+    bindings?: LoggerBindings,
   ) {
+    this.log = createLogger('prover-client:proving-agent', bindings);
     this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollIntervalMs);
   }
 
@@ -159,6 +161,7 @@ export class ProvingAgent {
         // no need to await this here. The controller will stay alive (in DONE state) until the result is send to the broker
         void this.runningPromise.trigger();
       },
+      this.log.getBindings(),
     );
 
     if (abortedProofJobId) {

package/src/proving_broker/proving_broker.ts

@@ -7,6 +7,7 @@ import {
   type GetProvingJobResponse,
   type ProofUri,
   type ProvingJob,
+  type ProvingJobBrokerDebug,
   type ProvingJobConsumer,
   type ProvingJobFilter,
   type ProvingJobId,
@@ -36,7 +37,7 @@ type EnqueuedProvingJob = Pick<ProvingJob, 'id' | 'epochNumber'>;
  * A broker that manages proof requests and distributes them to workers based on their priority.
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
  */
-export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable {
+export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, ProvingJobBrokerDebug, Traceable {
   private queues: ProvingQueues = {
     [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
     [ProvingRequestType.PUBLIC_CHONK_VERIFIER]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
@@ -114,6 +115,8 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
 
   private started = false;
 
+  private debugReplayEnabled: boolean;
+
   public constructor(
     private database: ProvingBrokerDatabase,
     {
@@ -121,6 +124,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
       proverBrokerPollIntervalMs,
       proverBrokerJobMaxRetries,
       proverBrokerMaxEpochsToKeepResultsFor,
+      proverBrokerDebugReplayEnabled,
     }: Required<
       Pick<
         ProverBrokerConfig,
@@ -128,6 +132,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
         | 'proverBrokerPollIntervalMs'
         | 'proverBrokerJobMaxRetries'
         | 'proverBrokerMaxEpochsToKeepResultsFor'
+        | 'proverBrokerDebugReplayEnabled'
       >
     > = defaultProverBrokerConfig,
     client: TelemetryClient = getTelemetryClient(),
@@ -139,6 +144,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
     this.jobTimeoutMs = proverBrokerJobTimeoutMs!;
     this.maxRetries = proverBrokerJobMaxRetries!;
     this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor!;
+    this.debugReplayEnabled = proverBrokerDebugReplayEnabled ?? false;
   }
 
   private measureQueueDepth: MonitorCallback = (type: ProvingRequestType) => {
@@ -241,6 +247,29 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
     return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter));
   }
 
+  public async replayProvingJob(
+    jobId: ProvingJobId,
+    type: ProvingRequestType,
+    epochNumber: EpochNumber,
+    inputsUri: ProofUri,
+  ): Promise<ProvingJobStatus> {
+    if (!this.debugReplayEnabled) {
+      throw new Error('Debug replay not enabled. Set PROVER_BROKER_DEBUG_REPLAY_ENABLED=true');
+    }
+
+    this.logger.info(`Replaying proving job`, { provingJobId: jobId, epochNumber, inputsUri });
+
+    // Clear existing state and enqueue
+    this.cleanUpProvingJobState([jobId]);
+
+    const job: ProvingJob = { id: jobId, type, epochNumber, inputsUri };
+    this.jobsCache.set(jobId, job);
+    await this.database.addProvingJob(job);
+    this.enqueueJobInternal(job);
+
+    return { status: 'in-queue' };
+  }
+
   async #enqueueProvingJob(job: ProvingJob): Promise<ProvingJobStatus> {
     // We return the job status at the start of this call
     const jobStatus = this.#getProvingJobStatus(job.id);
@@ -285,7 +314,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
     // notify listeners of the cancellation
     if (!this.resultsCache.has(id)) {
       this.logger.info(`Cancelling job id=${id}`, { provingJobId: id });
-      await this.#reportProvingJobError(id, 'Aborted', false);
+      await this.#reportProvingJobError(id, 'Aborted', false, undefined, true);
     }
   }
 
@@ -366,6 +395,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
     err: string,
     retry = false,
     filter?: ProvingJobFilter,
+    aborted = false,
   ): Promise<GetProvingJobResponse | undefined> {
     const info = this.inProgress.get(id);
     const item = this.jobsCache.get(id);
@@ -426,7 +456,11 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
     this.promises.get(id)!.resolve(result);
     this.completedJobNotifications.push(id);
 
-    this.instrumentation.incRejectedJobs(item.type);
+    if (aborted) {
+      this.instrumentation.incAbortedJobs(item.type);
+    } else {
+      this.instrumentation.incRejectedJobs(item.type);
+    }
     if (info) {
       const duration = this.msTimeSource() - info.startedAt;
       this.instrumentation.recordJobDuration(item.type, duration);

package/src/proving_broker/proving_broker_instrumentation.ts

@@ -8,6 +8,7 @@ import {
   type ObservableResult,
   type TelemetryClient,
   type UpDownCounter,
+  createUpDownCounterWithDefault,
 } from '@aztec/telemetry-client';
 
 export type MonitorCallback = (proofType: ProvingRequestType) => number;
@@ -17,6 +18,7 @@ export class ProvingBrokerInstrumentation {
   private activeJobs: ObservableGauge;
   private resolvedJobs: UpDownCounter;
   private rejectedJobs: UpDownCounter;
+  private abortedJobs: UpDownCounter;
   private timedOutJobs: UpDownCounter;
   private cachedJobs: UpDownCounter;
   private totalJobs: UpDownCounter;
@@ -31,17 +33,22 @@ export class ProvingBrokerInstrumentation {
 
     this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS);
 
-    this.resolvedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RESOLVED_JOBS);
+    const provingJobTypes = Object.values(ProvingRequestType).filter(v => typeof v === 'string');
+    const provingJobAttrs = { [Attributes.PROVING_JOB_TYPE]: provingJobTypes };
 
-    this.rejectedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_REJECTED_JOBS);
+    this.resolvedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RESOLVED_JOBS, provingJobAttrs);
 
-    this.retriedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RETRIED_JOBS);
+    this.rejectedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_REJECTED_JOBS, provingJobAttrs);
 
-    this.timedOutJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TIMED_OUT_JOBS);
+    this.abortedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_ABORTED_JOBS, provingJobAttrs);
 
-    this.cachedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_CACHED_JOBS);
+    this.retriedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RETRIED_JOBS, provingJobAttrs);
 
-    this.totalJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TOTAL_JOBS);
+    this.timedOutJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, provingJobAttrs);
+
+    this.cachedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_CACHED_JOBS, provingJobAttrs);
+
+    this.totalJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TOTAL_JOBS, provingJobAttrs);
 
     this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT);
 
@@ -68,6 +75,12 @@ export class ProvingBrokerInstrumentation {
     });
   }
 
+  incAbortedJobs(proofType: ProvingRequestType) {
+    this.abortedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
   incRetriedJobs(proofType: ProvingRequestType) {
     this.retriedJobs.add(1, {
       [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],

package/src/proving_broker/proving_job_controller.ts

@@ -1,7 +1,7 @@
 import { EpochNumber } from '@aztec/foundation/branded-types';
 import { randomBytes } from '@aztec/foundation/crypto/random';
 import { AbortError } from '@aztec/foundation/error';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import type {
   ProvingJobId,
   ProvingJobInputs,
@@ -21,6 +21,7 @@ export class ProvingJobController {
   private promise?: Promise<void>;
   private abortController = new AbortController();
   private result?: ProvingJobResultsMap[ProvingRequestType] | Error;
+  private log: Logger;
 
   constructor(
     private jobId: ProvingJobId,
@@ -29,8 +30,13 @@ export class ProvingJobController {
     private startedAt: number,
     private circuitProver: ServerCircuitProver,
     private onComplete: () => void,
-    private log = createLogger('prover-client:proving-agent:job-controller-' + randomBytes(4).toString('hex')),
-  ) {}
+    bindings?: LoggerBindings,
+  ) {
+    this.log = createLogger('prover-client:proving-agent:job-controller', {
+      instanceId: randomBytes(4).toString('hex'),
+      ...bindings,
+    });
+  }
 
   public start(): void {
     if (this.status !== ProvingJobControllerStatus.IDLE) {

package/src/proving_broker/rpc.ts

@@ -1,9 +1,11 @@
+import { EpochNumberSchema } from '@aztec/foundation/branded-types';
 import { createSafeJsonRpcClient } from '@aztec/foundation/json-rpc/client';
 import {
   type GetProvingJobResponse,
   ProofUri,
   ProvingJob,
   type ProvingJobBroker,
+  type ProvingJobBrokerDebug,
   type ProvingJobConsumer,
   ProvingJobId,
   type ProvingJobProducer,
@@ -16,6 +18,15 @@ import { makeTracedFetch } from '@aztec/telemetry-client';
 
 import { z } from 'zod';
 
+/** Indefinite backoff for broker communication: 1, 1, 1, 2, 4, 4, 4, ... seconds. */
+export function* proverBrokerBackoff() {
+  const v = [1, 1, 1, 2, 4];
+  let i = 0;
+  while (true) {
+    yield v[Math.min(i++, v.length - 1)];
+  }
+}
+
 const ProvingJobFilterSchema = z.object({
   allowList: z.array(z.nativeEnum(ProvingRequestType)),
 });
@@ -53,10 +64,22 @@ export const ProvingJobBrokerSchema: ApiSchemaFor<ProvingJobBroker> = {
   ...ProvingJobProducerSchema,
 };
 
+export const ProvingJobBrokerDebugSchema: ApiSchemaFor<ProvingJobBrokerDebug> = {
+  replayProvingJob: z
+    .function()
+    .args(ProvingJobId, z.nativeEnum(ProvingRequestType), EpochNumberSchema, ProofUri)
+    .returns(ProvingJobStatus),
+};
+
+export const ProvingJobBrokerSchemaWithDebug: ApiSchemaFor<ProvingJobBroker & ProvingJobBrokerDebug> = {
+  ...ProvingJobBrokerSchema,
+  ...ProvingJobBrokerDebugSchema,
+};
+
 export function createProvingJobBrokerClient(
   url: string,
   versions: Partial<ComponentsVersions>,
-  fetch = makeTracedFetch([1, 2, 3], false),
+  fetch = makeTracedFetch(proverBrokerBackoff, false),
 ): ProvingJobBroker {
   return createSafeJsonRpcClient(url, ProvingJobBrokerSchema, {
     namespaceMethods: 'proverBroker',
@@ -68,7 +91,7 @@ export function createProvingJobBrokerClient(
 export function createProvingJobProducerClient(
   url: string,
   versions: Partial<ComponentsVersions>,
-  fetch = makeTracedFetch([1, 2, 3], false),
+  fetch = makeTracedFetch(proverBrokerBackoff, false),
 ): ProvingJobProducer {
   return createSafeJsonRpcClient(url, ProvingJobProducerSchema, {
     namespaceMethods: 'provingJobProducer',
@@ -80,7 +103,7 @@ export function createProvingJobProducerClient(
 export function createProvingJobConsumerClient(
   url: string,
   versions: Partial<ComponentsVersions>,
-  fetch = makeTracedFetch([1, 2, 3], false),
+  fetch = makeTracedFetch(proverBrokerBackoff, false),
 ): ProvingJobConsumer {
   return createSafeJsonRpcClient(url, ProvingJobConsumerSchema, {
     namespaceMethods: 'provingJobConsumer',

package/dest/proving_broker/proof_store/gcs_proof_store.d.ts

@@ -1,14 +0,0 @@
-import { type ProofUri, type ProvingJobId, type ProvingJobInputs, type ProvingJobInputsMap, type ProvingJobResult, type ProvingJobResultsMap } from '@aztec/stdlib/interfaces/server';
-import { ProvingRequestType } from '@aztec/stdlib/proofs';
-import type { ProofStore } from './proof_store.js';
-export declare class GoogleCloudStorageProofStore implements ProofStore {
-    private readonly bucketName;
-    private readonly path;
-    private readonly storage;
-    constructor(bucketName: string, path: string);
-    saveProofInput<T extends ProvingRequestType>(id: ProvingJobId, type: T, inputs: ProvingJobInputsMap[T]): Promise<ProofUri>;
-    saveProofOutput<T extends ProvingRequestType>(_id: ProvingJobId, _type: T, _result: ProvingJobResultsMap[T]): Promise<ProofUri>;
-    getProofInput(uri: ProofUri): Promise<ProvingJobInputs>;
-    getProofOutput(_uri: ProofUri): Promise<ProvingJobResult>;
-}
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2NzX3Byb29mX3N0b3JlLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvb2Zfc3RvcmUvZ2NzX3Byb29mX3N0b3JlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFDTCxLQUFLLFFBQVEsRUFDYixLQUFLLFlBQVksRUFDakIsS0FBSyxnQkFBZ0IsRUFDckIsS0FBSyxtQkFBbUIsRUFDeEIsS0FBSyxnQkFBZ0IsRUFDckIsS0FBSyxvQkFBb0IsRUFFMUIsTUFBTSxpQ0FBaUMsQ0FBQztBQUN6QyxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUsxRCxPQUFPLEtBQUssRUFBRSxVQUFVLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUtuRCxxQkFBYSw0QkFBNkIsWUFBVyxVQUFVO0lBSTNELE9BQU8sQ0FBQyxRQUFRLENBQUMsVUFBVTtJQUMzQixPQUFPLENBQUMsUUFBUSxDQUFDLElBQUk7SUFKdkIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQVU7SUFFbEMsWUFDbUIsVUFBVSxFQUFFLE1BQU0sRUFDbEIsSUFBSSxFQUFFLE1BQU0sRUFHOUI7SUFFWSxjQUFjLENBQUMsQ0FBQyxTQUFTLGtCQUFrQixFQUN0RCxFQUFFLEVBQUUsWUFBWSxFQUNoQixJQUFJLEVBQUUsQ0FBQyxFQUNQLE1BQU0sRUFBRSxtQkFBbUIsQ0FBQyxDQUFDLENBQUMsR0FDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxDQUtuQjtJQUVELGVBQWUsQ0FBQyxDQUFDLFNBQVMsa0JBQWtCLEVBQzFDLEdBQUcsRUFBRSxZQUFZLEVBQ2pCLEtBQUssRUFBRSxDQUFDLEVBQ1IsT0FBTyxFQUFFLG9CQUFvQixDQUFDLENBQUMsQ0FBQyxHQUMvQixPQUFPLENBQUMsUUFBUSxDQUFDLENBRW5CO0lBRVksYUFBYSxDQUFDLEdBQUcsRUFBRSxRQUFRLEdBQUcsT0FBTyxDQUFDLGdCQUFnQixDQUFDLENBc0JuRTtJQUVELGNBQWMsQ0FBQyxJQUFJLEVBQUUsUUFBUSxHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxDQUV4RDtDQUNGIn0=

package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"gcs_proof_store.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proof_store/gcs_proof_store.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACrB,KAAK,oBAAoB,EAE1B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAK1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAKnD,qBAAa,4BAA6B,YAAW,UAAU;IAI3D,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAJvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAElC,YACmB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EAG9B;IAEY,cAAc,CAAC,CAAC,SAAS,kBAAkB,EACtD,EAAE,EAAE,YAAY,EAChB,IAAI,EAAE,CAAC,EACP,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAC7B,OAAO,CAAC,QAAQ,CAAC,CAKnB;IAED,eAAe,CAAC,CAAC,SAAS,kBAAkB,EAC1C,GAAG,EAAE,YAAY,EACjB,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,QAAQ,CAAC,CAEnB;IAEY,aAAa,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAsBnE;IAED,cAAc,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAExD;CACF"}