@aztec/prover-client 0.0.1-commit.fcb71a6 → 0.0.1-commit.ff7989d6c

package/src/orchestrator/orchestrator_metrics.ts

@@ -1,4 +1,4 @@
-import { type Histogram, Metrics, type TelemetryClient, type Tracer, ValueType } from '@aztec/telemetry-client';
+import { type Histogram, Metrics, type TelemetryClient, type Tracer } from '@aztec/telemetry-client';
 
 export class ProvingOrchestratorMetrics {
   public readonly tracer: Tracer;
@@ -9,11 +9,7 @@ export class ProvingOrchestratorMetrics {
     this.tracer = client.getTracer(name);
     const meter = client.getMeter(name);
 
-    this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION, {
-      unit: 'ms',
-      description: 'Duration to build base rollup inputs',
-      valueType: ValueType.INT,
-    });
+    this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION);
   }
 
   recordBaseRollupInputs(durationMs: number) {

package/src/orchestrator/tx-proving-state.ts

@@ -2,8 +2,8 @@ import { AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED, NESTED_RECURSIVE_ROLLUP_HONK_PROO
 import type { Fr } from '@aztec/foundation/curves/bn254';
 import { getVkData } from '@aztec/noir-protocol-circuits-types/server/vks';
 import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
-import type { ProofAndVerificationKey, PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
-import { ProofData, ProofDataForFixedVk } from '@aztec/stdlib/proofs';
+import type { PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
+import { ProofData, ProofDataForFixedVk, RecursiveProof } from '@aztec/stdlib/proofs';
 import {
   type BaseRollupHints,
   PrivateBaseRollupHints,
@@ -32,7 +32,7 @@ export class TxProvingState {
     PublicChonkVerifierPublicInputs,
     typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH
   >;
-  private avm?: ProofAndVerificationKey<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>;
+  private avmProof?: RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>;
 
   constructor(
     public readonly processedTx: ProcessedTx,
@@ -46,7 +46,7 @@ export class TxProvingState {
   }
 
   public ready() {
-    return !this.requireAvmProof || (!!this.avm && !!this.publicChonkVerifier);
+    return !this.requireAvmProof || (!!this.avmProof && !!this.publicChonkVerifier);
   }
 
   public getAvmInputs(): AvmCircuitInputs {
@@ -80,8 +80,8 @@ export class TxProvingState {
     this.publicChonkVerifier = publicChonkVerifierProofAndVk;
   }
 
-  public setAvmProof(avmProofAndVk: ProofAndVerificationKey<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>) {
-    this.avm = avmProofAndVk;
+  public setAvmProof(avmProof: RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>) {
+    this.avmProof = avmProof;
   }
 
   #getPrivateBaseInputs() {
@@ -105,7 +105,7 @@ export class TxProvingState {
     if (!this.publicChonkVerifier) {
       throw new Error('Tx not ready for proving base rollup: public chonk verifier proof undefined');
     }
-    if (!this.avm) {
+    if (!this.avmProof) {
       throw new Error('Tx not ready for proving base rollup: avm proof undefined');
     }
     if (!(this.baseRollupHints instanceof PublicBaseRollupHints)) {
@@ -114,10 +114,7 @@ export class TxProvingState {
 
     const publicChonkVerifierProofData = toProofData(this.publicChonkVerifier);
 
-    const avmProofData = new ProofDataForFixedVk(
-      this.processedTx.avmProvingRequest.inputs.publicInputs,
-      this.avm.proof,
-    );
+    const avmProofData = new ProofDataForFixedVk(this.processedTx.avmProvingRequest.inputs.publicInputs, this.avmProof);
 
     return new PublicTxBaseRollupPrivateInputs(publicChonkVerifierProofData, avmProofData, this.baseRollupHints);
   }

package/src/prover-client/prover-client.ts

@@ -1,7 +1,7 @@
 import { type ACVMConfig, type BBConfig, BBNativeRollupProver, TestCircuitProver } from '@aztec/bb-prover';
 import { times } from '@aztec/foundation/collection';
 import type { EthAddress } from '@aztec/foundation/eth-address';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, createLogger } from '@aztec/foundation/log';
 import { NativeACVMSimulator } from '@aztec/simulator/server';
 import {
   type ActualProverConfig,
@@ -29,24 +29,34 @@ export class ProverClient implements EpochProverManager {
   private running = false;
   private agents: ProvingAgent[] = [];
 
-  private proofStore: ProofStore;
-  private failedProofStore: ProofStore | undefined;
-
   private constructor(
     private config: ProverClientConfig,
     private worldState: ForkMerkleTreeOperations & ReadonlyWorldStateAccess,
     private orchestratorClient: ProvingJobProducer,
+    private proofStore: ProofStore,
+    private failedProofStore: ProofStore | undefined,
     private agentClient?: ProvingJobConsumer,
     private telemetry: TelemetryClient = getTelemetryClient(),
-    private log = createLogger('prover-client:tx-prover'),
-  ) {
-    this.proofStore = new InlineProofStore();
-    this.failedProofStore = this.config.failedProofStore ? createProofStore(this.config.failedProofStore) : undefined;
-  }
+    private log: Logger = createLogger('prover-client:tx-prover'),
+  ) {}
 
   public createEpochProver(): EpochProver {
-    const facade = new BrokerCircuitProverFacade(this.orchestratorClient, this.proofStore, this.failedProofStore);
-    const orchestrator = new ProvingOrchestrator(this.worldState, facade, this.config.proverId, this.telemetry);
+    const bindings = this.log.getBindings();
+    const facade = new BrokerCircuitProverFacade(
+      this.orchestratorClient,
+      this.proofStore,
+      this.failedProofStore,
+      undefined,
+      bindings,
+    );
+    const orchestrator = new ProvingOrchestrator(
+      this.worldState,
+      facade,
+      this.config.proverId,
+      this.config.cancelJobsOnStop,
+      this.telemetry,
+      bindings,
+    );
     return new ServerEpochProver(facade, orchestrator);
   }
 
@@ -104,7 +114,9 @@ export class ProverClient implements EpochProverManager {
     broker: ProvingJobBroker,
     telemetry: TelemetryClient = getTelemetryClient(),
   ) {
-    const prover = new ProverClient(config, worldState, broker, broker, telemetry);
+    const proofStore = await createProofStore(config.proofStore);
+    const failedProofStore = config.failedProofStore ? await createProofStore(config.failedProofStore) : undefined;
+    const prover = new ProverClient(config, worldState, broker, proofStore, failedProofStore, broker, telemetry);
     await prover.start();
     return prover;
   }
@@ -128,17 +140,11 @@ export class ProverClient implements EpochProverManager {
 
     const proofStore = new InlineProofStore();
     const prover = await buildServerCircuitProver(this.config, this.telemetry);
+    const bindings = this.log.getBindings();
     this.agents = times(
       this.config.proverAgentCount,
       () =>
-        new ProvingAgent(
-          this.agentClient!,
-          proofStore,
-          prover,
-          [],
-          this.config.proverAgentPollIntervalMs,
-          this.telemetry,
-        ),
+        new ProvingAgent(this.agentClient!, proofStore, prover, [], this.config.proverAgentPollIntervalMs, bindings),
     );
 
     await Promise.all(this.agents.map(agent => agent.start()));
@@ -157,8 +163,9 @@ export function buildServerCircuitProver(
     return BBNativeRollupProver.new(config, telemetry);
   }
 
+  const logger = createLogger('prover-client:acvm-native');
   const simulator = config.acvmBinaryPath
-    ? new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath)
+    ? new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath, undefined, logger)
     : undefined;
 
   return Promise.resolve(new TestCircuitProver(simulator, config, telemetry));

package/src/proving_broker/broker_prover_facade.ts

@@ -6,12 +6,11 @@ import type {
 } from '@aztec/constants';
 import { EpochNumber } from '@aztec/foundation/branded-types';
 import { sha256 } from '@aztec/foundation/crypto/sha256';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { truncate } from '@aztec/foundation/string';
 import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
 import {
-  type ProofAndVerificationKey,
   type ProofUri,
   type ProvingJobId,
   type ProvingJobInputsMap,
@@ -23,7 +22,7 @@ import {
   makeProvingJobId,
 } from '@aztec/stdlib/interfaces/server';
 import type { ParityBasePrivateInputs, ParityPublicInputs, ParityRootPrivateInputs } from '@aztec/stdlib/parity';
-import { ProvingRequestType } from '@aztec/stdlib/proofs';
+import { ProvingRequestType, RecursiveProof } from '@aztec/stdlib/proofs';
 import type {
   BlockMergeRollupPrivateInputs,
   BlockRollupPublicInputs,
@@ -69,14 +68,17 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
   private runningPromise?: RunningPromise;
   private timeOfLastSnapshotSync = Date.now();
   private jobsToRetrieve: Set<ProvingJobId> = new Set();
+  private log: Logger;
 
   constructor(
     private broker: ProvingJobProducer,
     private proofStore: ProofStore = new InlineProofStore(),
     private failedProofStore?: ProofStore,
     private pollIntervalMs = 1000,
-    private log = createLogger('prover-client:broker-circuit-prover-facade'),
-  ) {}
+    bindings?: LoggerBindings,
+  ) {
+    this.log = createLogger('prover-client:broker-circuit-prover-facade', bindings);
+  }
 
   /**
    * This is a critical section. This function can not be async since it writes
@@ -399,7 +401,7 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
     inputs: AvmCircuitInputs,
     signal?: AbortSignal,
     epochNumber?: EpochNumber,
-  ): Promise<ProofAndVerificationKey<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>> {
+  ): Promise<RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>> {
     return this.enqueueJob(
       this.generateId(ProvingRequestType.PUBLIC_VM, inputs, epochNumber),
       ProvingRequestType.PUBLIC_VM,

package/src/proving_broker/config.ts

@@ -31,6 +31,8 @@ export const ProverBrokerConfig = z.object({
   proverBrokerBatchIntervalMs: z.number().int().nonnegative(),
   /** The maximum number of epochs to keep results for */
   proverBrokerMaxEpochsToKeepResultsFor: z.number().int().nonnegative(),
+  /** Enable debug replay mode for replaying proving jobs from stored inputs */
+  proverBrokerDebugReplayEnabled: z.boolean(),
 });
 
 export type ProverBrokerConfig = z.infer<typeof ProverBrokerConfig> &
@@ -74,6 +76,11 @@ export const proverBrokerConfigMappings: ConfigMappingsType<ProverBrokerConfig>
     parseEnv: (val: string | undefined) => (val ? +val : undefined),
     description: "The size of the prover broker's database. Will override the dataStoreMapSizeKb if set.",
   },
+  proverBrokerDebugReplayEnabled: {
+    env: 'PROVER_BROKER_DEBUG_REPLAY_ENABLED',
+    description: 'Enable debug replay mode for replaying proving jobs from stored inputs',
+    ...booleanConfigHelper(false),
+  },
   ...dataConfigMappings,
   ...l1ReaderConfigMappings,
   ...pickConfigMappings(chainConfigMappings, ['rollupVersion']),
@@ -100,6 +107,10 @@ export const ProverAgentConfig = z.object({
   proverTestDelayFactor: z.number(),
   /** The delay (ms) to inject during fake proof verification */
   proverTestVerificationDelayMs: z.number().optional(),
+  /** Whether to abort pending proving jobs when the orchestrator is cancelled */
+  cancelJobsOnStop: z.boolean(),
+  /** Where to store proving results. Must be accessible to both prover node and agents. If not set will inline-encode the parameters */
+  proofStore: z.string().optional(),
 });
 
 export type ProverAgentConfig = z.infer<typeof ProverAgentConfig>;
@@ -153,4 +164,15 @@ export const proverAgentConfigMappings: ConfigMappingsType<ProverAgentConfig> =
     description: 'The delay (ms) to inject during fake proof verification',
     ...numberConfigHelper(10),
   },
+  cancelJobsOnStop: {
+    env: 'PROVER_CANCEL_JOBS_ON_STOP',
+    description:
+      'Whether to abort pending proving jobs when the orchestrator is cancelled. ' +
+      'When false (default), jobs remain in the broker queue and can be reused on restart/reorg.',
+    ...booleanConfigHelper(false),
+  },
+  proofStore: {
+    env: 'PROVER_PROOF_STORE',
+    description: 'Optional proof input store for the prover',
+  },
 };

package/src/proving_broker/proof_store/factory.ts

@@ -1,42 +1,20 @@
 import { createLogger } from '@aztec/foundation/log';
+import { createFileStore } from '@aztec/stdlib/file-store';
 
-import { GoogleCloudStorageProofStore } from './gcs_proof_store.js';
+import { FileStoreProofStore } from './file_store_proof_store.js';
 import { InlineProofStore } from './inline_proof_store.js';
 import type { ProofStore } from './proof_store.js';
 
-export function createProofStore(config: string | undefined, logger = createLogger('prover-client:proof-store')) {
-  if (config === undefined) {
+export async function createProofStore(
+  config: string | undefined,
+  logger = createLogger('prover-client:proof-store'),
+): Promise<ProofStore> {
+  if (!config) {
     logger.info('Creating inline proof store');
     return new InlineProofStore();
-  } else if (config.startsWith('gs://')) {
-    try {
-      const url = new URL(config);
-      const bucket = url.host;
-      const path = url.pathname.replace(/^\/+/, '');
-      logger.info(`Creating google cloud proof store at ${bucket}`, { bucket, path });
-      return new GoogleCloudStorageProofStore(bucket, path);
-    } catch {
-      throw new Error(
-        `Invalid google cloud proof store definition: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`,
-      );
-    }
-  } else {
-    throw new Error(`Unknown proof store config: '${config}'. Supported values are 'gs://bucket-name/path/to/store'.`);
   }
-}
 
-export function createProofStoreForUri(
-  uri: string,
-  logger = createLogger('prover-client:proof-store'),
-): Pick<ProofStore, 'getProofInput' | 'getProofOutput'> {
-  if (uri.startsWith('data://')) {
-    return createProofStore(undefined, logger);
-  } else if (uri.startsWith('gs://')) {
-    const url = new URL(uri);
-    const basePath = url.pathname.replace(/^\/+/, '').split('/').slice(0, -3);
-    url.pathname = basePath.join('/');
-    return createProofStore(uri, logger);
-  } else {
-    throw new Error(`Unknown proof store config: '${uri}'. Supported protocols are 'data://' and 'gs://'.`);
-  }
+  const fileStore = await createFileStore(config, logger);
+  logger.info(`Creating file store proof store at ${config}`);
+  return new FileStoreProofStore(fileStore);
 }

package/src/proving_broker/proof_store/file_store_proof_store.ts

@@ -0,0 +1,78 @@
+import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
+import type { FileStore } from '@aztec/stdlib/file-store';
+import {
+  type ProofUri,
+  type ProvingJobId,
+  type ProvingJobInputs,
+  type ProvingJobInputsMap,
+  ProvingJobResult,
+  type ProvingJobResultsMap,
+  getProvingJobInputClassFor,
+} from '@aztec/stdlib/interfaces/server';
+import { ProvingRequestType } from '@aztec/stdlib/proofs';
+
+import type { ProofStore } from './proof_store.js';
+
+const INPUTS_PATH = 'inputs';
+const OUTPUTS_PATH = 'outputs';
+
+/**
+ * A proof store implementation backed by a generic FileStore.
+ * Supports any storage backend (GCS, S3, local filesystem) via the FileStore abstraction.
+ */
+export class FileStoreProofStore implements ProofStore {
+  constructor(private readonly fileStore: FileStore) {}
+
+  async saveProofInput<T extends ProvingRequestType>(
+    id: ProvingJobId,
+    type: T,
+    inputs: ProvingJobInputsMap[T],
+  ): Promise<ProofUri> {
+    const path = `${INPUTS_PATH}/${ProvingRequestType[type]}/${id}`;
+    const uri = await this.fileStore.save(path, inputs.toBuffer());
+    return uri as ProofUri;
+  }
+
+  async saveProofOutput<T extends ProvingRequestType>(
+    id: ProvingJobId,
+    type: T,
+    result: ProvingJobResultsMap[T],
+  ): Promise<ProofUri> {
+    const jobResult = { type, result } as ProvingJobResult;
+    const json = jsonStringify(jobResult);
+    const path = `${OUTPUTS_PATH}/${ProvingRequestType[type]}/${id}.json`;
+    const uri = await this.fileStore.save(path, Buffer.from(json, 'utf-8'));
+    return uri as ProofUri;
+  }
+
+  async getProofInput(uri: ProofUri): Promise<ProvingJobInputs> {
+    try {
+      const buffer = await this.fileStore.read(uri);
+      const type = this.extractTypeFromUri(uri);
+      const inputs = getProvingJobInputClassFor(type).fromBuffer(buffer);
+      return { inputs, type } as ProvingJobInputs;
+    } catch (err) {
+      throw new Error(`Error getting proof input at ${uri}: ${err}`);
+    }
+  }
+
+  async getProofOutput(uri: ProofUri): Promise<ProvingJobResult> {
+    try {
+      const buffer = await this.fileStore.read(uri);
+      return jsonParseWithSchema(buffer.toString('utf-8'), ProvingJobResult);
+    } catch (err) {
+      throw new Error(`Error getting proof output at ${uri}: ${err}`);
+    }
+  }
+
+  private extractTypeFromUri(uri: string): ProvingRequestType {
+    const url = new URL(uri);
+    const pathParts = url.pathname.split('/').filter(Boolean);
+    const typeString = pathParts.at(-2);
+    const type = typeString ? ProvingRequestType[typeString as keyof typeof ProvingRequestType] : undefined;
+    if (type === undefined) {
+      throw new Error(`Unrecognized proof type ${typeString} in URI ${uri}`);
+    }
+    return type;
+  }
+}

package/src/proving_broker/proof_store/index.ts

@@ -1,4 +1,4 @@
 export * from './proof_store.js';
 export * from './inline_proof_store.js';
 export * from './factory.js';
-export * from './gcs_proof_store.js';
+export * from './file_store_proof_store.js';

package/src/proving_broker/proving_agent.ts

@@ -1,5 +1,5 @@
 import { AbortError } from '@aztec/foundation/error';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 import { truncate } from '@aztec/foundation/string';
 import { ProvingError } from '@aztec/stdlib/errors';
@@ -13,27 +13,17 @@ import type {
   ServerCircuitProver,
 } from '@aztec/stdlib/interfaces/server';
 import { ProvingRequestType } from '@aztec/stdlib/proofs';
-import {
-  type TelemetryClient,
-  type Traceable,
-  type Tracer,
-  getTelemetryClient,
-  trackSpan,
-} from '@aztec/telemetry-client';
 
 import type { ProofStore } from './proof_store/index.js';
-import { ProvingAgentInstrumentation } from './proving_agent_instrumentation.js';
 import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_controller.js';
 
 /**
  * A helper class that encapsulates a circuit prover and connects it to a job source.
  */
-export class ProvingAgent implements Traceable {
+export class ProvingAgent {
   private currentJobController?: ProvingJobController;
   private runningPromise: RunningPromise;
-  private instrumentation: ProvingAgentInstrumentation;
-
-  public readonly tracer: Tracer;
+  private log: Logger;
 
   constructor(
     /** The source of proving jobs */
@@ -46,12 +36,9 @@ export class ProvingAgent implements Traceable {
     private proofAllowList: Array<ProvingRequestType> = [],
     /** How long to wait between jobs */
     private pollIntervalMs = 1000,
-    /** A telemetry client through which to emit metrics */
-    client: TelemetryClient = getTelemetryClient(),
-    private log = createLogger('prover-client:proving-agent'),
+    bindings?: LoggerBindings,
   ) {
-    this.tracer = client.getTracer('ProvingAgent');
-    this.instrumentation = new ProvingAgentInstrumentation(client);
+    this.log = createLogger('prover-client:proving-agent', bindings);
     this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollIntervalMs);
   }
 
@@ -85,7 +72,6 @@ export class ProvingAgent implements Traceable {
     return this.runningPromise.isRunning() ? { status: 'running' } : { status: 'stopped' };
   }
 
-  @trackSpan('ProvingAgent.safeWork')
   private async work() {
     // every tick we need to take one of the following actions:
     // 1. send a hearbeat to the broker that we're working on some job
@@ -175,6 +161,7 @@ export class ProvingAgent implements Traceable {
         // no need to await this here. The controller will stay alive (in DONE state) until the result is send to the broker
         void this.runningPromise.trigger();
       },
+      this.log.getBindings(),
     );
 
     if (abortedProofJobId) {

package/src/proving_broker/proving_broker.ts

@@ -7,6 +7,7 @@ import {
   type GetProvingJobResponse,
   type ProofUri,
   type ProvingJob,
+  type ProvingJobBrokerDebug,
   type ProvingJobConsumer,
   type ProvingJobFilter,
   type ProvingJobId,
@@ -16,13 +17,7 @@ import {
   tryStop,
 } from '@aztec/stdlib/interfaces/server';
 import { ProvingRequestType } from '@aztec/stdlib/proofs';
-import {
-  type TelemetryClient,
-  type Traceable,
-  type Tracer,
-  getTelemetryClient,
-  trackSpan,
-} from '@aztec/telemetry-client';
+import { type TelemetryClient, type Traceable, type Tracer, getTelemetryClient } from '@aztec/telemetry-client';
 
 import assert from 'assert';
 
@@ -42,7 +37,7 @@ type EnqueuedProvingJob = Pick<ProvingJob, 'id' | 'epochNumber'>;
  * A broker that manages proof requests and distributes them to workers based on their priority.
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
  */
-export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable {
+export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, ProvingJobBrokerDebug, Traceable {
   private queues: ProvingQueues = {
     [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
     [ProvingRequestType.PUBLIC_CHONK_VERIFIER]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
@@ -120,6 +115,8 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
 
   private started = false;
 
+  private debugReplayEnabled: boolean;
+
   public constructor(
     private database: ProvingBrokerDatabase,
     {
@@ -127,6 +124,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
       proverBrokerPollIntervalMs,
       proverBrokerJobMaxRetries,
       proverBrokerMaxEpochsToKeepResultsFor,
+      proverBrokerDebugReplayEnabled,
     }: Required<
       Pick<
         ProverBrokerConfig,
@@ -134,6 +132,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
         | 'proverBrokerPollIntervalMs'
         | 'proverBrokerJobMaxRetries'
         | 'proverBrokerMaxEpochsToKeepResultsFor'
+        | 'proverBrokerDebugReplayEnabled'
       >
     > = defaultProverBrokerConfig,
     client: TelemetryClient = getTelemetryClient(),
@@ -145,6 +144,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
     this.jobTimeoutMs = proverBrokerJobTimeoutMs!;
     this.maxRetries = proverBrokerJobMaxRetries!;
     this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor!;
+    this.debugReplayEnabled = proverBrokerDebugReplayEnabled ?? false;
   }
 
   private measureQueueDepth: MonitorCallback = (type: ProvingRequestType) => {
@@ -247,6 +247,29 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
     return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter));
   }
 
+  public async replayProvingJob(
+    jobId: ProvingJobId,
+    type: ProvingRequestType,
+    epochNumber: EpochNumber,
+    inputsUri: ProofUri,
+  ): Promise<ProvingJobStatus> {
+    if (!this.debugReplayEnabled) {
+      throw new Error('Debug replay not enabled. Set PROVER_BROKER_DEBUG_REPLAY_ENABLED=true');
+    }
+
+    this.logger.info(`Replaying proving job`, { provingJobId: jobId, epochNumber, inputsUri });
+
+    // Clear existing state and enqueue
+    this.cleanUpProvingJobState([jobId]);
+
+    const job: ProvingJob = { id: jobId, type, epochNumber, inputsUri };
+    this.jobsCache.set(jobId, job);
+    await this.database.addProvingJob(job);
+    this.enqueueJobInternal(job);
+
+    return { status: 'in-queue' };
+  }
+
   async #enqueueProvingJob(job: ProvingJob): Promise<ProvingJobStatus> {
     // We return the job status at the start of this call
     const jobStatus = this.#getProvingJobStatus(job.id);
@@ -565,7 +588,6 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
     return this.#getProvingJob(filter);
   }
 
-  @trackSpan('ProvingBroker.cleanupPass')
   private async cleanupPass() {
     this.cleanupStaleJobs();
     this.reEnqueueExpiredJobs();

package/src/proving_broker/proving_broker_database/persisted.ts

@@ -11,7 +11,14 @@ import {
   ProvingJobSettledResult,
   getEpochFromProvingJobId,
 } from '@aztec/stdlib/interfaces/server';
-import { Attributes, LmdbMetrics, type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
+import {
+  Attributes,
+  LmdbMetrics,
+  type TelemetryClient,
+  type Tracer,
+  getTelemetryClient,
+  trackSpan,
+} from '@aztec/telemetry-client';
 
 import { mkdir, readdir } from 'fs/promises';
 import { join } from 'path';
@@ -78,6 +85,8 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
 
   private batchQueue: BatchQueue<ProvingJob | [ProvingJobId, ProvingJobSettledResult], number>;
 
+  public readonly tracer: Tracer;
+
   private constructor(
     private epochs: Map<number, SingleEpochDatabase>,
     private config: ProverBrokerConfig,
@@ -92,6 +101,8 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
       () => this.estimateSize(),
     );
 
+    this.tracer = client.getTracer('KVBrokerDatabase');
+
     this.batchQueue = new BatchQueue(
       (items, key) => this.commitWrites(items, key),
       config.proverBrokerBatchSize,
@@ -165,6 +176,9 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
     }
   }
 
+  @trackSpan('KVBrokerDatabase.deleteAllProvingJobsOlderThanEpoch', epochNumber => ({
+    [Attributes.EPOCH_NUMBER]: epochNumber,
+  }))
   async deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void> {
     const oldEpochs = Array.from(this.epochs.keys()).filter(e => e < Number(epochNumber));
     for (const old of oldEpochs) {