@aztec/prover-client 0.42.0 → 0.44.0

package/src/orchestrator/orchestrator.ts

@@ -9,6 +9,7 @@ import {
   type TxEffect,
   makeEmptyProcessedTx,
   makePaddingProcessedTx,
+  mapPublicKernelToCircuitName,
   toTxEffect,
 } from '@aztec/circuit-types';
 import {
@@ -20,6 +21,7 @@ import {
   type PublicInputsAndRecursiveProof,
   type ServerCircuitProver,
 } from '@aztec/circuit-types/interfaces';
+import { type CircuitName } from '@aztec/circuit-types/stats';
 import {
   AGGREGATION_OBJECT_LENGTH,
   AvmCircuitInputs,
@@ -48,11 +50,12 @@ import {
 } from '@aztec/circuits.js';
 import { makeTuple } from '@aztec/foundation/array';
 import { padArrayEnd } from '@aztec/foundation/collection';
-import { AbortedError } from '@aztec/foundation/error';
+import { AbortError } from '@aztec/foundation/error';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { BufferReader, type Tuple } from '@aztec/foundation/serialize';
 import { pushTestData } from '@aztec/foundation/testing';
+import { Attributes, type TelemetryClient, type Tracer, trackSpan, wrapCallbackInSpan } from '@aztec/telemetry-client';
 import { type MerkleTreeOperations } from '@aztec/world-state';
 
 import { inspect } from 'util';
@@ -91,7 +94,16 @@ export class ProvingOrchestrator {
   private pendingProvingJobs: AbortController[] = [];
   private paddingTx: PaddingProcessedTx | undefined = undefined;
 
-  constructor(private db: MerkleTreeOperations, private prover: ServerCircuitProver, private initialHeader?: Header) {}
+  public readonly tracer: Tracer;
+
+  constructor(
+    private db: MerkleTreeOperations,
+    private prover: ServerCircuitProver,
+    telemetryClient: TelemetryClient,
+    private initialHeader?: Header,
+  ) {
+    this.tracer = telemetryClient.getTracer('ProvingOrchestrator');
+  }
 
   /**
    * Resets the orchestrator's cached padding tx.
@@ -108,6 +120,10 @@ export class ProvingOrchestrator {
    * @param verificationKeys - The private kernel verification keys
    * @returns A proving ticket, containing a promise notifying of proving completion
    */
+  @trackSpan('ProvingOrchestrator.startNewBlock', (numTxs, globalVariables) => ({
+    [Attributes.BLOCK_SIZE]: numTxs,
+    [Attributes.BLOCK_NUMBER]: globalVariables.blockNumber.toNumber(),
+  }))
   public async startNewBlock(
     numTxs: number,
     globalVariables: GlobalVariables,
@@ -193,6 +209,9 @@ export class ProvingOrchestrator {
    * The interface to add a simulated transaction to the scheduler
    * @param tx - The transaction to be proven
    */
+  @trackSpan('ProvingOrchestrator.addNewTx', tx => ({
+    [Attributes.TX_HASH]: tx.hash.toString(),
+  }))
   public async addNewTx(tx: ProcessedTx): Promise<void> {
     if (!this.provingState) {
       throw new Error(`Invalid proving state, call startNewBlock before adding transactions`);
@@ -213,6 +232,17 @@ export class ProvingOrchestrator {
   /**
    * Marks the block as full and pads it to the full power of 2 block size, no more transactions will be accepted.
    */
+  @trackSpan('ProvingOrchestrator.setBlockCompleted', function () {
+    if (!this.provingState) {
+      return {};
+    }
+
+    return {
+      [Attributes.BLOCK_NUMBER]: this.provingState!.globalVariables.blockNumber.toNumber(),
+      [Attributes.BLOCK_SIZE]: this.provingState!.totalNumTxs,
+      [Attributes.BLOCK_TXS_COUNT]: this.provingState!.transactionsReceived,
+    };
+  })
   public async setBlockCompleted() {
     if (!this.provingState) {
       throw new Error(`Invalid proving state, call startNewBlock before adding transactions or completing the block`);
@@ -264,18 +294,26 @@ export class ProvingOrchestrator {
     logger.debug(`Enqueuing deferred proving for padding txs to enqueue ${txInputs.length} paddings`);
     this.deferredProving(
       provingState,
-      signal =>
-        this.prover.getEmptyPrivateKernelProof(
-          {
-            // Chain id and version should not change even if the proving state does, so it's safe to use them for the padding tx
-            // which gets cached across multiple runs of the orchestrator with different proving states. If they were to change,
-            // we'd have to clear out the paddingTx here and regenerate it when they do.
-            chainId: unprovenPaddingTx.data.constants.txContext.chainId,
-            version: unprovenPaddingTx.data.constants.txContext.version,
-            header: unprovenPaddingTx.data.constants.historicalHeader,
-          },
-          signal,
-        ),
+      wrapCallbackInSpan(
+        this.tracer,
+        'ProvingOrchestrator.prover.getEmptyPrivateKernelProof',
+        {
+          [Attributes.PROTOCOL_CIRCUIT_TYPE]: 'server',
+          [Attributes.PROTOCOL_CIRCUIT_NAME]: 'private-kernel-empty' as CircuitName,
+        },
+        signal =>
+          this.prover.getEmptyPrivateKernelProof(
+            {
+              // Chain id and version should not change even if the proving state does, so it's safe to use them for the padding tx
+              // which gets cached across multiple runs of the orchestrator with different proving states. If they were to change,
+              // we'd have to clear out the paddingTx here and regenerate it when they do.
+              chainId: unprovenPaddingTx.data.constants.txContext.chainId,
+              version: unprovenPaddingTx.data.constants.txContext.version,
+              header: unprovenPaddingTx.data.constants.historicalHeader,
+            },
+            signal,
+          ),
+      ),
       result => {
         logger.debug(`Completed proof for padding tx, now enqueuing ${txInputs.length} padding txs`);
         this.paddingTx = makePaddingProcessedTx(result);
@@ -319,6 +357,13 @@ export class ProvingOrchestrator {
    * Performs the final tree update for the block and returns the fully proven block.
    * @returns The fully proven block and proof.
    */
+  @trackSpan('ProvingOrchestrator.finaliseBlock', function () {
+    return {
+      [Attributes.BLOCK_NUMBER]: this.provingState!.globalVariables.blockNumber.toNumber(),
+      [Attributes.BLOCK_TXS_COUNT]: this.provingState!.transactionsReceived,
+      [Attributes.BLOCK_SIZE]: this.provingState!.totalNumTxs,
+    };
+  })
   public async finaliseBlock() {
     try {
       if (
@@ -475,7 +520,7 @@ export class ProvingOrchestrator {
 
         await callback(result);
       } catch (err) {
-        if (err instanceof AbortedError) {
+        if (err instanceof AbortError) {
           // operation was cancelled, probably because the block was cancelled
           // drop this result
           return;
@@ -496,6 +541,9 @@ export class ProvingOrchestrator {
   }
 
   // Updates the merkle trees for a transaction. The first enqueued job for a transaction
+  @trackSpan('ProvingOrchestrator.prepareBaseRollupInputs', (_, tx) => ({
+    [Attributes.TX_HASH]: tx.hash.toString(),
+  }))
   private async prepareBaseRollupInputs(
     provingState: ProvingState | undefined,
     tx: ProcessedTx,
@@ -593,7 +641,16 @@ export class ProvingOrchestrator {
 
     this.deferredProving(
       provingState,
-      signal => this.prover.getBaseRollupProof(tx.baseRollupInputs, signal),
+      wrapCallbackInSpan(
+        this.tracer,
+        'ProvingOrchestrator.prover.getBaseRollupProof',
+        {
+          [Attributes.TX_HASH]: tx.processedTx.hash.toString(),
+          [Attributes.PROTOCOL_CIRCUIT_TYPE]: 'server',
+          [Attributes.PROTOCOL_CIRCUIT_NAME]: 'base-rollup' as CircuitName,
+        },
+        signal => this.prover.getBaseRollupProof(tx.baseRollupInputs, signal),
+      ),
       result => {
         logger.debug(`Completed proof for base rollup for tx ${tx.processedTx.hash.toString()}`);
         validatePartialState(result.inputs.end, tx.treeSnapshots);
@@ -622,7 +679,15 @@ export class ProvingOrchestrator {
 
     this.deferredProving(
       provingState,
-      signal => this.prover.getMergeRollupProof(inputs, signal),
+      wrapCallbackInSpan(
+        this.tracer,
+        'ProvingOrchestrator.prover.getMergeRollupProof',
+        {
+          [Attributes.PROTOCOL_CIRCUIT_TYPE]: 'server',
+          [Attributes.PROTOCOL_CIRCUIT_NAME]: 'merge-rollup' as CircuitName,
+        },
+        signal => this.prover.getMergeRollupProof(inputs, signal),
+      ),
       result => {
         this.storeAndExecuteNextMergeLevel(provingState, level, index, [
           result.inputs,
@@ -658,7 +723,15 @@ export class ProvingOrchestrator {
 
     this.deferredProving(
       provingState,
-      signal => this.prover.getRootRollupProof(inputs, signal),
+      wrapCallbackInSpan(
+        this.tracer,
+        'ProvingOrchestrator.prover.getRootRollupProof',
+        {
+          [Attributes.PROTOCOL_CIRCUIT_TYPE]: 'server',
+          [Attributes.PROTOCOL_CIRCUIT_NAME]: 'root-rollup' as CircuitName,
+        },
+        signal => this.prover.getRootRollupProof(inputs, signal),
+      ),
       result => {
         provingState.rootRollupPublicInputs = result.inputs;
         provingState.finalAggregationObject = extractAggregationObject(
@@ -680,7 +753,15 @@ export class ProvingOrchestrator {
   private enqueueBaseParityCircuit(provingState: ProvingState, inputs: BaseParityInputs, index: number) {
     this.deferredProving(
       provingState,
-      signal => this.prover.getBaseParityProof(inputs, signal),
+      wrapCallbackInSpan(
+        this.tracer,
+        'ProvingOrchestrator.prover.getBaseParityProof',
+        {
+          [Attributes.PROTOCOL_CIRCUIT_TYPE]: 'server',
+          [Attributes.PROTOCOL_CIRCUIT_NAME]: 'base-parity' as CircuitName,
+        },
+        signal => this.prover.getBaseParityProof(inputs, signal),
+      ),
       rootInput => {
         provingState.setRootParityInputs(rootInput, index);
         if (provingState.areRootParityInputsReady()) {
@@ -701,7 +782,15 @@ export class ProvingOrchestrator {
   private enqueueRootParityCircuit(provingState: ProvingState | undefined, inputs: RootParityInputs) {
     this.deferredProving(
       provingState,
-      signal => this.prover.getRootParityProof(inputs, signal),
+      wrapCallbackInSpan(
+        this.tracer,
+        'ProvingOrchestrator.prover.getRootParityProof',
+        {
+          [Attributes.PROTOCOL_CIRCUIT_TYPE]: 'server',
+          [Attributes.PROTOCOL_CIRCUIT_NAME]: 'root-parity' as CircuitName,
+        },
+        signal => this.prover.getRootParityProof(inputs, signal),
+      ),
       async rootInput => {
         provingState!.finalRootParityInput = rootInput;
         await this.checkAndEnqueueRootRollup(provingState);
@@ -770,25 +859,34 @@ export class ProvingOrchestrator {
     if (publicFunction.vmRequest) {
       // This function tries to do AVM proving. If there is a failure, it fakes the proof unless AVM_PROVING_STRICT is defined.
       // Nothing downstream depends on the AVM proof yet. So having this mode lets us incrementally build the AVM circuit.
-      const doAvmProving = async (signal: AbortSignal) => {
-        const inputs: AvmCircuitInputs = new AvmCircuitInputs(
-          publicFunction.vmRequest!.bytecode,
-          publicFunction.vmRequest!.calldata,
-          publicFunction.vmRequest!.kernelRequest.inputs.publicCall.callStackItem.publicInputs,
-          publicFunction.vmRequest!.avmHints,
-        );
-        try {
-          return await this.prover.getAvmProof(inputs, signal);
-        } catch (err) {
-          if (process.env.AVM_PROVING_STRICT) {
-            throw err;
-          } else {
-            logger.warn(`Error thrown when proving AVM circuit: ${err}`);
-            logger.warn(`AVM_PROVING_STRICT is off, faking AVM proof and carrying on...`);
-            return { proof: makeEmptyProof(), verificationKey: VerificationKeyData.makeFake() };
+      const doAvmProving = wrapCallbackInSpan(
+        this.tracer,
+        'ProvingOrchestrator.prover.getAvmProof',
+        {
+          [Attributes.TX_HASH]: txProvingState.processedTx.hash.toString(),
+          [Attributes.APP_CIRCUIT_NAME]: publicFunction.vmRequest!.functionName,
+        },
+        async (signal: AbortSignal) => {
+          const inputs: AvmCircuitInputs = new AvmCircuitInputs(
+            publicFunction.vmRequest!.functionName,
+            publicFunction.vmRequest!.bytecode,
+            publicFunction.vmRequest!.calldata,
+            publicFunction.vmRequest!.kernelRequest.inputs.publicCall.callStackItem.publicInputs,
+            publicFunction.vmRequest!.avmHints,
+          );
+          try {
+            return await this.prover.getAvmProof(inputs, signal);
+          } catch (err) {
+            if (process.env.AVM_PROVING_STRICT) {
+              throw err;
+            } else {
+              logger.warn(`Error thrown when proving AVM circuit: ${err}`);
+              logger.warn(`AVM_PROVING_STRICT is off, faking AVM proof and carrying on...`);
+              return { proof: makeEmptyProof(), verificationKey: VerificationKeyData.makeFake() };
+            }
           }
-        }
-      };
+        },
+      );
       this.deferredProving(provingState, doAvmProving, proofAndVk => {
         logger.debug(`Proven VM for function index ${functionIndex} of tx index ${txIndex}`);
         this.checkAndEnqueuePublicKernel(provingState, txIndex, functionIndex, proofAndVk.proof);
@@ -834,13 +932,25 @@ export class ProvingOrchestrator {
 
     this.deferredProving(
       provingState,
-      (signal): Promise<PublicInputsAndRecursiveProof<KernelCircuitPublicInputs | PublicKernelCircuitPublicInputs>> => {
-        if (request.type === PublicKernelType.TAIL) {
-          return this.prover.getPublicTailProof(request, signal);
-        } else {
-          return this.prover.getPublicKernelProof(request, signal);
-        }
-      },
+      wrapCallbackInSpan(
+        this.tracer,
+        request.type === PublicKernelType.TAIL
+          ? 'ProvingOrchestrator.prover.getPublicTailProof'
+          : 'ProvingOrchestrator.prover.getPublicKernelProof',
+        {
+          [Attributes.PROTOCOL_CIRCUIT_TYPE]: 'server',
+          [Attributes.PROTOCOL_CIRCUIT_NAME]: mapPublicKernelToCircuitName(request.type),
+        },
+        (
+          signal,
+        ): Promise<PublicInputsAndRecursiveProof<KernelCircuitPublicInputs | PublicKernelCircuitPublicInputs>> => {
+          if (request.type === PublicKernelType.TAIL) {
+            return this.prover.getPublicTailProof(request, signal);
+          } else {
+            return this.prover.getPublicKernelProof(request, signal);
+          }
+        },
+      ),
       result => {
         const nextKernelRequest = txProvingState.getNextPublicKernelFromKernelProof(
           functionIndex,
@@ -879,5 +989,9 @@ function extractAggregationObject(proof: Proof, numPublicInputs: number): Fr[] {
     Fr.SIZE_IN_BYTES * (numPublicInputs - AGGREGATION_OBJECT_LENGTH),
     Fr.SIZE_IN_BYTES * numPublicInputs,
   );
+  // TODO(#7159): Remove the following workaround
+  if (buffer.length === 0) {
+    return Array.from({ length: AGGREGATION_OBJECT_LENGTH }, () => Fr.ZERO);
+  }
   return BufferReader.asReader(buffer).readArray(AGGREGATION_OBJECT_LENGTH, Fr);
 }

package/src/prover-agent/memory-proving-queue.ts

@@ -27,21 +27,23 @@ import type {
   RootRollupPublicInputs,
 } from '@aztec/circuits.js';
 import { randomBytes } from '@aztec/foundation/crypto';
-import { AbortedError, TimeoutError } from '@aztec/foundation/error';
+import { AbortError, TimeoutError } from '@aztec/foundation/error';
 import { MemoryFifo } from '@aztec/foundation/fifo';
 import { createDebugLogger } from '@aztec/foundation/log';
-import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise';
+import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 
 type ProvingJobWithResolvers<T extends ProvingRequest = ProvingRequest> = {
   id: string;
   request: T;
   signal?: AbortSignal;
   attempts: number;
+  heartbeat: number;
 } & PromiseWithResolvers<ProvingRequestResult<T['type']>>;
 
 const MAX_RETRIES = 3;
 
 const defaultIdGenerator = () => randomBytes(4).toString('hex');
+const defaultTimeSource = () => Date.now();
 
 /**
  * A helper class that sits in between services that need proofs created and agents that can create them.
@@ -52,9 +54,44 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   private queue = new MemoryFifo<ProvingJobWithResolvers>();
   private jobsInProgress = new Map<string, ProvingJobWithResolvers>();
 
-  constructor(private generateId = defaultIdGenerator) {}
+  private runningPromise: RunningPromise;
+
+  constructor(
+    /** Timeout the job if an agent doesn't report back in this time */
+    private jobTimeoutMs = 60 * 1000,
+    /** How often to check for timed out jobs */
+    pollingIntervalMs = 1000,
+    private generateId = defaultIdGenerator,
+    private timeSource = defaultTimeSource,
+  ) {
+    this.runningPromise = new RunningPromise(this.poll, pollingIntervalMs);
+  }
+
+  public start() {
+    if (this.runningPromise.isRunning()) {
+      this.log.warn('Proving queue is already running');
+      return;
+    }
+
+    this.runningPromise.start();
+    this.log.info('Proving queue started');
+  }
+
+  public async stop() {
+    if (!this.runningPromise.isRunning()) {
+      this.log.warn('Proving queue is already stopped');
+      return;
+    }
+
+    await this.runningPromise.stop();
+    this.log.info('Proving queue stopped');
+  }
+
+  public async getProvingJob({ timeoutSec = 1 } = {}): Promise<ProvingJob<ProvingRequest> | undefined> {
+    if (!this.runningPromise.isRunning()) {
+      throw new Error('Proving queue is not running. Start the queue before getting jobs.');
+    }
 
-  async getProvingJob({ timeoutSec = 1 } = {}): Promise<ProvingJob<ProvingRequest> | undefined> {
     try {
       const job = await this.queue.get(timeoutSec);
       if (!job) {
@@ -62,10 +99,10 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
       }
 
       if (job.signal?.aborted) {
-        this.log.debug(`Job ${job.id} type=${job.request.type} has been aborted`);
         return undefined;
       }
 
+      job.heartbeat = this.timeSource();
       this.jobsInProgress.set(job.id, job);
       return {
         id: job.id,
@@ -81,25 +118,33 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   }
 
   resolveProvingJob<T extends ProvingRequestType>(jobId: string, result: ProvingRequestResult<T>): Promise<void> {
+    if (!this.runningPromise.isRunning()) {
+      throw new Error('Proving queue is not running.');
+    }
+
     const job = this.jobsInProgress.get(jobId);
     if (!job) {
-      return Promise.reject(new Error('Job not found'));
+      this.log.warn(`Job id=${jobId} not found. Can't resolve`);
+      return Promise.resolve();
     }
 
     this.jobsInProgress.delete(jobId);
-
-    if (job.signal?.aborted) {
-      return Promise.resolve();
+    if (!job.signal?.aborted) {
+      job.resolve(result);
     }
 
-    job.resolve(result);
     return Promise.resolve();
   }
 
   rejectProvingJob(jobId: string, err: any): Promise<void> {
+    if (!this.runningPromise.isRunning()) {
+      throw new Error('Proving queue is not running.');
+    }
+
     const job = this.jobsInProgress.get(jobId);
     if (!job) {
-      return Promise.reject(new Error('Job not found'));
+      this.log.warn(`Job id=${jobId} not found. Can't reject`);
+      return Promise.resolve();
     }
 
     this.jobsInProgress.delete(jobId);
@@ -123,10 +168,50 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
     return Promise.resolve();
   }
 
+  public heartbeat(jobId: string): Promise<void> {
+    if (!this.runningPromise.isRunning()) {
+      throw new Error('Proving queue is not running.');
+    }
+
+    const job = this.jobsInProgress.get(jobId);
+    if (job) {
+      job.heartbeat = this.timeSource();
+    }
+
+    return Promise.resolve();
+  }
+
+  public isJobRunning(jobId: string): boolean {
+    return this.jobsInProgress.has(jobId);
+  }
+
+  private poll = () => {
+    const now = this.timeSource();
+
+    for (const job of this.jobsInProgress.values()) {
+      if (job.signal?.aborted) {
+        this.jobsInProgress.delete(job.id);
+        continue;
+      }
+
+      if (job.heartbeat + this.jobTimeoutMs < now) {
+        this.log.warn(`Job ${job.id} type=${ProvingRequestType[job.request.type]} has timed out`);
+
+        this.jobsInProgress.delete(job.id);
+        job.heartbeat = 0;
+        this.queue.put(job);
+      }
+    }
+  };
+
   private enqueue<T extends ProvingRequest>(
     request: T,
     signal?: AbortSignal,
   ): Promise<ProvingRequestResult<T['type']>> {
+    if (!this.runningPromise.isRunning()) {
+      return Promise.reject(new Error('Proving queue is not running.'));
+    }
+
     const { promise, resolve, reject } = promiseWithResolvers<ProvingRequestResult<T['type']>>();
     const item: ProvingJobWithResolvers<T> = {
       id: this.generateId(),
@@ -136,10 +221,11 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
       resolve,
       reject,
       attempts: 1,
+      heartbeat: 0,
     };
 
     if (signal) {
-      signal.addEventListener('abort', () => reject(new AbortedError('Operation has been aborted')));
+      signal.addEventListener('abort', () => reject(new AbortError('Operation has been aborted')));
     }
 
     this.log.debug(

package/src/prover-agent/prover-agent.ts

@@ -16,7 +16,7 @@ import { ProvingError } from './proving-error.js';
  * A helper class that encapsulates a circuit prover and connects it to a job source.
  */
 export class ProverAgent {
-  private inFlightPromises = new Set<Promise<any>>();
+  private inFlightPromises = new Map<string, Promise<any>>();
   private runningPromise?: RunningPromise;
 
   constructor(
@@ -50,20 +50,28 @@ export class ProverAgent {
     }
 
     this.runningPromise = new RunningPromise(async () => {
+      for (const jobId of this.inFlightPromises.keys()) {
+        await jobSource.heartbeat(jobId);
+      }
+
       while (this.inFlightPromises.size < this.maxConcurrency) {
-        const job = await jobSource.getProvingJob();
-        if (!job) {
-          // job source is fully drained, sleep for a bit and try again
-          return;
+        try {
+          const job = await jobSource.getProvingJob();
+          if (!job) {
+            // job source is fully drained, sleep for a bit and try again
+            return;
+          }
+
+          const promise = this.work(jobSource, job).finally(() => this.inFlightPromises.delete(job.id));
+          this.inFlightPromises.set(job.id, promise);
+        } catch (err) {
+          this.log.warn(`Error processing job: ${err}`);
         }
-
-        const promise = this.work(jobSource, job).finally(() => this.inFlightPromises.delete(promise));
-        this.inFlightPromises.add(promise);
       }
     }, this.pollIntervalMs);
 
     this.runningPromise.start();
-    this.log.info('Agent started');
+    this.log.info(`Agent started with concurrency=${this.maxConcurrency}`);
   }
 
   async stop(): Promise<void> {
@@ -79,14 +87,31 @@ export class ProverAgent {
 
   private async work(jobSource: ProvingJobSource, job: ProvingJob<ProvingRequest>): Promise<void> {
     try {
+      this.log.debug(`Picked up proving job id=${job.id} type=${ProvingRequestType[job.request.type]}`);
       const [time, result] = await elapsed(this.getProof(job.request));
-      await jobSource.resolveProvingJob(job.id, result);
-      this.log.debug(
-        `Processed proving job id=${job.id} type=${ProvingRequestType[job.request.type]} duration=${time}ms`,
-      );
+      if (this.isRunning()) {
+        this.log.debug(
+          `Processed proving job id=${job.id} type=${ProvingRequestType[job.request.type]} duration=${time}ms`,
+        );
+        await jobSource.resolveProvingJob(job.id, result);
+      } else {
+        this.log.debug(
+          `Dropping proving job id=${job.id} type=${
+            ProvingRequestType[job.request.type]
+          } duration=${time}ms: agent stopped`,
+        );
+      }
     } catch (err) {
-      this.log.error(`Error processing proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: ${err}`);
-      await jobSource.rejectProvingJob(job.id, new ProvingError((err as any)?.message ?? String(err)));
+      if (this.isRunning()) {
+        this.log.error(
+          `Error processing proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: ${err}`,
+        );
+        await jobSource.rejectProvingJob(job.id, new ProvingError((err as any)?.message ?? String(err)));
+      } else {
+        this.log.debug(
+          `Dropping proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: agent stopped: ${err}`,
+        );
+      }
     }
   }