@aztec/prover-client 0.66.0 → 0.67.1-devnet

package/src/orchestrator/orchestrator.ts

@@ -4,6 +4,7 @@ import {
   type ProcessedTx,
   type ServerCircuitProver,
   makeEmptyProcessedTx,
+  toNumBlobFields,
 } from '@aztec/circuit-types';
 import {
   type EpochProver,
@@ -16,15 +17,18 @@ import {
   AVM_PROOF_LENGTH_IN_FIELDS,
   AVM_VERIFICATION_KEY_LENGTH_IN_FIELDS,
   type AppendOnlyTreeSnapshot,
+  BLOBS_PER_BLOCK,
   type BaseOrMergeRollupPublicInputs,
   BaseParityInputs,
   type BaseRollupHints,
+  BlobPublicInputs,
+  type BlockHeader,
   type BlockRootOrBlockMergePublicInputs,
   BlockRootRollupInputs,
   EmptyBlockRootRollupInputs,
+  FIELDS_PER_BLOB,
   Fr,
   type GlobalVariables,
-  type Header,
   L1_TO_L2_MSG_SUBTREE_HEIGHT,
   L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH,
   type NESTED_RECURSIVE_PROOF_LENGTH,
@@ -40,9 +44,11 @@ import {
   makeEmptyRecursiveProof,
 } from '@aztec/circuits.js';
 import { makeTuple } from '@aztec/foundation/array';
+import { Blob } from '@aztec/foundation/blob';
 import { maxBy, padArrayEnd } from '@aztec/foundation/collection';
+import { sha256ToField } from '@aztec/foundation/crypto';
 import { AbortError } from '@aztec/foundation/error';
-import { createDebugLogger } from '@aztec/foundation/log';
+import { createLogger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { type Tuple } from '@aztec/foundation/serialize';
 import { pushTestData } from '@aztec/foundation/testing';
@@ -77,7 +83,7 @@ import {
 import { ProvingOrchestratorMetrics } from './orchestrator_metrics.js';
 import { TxProvingState } from './tx-proving-state.js';
 
-const logger = createDebugLogger('aztec:prover:proving-orchestrator');
+const logger = createLogger('prover-client:orchestrator');
 
 /**
  * Implements an event driven proving scheduler to build the recursive proof tree. The idea being:
@@ -139,17 +145,14 @@ export class ProvingOrchestrator implements EpochProver {
 
   /**
    * Starts off a new block
-   * @param numTxs - The total number of transactions in the block.
    * @param globalVariables - The global variables for the block
    * @param l1ToL2Messages - The l1 to l2 messages for the block
-   * @param verificationKeys - The private kernel verification keys
    * @returns A proving ticket, containing a promise notifying of proving completion
    */
-  @trackSpan('ProvingOrchestrator.startNewBlock', (numTxs, globalVariables) => ({
-    [Attributes.BLOCK_SIZE]: numTxs,
+  @trackSpan('ProvingOrchestrator.startNewBlock', globalVariables => ({
     [Attributes.BLOCK_NUMBER]: globalVariables.blockNumber.toNumber(),
   }))
-  public async startNewBlock(numTxs: number, globalVariables: GlobalVariables, l1ToL2Messages: Fr[]) {
+  public async startNewBlock(globalVariables: GlobalVariables, l1ToL2Messages: Fr[]) {
     if (!this.provingState) {
       throw new Error(`Invalid proving state, call startNewEpoch before starting a block`);
     }
@@ -158,12 +161,8 @@ export class ProvingOrchestrator implements EpochProver {
       throw new Error(`Epoch not accepting further blocks`);
     }
 
-    if (!Number.isInteger(numTxs) || numTxs < 2) {
-      throw new Error(`Invalid number of txs for block (got ${numTxs})`);
-    }
-
     logger.info(
-      `Starting block ${globalVariables.blockNumber.toNumber()} for slot ${globalVariables.slotNumber.toNumber()} with ${numTxs} transactions`,
+      `Starting block ${globalVariables.blockNumber.toNumber()} for slot ${globalVariables.slotNumber.toNumber()}`,
     );
 
     // Fork world state at the end of the immediately previous block
@@ -210,7 +209,6 @@ export class ProvingOrchestrator implements EpochProver {
     );
 
     const blockProvingState = this.provingState!.startNewBlock(
-      numTxs,
       globalVariables,
       l1ToL2MessagesPadded,
       messageTreeSnapshot,
@@ -228,47 +226,56 @@ export class ProvingOrchestrator implements EpochProver {
   }
 
   /**
-   * The interface to add a simulated transaction to the scheduler
-   * @param tx - The transaction to be proven
+   * The interface to add simulated transactions to the scheduler
+   * @param txs - The transactions to be proven
    */
-  @trackSpan('ProvingOrchestrator.addNewTx', tx => ({
-    [Attributes.TX_HASH]: tx.hash.toString(),
+  @trackSpan('ProvingOrchestrator.addTxs', txs => ({
+    [Attributes.BLOCK_TXS_COUNT]: txs.length,
   }))
-  public async addNewTx(tx: ProcessedTx): Promise<void> {
-    const blockNumber = tx.constants.globalVariables.blockNumber.toNumber();
-    try {
-      const provingState = this.provingState?.getBlockProvingStateByBlockNumber(blockNumber);
-      if (!provingState) {
-        throw new Error(`Block proving state for ${blockNumber} not found`);
-      }
-
-      if (!provingState.isAcceptingTransactions()) {
-        throw new Error(`Rollup not accepting further transactions`);
-      }
+  public async addTxs(txs: ProcessedTx[]): Promise<void> {
+    if (!txs.length) {
+      // To avoid an ugly throw below. If we require an empty block, we can just call setBlockCompleted
+      // on a block with no txs. We cannot do that here because we cannot find the blockNumber without any txs.
+      logger.warn(`Provided no txs to orchestrator addTxs.`);
+      return;
+    }
+    const blockNumber = txs[0].constants.globalVariables.blockNumber.toNumber();
+    const provingState = this.provingState?.getBlockProvingStateByBlockNumber(blockNumber!);
+    if (!provingState) {
+      throw new Error(`Block proving state for ${blockNumber} not found`);
+    }
 
-      if (!provingState.verifyState()) {
-        throw new Error(`Invalid proving state when adding a tx`);
-      }
+    const numBlobFields = toNumBlobFields(txs);
+    provingState.startNewBlock(Math.max(2, txs.length), numBlobFields);
 
-      validateTx(tx);
+    logger.info(
+      `Adding ${txs.length} transactions with ${numBlobFields} blob fields to block ${provingState?.blockNumber}`,
+    );
+    for (const tx of txs) {
+      try {
+        if (!provingState.verifyState()) {
+          throw new Error(`Invalid proving state when adding a tx`);
+        }
 
-      logger.info(`Received transaction: ${tx.hash}`);
+        validateTx(tx);
 
-      if (tx.isEmpty) {
-        logger.warn(`Ignoring empty transaction ${tx.hash} - it will not be added to this block`);
-        return;
-      }
+        logger.info(`Received transaction: ${tx.hash}`);
 
-      const [hints, treeSnapshots] = await this.prepareTransaction(tx, provingState);
-      this.enqueueFirstProofs(hints, treeSnapshots, tx, provingState);
+        if (tx.isEmpty) {
+          logger.warn(`Ignoring empty transaction ${tx.hash} - it will not be added to this block`);
+          continue;
+        }
 
-      if (provingState.transactionsReceived === provingState.totalNumTxs) {
-        logger.verbose(`All transactions received for block ${provingState.globalVariables.blockNumber}.`);
+        const [hints, treeSnapshots] = await this.prepareTransaction(tx, provingState);
+        this.enqueueFirstProofs(hints, treeSnapshots, tx, provingState);
+      } catch (err: any) {
+        throw new Error(`Error adding transaction ${tx.hash.toString()} to block ${blockNumber}: ${err.message}`, {
+          cause: err,
+        });
       }
-    } catch (err: any) {
-      throw new Error(`Error adding transaction ${tx.hash.toString()} to block ${blockNumber}: ${err.message}`, {
-        cause: err,
-      });
+    }
+    if (provingState.transactionsReceived === provingState.totalNumTxs) {
+      logger.verbose(`All transactions received for block ${provingState.globalVariables.blockNumber}.`);
     }
   }
 
@@ -279,12 +286,18 @@ export class ProvingOrchestrator implements EpochProver {
   @trackSpan('ProvingOrchestrator.setBlockCompleted', (blockNumber: number) => ({
     [Attributes.BLOCK_NUMBER]: blockNumber,
   }))
-  public async setBlockCompleted(blockNumber: number, expectedHeader?: Header): Promise<L2Block> {
+  public async setBlockCompleted(blockNumber: number, expectedHeader?: BlockHeader): Promise<L2Block> {
     const provingState = this.provingState?.getBlockProvingStateByBlockNumber(blockNumber);
     if (!provingState) {
       throw new Error(`Block proving state for ${blockNumber} not found`);
     }
 
+    if (!provingState.spongeBlobState) {
+      // If we are completing an empty block, initialise the provingState.
+      // We will have 2 padding txs, and => no blob fields.
+      provingState.startNewBlock(2, 0);
+    }
+
     if (!provingState.verifyState()) {
       throw new Error(`Block proving failed: ${provingState.error}`);
     }
@@ -326,7 +339,7 @@ export class ProvingOrchestrator implements EpochProver {
     }
 
     // And build the block header
-    logger.verbose(`Block ${provingState.globalVariables.blockNumber} completed. Assembling header.`);
+    logger.verbose(`Block ${blockNumber} completed. Assembling header.`);
     await this.buildBlock(provingState, expectedHeader);
 
     // If the proofs were faster than the block building, then we need to try the block root rollup again here
@@ -407,7 +420,7 @@ export class ProvingOrchestrator implements EpochProver {
     return Promise.resolve();
   }
 
-  private async buildBlock(provingState: BlockProvingState, expectedHeader?: Header) {
+  private async buildBlock(provingState: BlockProvingState, expectedHeader?: BlockHeader) {
     // Collect all new nullifiers, commitments, and contracts from all txs in this block to build body
     const txs = provingState!.allTxs.map(a => a.processedTx);
 
@@ -435,14 +448,6 @@ export class ProvingOrchestrator implements EpochProver {
     const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
     const l2Block = new L2Block(newArchive, header, body);
 
-    if (!l2Block.body.getTxsEffectsHash().equals(header.contentCommitment.txsEffectsHash)) {
-      throw new Error(
-        `Txs effects hash mismatch, ${l2Block.body
-          .getTxsEffectsHash()
-          .toString('hex')} == ${header.contentCommitment.txsEffectsHash.toString('hex')} `,
-      );
-    }
-
     await this.verifyBuiltBlockAgainstSyncedState(l2Block, newArchive);
 
     logger.verbose(`Orchestrator finalised block ${l2Block.number}`);
@@ -566,6 +571,22 @@ export class ProvingOrchestrator implements EpochProver {
     );
   }
 
+  /**
+   * Collect all new nullifiers, commitments, and contracts from all txs in a block
+   * @returns The array of non empty tx effects.
+   */
+  private extractTxEffects(provingState: BlockProvingState) {
+    // Note: this check should ensure that we have all txs and their effects ready.
+    if (!provingState.finalRootParityInput?.publicInputs.shaRoot) {
+      throw new Error(`Invalid proving state, a block must be ready to be proven before its effects can be extracted.`);
+    }
+    const nonEmptyTxEffects = provingState.allTxs
+      .map(txProvingState => txProvingState.processedTx.txEffect)
+      .filter(txEffect => !txEffect.isEmpty());
+
+    return nonEmptyTxEffects;
+  }
+
   /**
    * Returns the proof for the current epoch.
    */
@@ -692,7 +713,7 @@ export class ProvingOrchestrator implements EpochProver {
     provingState: BlockProvingState | undefined,
     tx: ProcessedTx,
   ): Promise<[BaseRollupHints, TreeSnapshots] | undefined> {
-    if (!provingState?.verifyState()) {
+    if (!provingState?.verifyState() || !provingState.spongeBlobState) {
       logger.debug('Not preparing base rollup inputs, state invalid');
       return;
     }
@@ -701,7 +722,9 @@ export class ProvingOrchestrator implements EpochProver {
 
     // We build the base rollup inputs using a mock proof and verification key.
     // These will be overwritten later once we have proven the tube circuit and any public kernels
-    const [ms, hints] = await elapsed(buildBaseRollupHints(tx, provingState.globalVariables, db));
+    const [ms, hints] = await elapsed(
+      buildBaseRollupHints(tx, provingState.globalVariables, db, provingState.spongeBlobState),
+    );
 
     if (!tx.isEmpty) {
       this.metrics.recordBaseRollupInputs(ms);
@@ -774,7 +797,7 @@ export class ProvingOrchestrator implements EpochProver {
     );
   }
 
-  // Enqueues the tub circuit for a given transaction index
+  // Enqueues the tube circuit for a given transaction index
   // Once completed, will enqueue the next circuit, either a public kernel or the base rollup
   private enqueueTube(provingState: BlockProvingState, txIndex: number) {
     if (!provingState?.verifyState()) {
@@ -856,9 +879,14 @@ export class ProvingOrchestrator implements EpochProver {
     provingState.blockRootRollupStarted = true;
     const mergeInputData = provingState.getMergeInputs(0);
     const rootParityInput = provingState.finalRootParityInput!;
+    const blobFields = this.extractTxEffects(provingState)
+      .map(tx => tx.toBlobFields())
+      .flat();
+    const blobs = Blob.getBlobs(blobFields);
+    const blobsHash = sha256ToField(blobs.map(b => b.getEthVersionedBlobHash()));
 
     logger.debug(
-      `Enqueuing block root rollup for block ${provingState.blockNumber} with ${provingState.newL1ToL2Messages.length} l1 to l2 msgs`,
+      `Enqueuing block root rollup for block ${provingState.blockNumber} with ${provingState.newL1ToL2Messages.length} l1 to l2 msgs and ${blobs.length} blobs.`,
     );
 
     const previousRollupData: BlockRootRollupInputs['previousRollupData'] = makeTuple(2, i =>
@@ -879,6 +907,13 @@ export class ProvingOrchestrator implements EpochProver {
       newArchiveSiblingPath: provingState.archiveTreeRootSiblingPath,
       previousBlockHash: provingState.previousBlockHash,
       proverId: this.proverId,
+      blobFields: padArrayEnd(blobFields, Fr.ZERO, FIELDS_PER_BLOB * BLOBS_PER_BLOCK),
+      blobCommitments: padArrayEnd(
+        blobs.map(b => b.commitmentToFields()),
+        [Fr.ZERO, Fr.ZERO],
+        BLOBS_PER_BLOCK,
+      ),
+      blobsHash: blobsHash,
     });
 
     this.deferredProving(
@@ -903,6 +938,17 @@ export class ProvingOrchestrator implements EpochProver {
 
         provingState.blockRootRollupPublicInputs = result.inputs;
         provingState.finalProof = result.proof.binaryProof;
+        const blobOutputs = result.inputs.blobPublicInputs[0];
+        blobOutputs.inner.forEach((blobOutput, i) => {
+          if (!blobOutput.isEmpty() && !blobOutput.equals(BlobPublicInputs.fromBlob(blobs[i]))) {
+            throw new Error(
+              `Rollup circuits produced mismatched blob evaluation:
+              z: ${blobOutput.z} == ${blobs[i].challengeZ},
+              y: ${blobOutput.y.toString(16)} == ${blobs[i].evaluationY.toString('hex')},
+              C: ${blobOutput.kzgCommitment} == ${blobs[i].commitmentToFields()}`,
+            );
+          }
+        });
 
         logger.debug(`Completed proof for block root rollup for ${provingState.block?.number}`);
         // validatePartialState(result.inputs.end, tx.treeSnapshots); // TODO(palla/prover)
@@ -1196,10 +1242,13 @@ export class ProvingOrchestrator implements EpochProver {
           return await this.prover.getAvmProof(inputs, signal, provingState.epochNumber);
         } catch (err) {
           if (process.env.AVM_PROVING_STRICT) {
+            logger.error(`Error thrown when proving AVM circuit with AVM_PROVING_STRICT on`, err);
             throw err;
           } else {
             logger.warn(
-              `Error thrown when proving AVM circuit, but AVM_PROVING_STRICT is off, so faking AVM proof and carrying on. Error: ${err}.`,
+              `Error thrown when proving AVM circuit but AVM_PROVING_STRICT is off. Faking AVM proof and carrying on. ${inspect(
+                err,
+              )}.`,
             );
             return {
               proof: makeEmptyRecursiveProof(AVM_PROOF_LENGTH_IN_FIELDS),

package/src/orchestrator/orchestrator_metrics.ts

@@ -1,11 +1,4 @@
-import {
-  type Histogram,
-  Metrics,
-  type TelemetryClient,
-  type Tracer,
-  ValueType,
-  millisecondBuckets,
-} from '@aztec/telemetry-client';
+import { type Histogram, Metrics, type TelemetryClient, type Tracer, ValueType } from '@aztec/telemetry-client';
 
 export class ProvingOrchestratorMetrics {
   public readonly tracer: Tracer;
@@ -20,9 +13,6 @@ export class ProvingOrchestratorMetrics {
       unit: 'ms',
       description: 'Duration to build base rollup inputs',
       valueType: ValueType.INT,
-      advice: {
-        explicitBucketBoundaries: millisecondBuckets(1), // 10ms -> ~327s
-      },
     });
   }
 

package/src/prover-agent/memory-proving-queue.ts

@@ -32,10 +32,10 @@ import type {
 } from '@aztec/circuits.js';
 import { randomBytes } from '@aztec/foundation/crypto';
 import { AbortError, TimeoutError } from '@aztec/foundation/error';
-import { createDebugLogger } from '@aztec/foundation/log';
+import { createLogger } from '@aztec/foundation/log';
 import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { PriorityMemoryQueue } from '@aztec/foundation/queue';
-import { type TelemetryClient } from '@aztec/telemetry-client';
+import { type TelemetryClient, type Tracer, trackSpan } from '@aztec/telemetry-client';
 
 import { InlineProofStore, type ProofStore } from '../proving_broker/proof_store.js';
 import { ProvingQueueMetrics } from './queue_metrics.js';
@@ -57,7 +57,7 @@ const defaultTimeSource = () => Date.now();
  * The queue accumulates jobs and provides them to agents prioritized by block number.
  */
 export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource {
-  private log = createDebugLogger('aztec:prover-client:prover-pool:queue');
+  private log = createLogger('prover-client:prover-pool:queue');
   private queue = new PriorityMemoryQueue<ProvingJobWithResolvers>(
     (a, b) => (a.epochNumber ?? 0) - (b.epochNumber ?? 0),
   );
@@ -65,6 +65,8 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   private runningPromise: RunningPromise;
   private metrics: ProvingQueueMetrics;
 
+  public readonly tracer: Tracer;
+
   constructor(
     client: TelemetryClient,
     /** Timeout the job if an agent doesn't report back in this time */
@@ -75,8 +77,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
     private timeSource = defaultTimeSource,
     private proofStore: ProofStore = new InlineProofStore(),
   ) {
+    this.tracer = client.getTracer('MemoryProvingQueue');
     this.metrics = new ProvingQueueMetrics(client, 'MemoryProvingQueue');
-    this.runningPromise = new RunningPromise(this.poll, pollingIntervalMs);
+    this.runningPromise = new RunningPromise(this.poll.bind(this), this.log, pollingIntervalMs);
   }
 
   public start() {
@@ -120,6 +123,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
         id: job.id,
         type: job.type,
         inputsUri: job.inputsUri,
+        epochNumber: job.epochNumber,
       };
     } catch (err) {
       if (err instanceof TimeoutError) {
@@ -201,7 +205,8 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
     return this.jobsInProgress.has(jobId);
   }
 
-  private poll = () => {
+  @trackSpan('MemoryProvingQueue.poll')
+  private poll() {
     const now = this.timeSource();
     this.metrics.recordQueueSize(this.queue.length());
 
@@ -219,7 +224,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
         this.queue.put(job);
       }
     }
-  };
+  }
 
   private async enqueue<T extends ProvingRequestType>(
     type: T,
@@ -244,7 +249,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
       reject,
       attempts: 1,
       heartbeat: 0,
-      epochNumber,
+      epochNumber: epochNumber ?? 0,
     };
 
     if (signal) {

package/src/prover-agent/prover-agent.ts

@@ -8,29 +8,32 @@ import {
   type ServerCircuitProver,
   makeProvingRequestResult,
 } from '@aztec/circuit-types';
-import { createDebugLogger } from '@aztec/foundation/log';
+import { createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 import { elapsed } from '@aztec/foundation/timer';
+import { Attributes, type TelemetryClient, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
+import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 
 import { InlineProofStore } from '../proving_broker/proof_store.js';
 
 const PRINT_THRESHOLD_NS = 6e10; // 60 seconds
 
+type InFlightPromise = {
+  id: string;
+  type: ProvingRequestType;
+  promise: Promise<any>;
+};
+
 /**
  * A helper class that encapsulates a circuit prover and connects it to a job source.
  */
-export class ProverAgent implements ProverAgentApi {
-  private inFlightPromises = new Map<
-    string,
-    {
-      id: string;
-      type: ProvingRequestType;
-      promise: Promise<any>;
-    }
-  >();
+export class ProverAgent implements ProverAgentApi, Traceable {
+  private inFlightPromises = new Map<string, InFlightPromise>();
   private runningPromise?: RunningPromise;
   private proofInputsDatabase = new InlineProofStore();
 
+  public readonly tracer: Tracer;
+
   constructor(
     /** The prover implementation to defer jobs to */
     private circuitProver: ServerCircuitProver,
@@ -38,8 +41,13 @@ export class ProverAgent implements ProverAgentApi {
     private maxConcurrency = 1,
     /** How long to wait between jobs */
     private pollIntervalMs = 100,
-    private log = createDebugLogger('aztec:prover-client:prover-agent'),
-  ) {}
+    /** Telemetry client */
+    private telemetry: TelemetryClient = new NoopTelemetryClient(),
+    /** Logger */
+    private log = createLogger('prover-client:prover-agent'),
+  ) {
+    this.tracer = telemetry.getTracer('ProverAgent');
+  }
 
   setMaxConcurrency(maxConcurrency: number): Promise<void> {
     if (maxConcurrency < 1) {
@@ -74,49 +82,53 @@ export class ProverAgent implements ProverAgentApi {
 
     let lastPrint = process.hrtime.bigint();
 
-    this.runningPromise = new RunningPromise(async () => {
-      for (const jobId of this.inFlightPromises.keys()) {
-        await jobSource.heartbeat(jobId);
-      }
-
-      const now = process.hrtime.bigint();
-
-      if (now - lastPrint >= PRINT_THRESHOLD_NS) {
-        // only log if we're actually doing work
-        if (this.inFlightPromises.size > 0) {
-          const jobs = Array.from(this.inFlightPromises.values())
-            .map(job => `id=${job.id},type=${ProvingRequestType[job.type]}`)
-            .join(' ');
-          this.log.info(`Agent is running with ${this.inFlightPromises.size} in-flight jobs: ${jobs}`);
+    this.runningPromise = new RunningPromise(
+      async () => {
+        for (const jobId of this.inFlightPromises.keys()) {
+          await jobSource.heartbeat(jobId);
         }
-        lastPrint = now;
-      }
 
-      while (this.inFlightPromises.size < this.maxConcurrency) {
-        try {
-          const job = await jobSource.getProvingJob();
-          if (!job) {
-            // job source is fully drained, sleep for a bit and try again
-            return;
+        const now = process.hrtime.bigint();
+
+        if (now - lastPrint >= PRINT_THRESHOLD_NS) {
+          // only log if we're actually doing work
+          if (this.inFlightPromises.size > 0) {
+            const jobs = Array.from(this.inFlightPromises.values())
+              .map(job => `id=${job.id},type=${ProvingRequestType[job.type]}`)
+              .join(' ');
+            this.log.info(`Agent is running with ${this.inFlightPromises.size} in-flight jobs: ${jobs}`);
           }
+          lastPrint = now;
+        }
 
+        while (this.inFlightPromises.size < this.maxConcurrency) {
           try {
-            const promise = this.work(jobSource, job).finally(() => this.inFlightPromises.delete(job.id));
-            this.inFlightPromises.set(job.id, {
-              id: job.id,
-              type: job.type,
-              promise,
-            });
+            const job = await jobSource.getProvingJob();
+            if (!job) {
+              // job source is fully drained, sleep for a bit and try again
+              return;
+            }
+
+            try {
+              const promise = this.work(jobSource, job).finally(() => this.inFlightPromises.delete(job.id));
+              this.inFlightPromises.set(job.id, {
+                id: job.id,
+                type: job.type,
+                promise,
+              });
+            } catch (err) {
+              this.log.warn(
+                `Error processing job! type=${ProvingRequestType[job.type]}: ${err}. ${(err as Error).stack}`,
+              );
+            }
           } catch (err) {
-            this.log.warn(
-              `Error processing job! type=${ProvingRequestType[job.type]}: ${err}. ${(err as Error).stack}`,
-            );
+            this.log.error(`Error fetching job`, err);
           }
-        } catch (err) {
-          this.log.error(`Error fetching job`, err);
         }
-      }
-    }, this.pollIntervalMs);
+      },
+      this.log,
+      this.pollIntervalMs,
+    );
 
     this.runningPromise.start();
     this.log.info(`Agent started with concurrency=${this.maxConcurrency}`);
@@ -133,9 +145,16 @@ export class ProverAgent implements ProverAgentApi {
     this.log.info('Agent stopped');
   }
 
+  @trackSpan('ProverAgent.work', (_jobSoure, job) => ({
+    [Attributes.PROVING_JOB_ID]: job.id,
+    [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[job.type],
+  }))
   private async work(jobSource: ProvingJobSource, job: ProvingJob): Promise<void> {
     try {
-      this.log.debug(`Picked up proving job id=${job.id} type=${ProvingRequestType[job.type]}`);
+      this.log.debug(`Picked up proving job ${job.id} ${ProvingRequestType[job.type]}`, {
+        jobId: job.id,
+        jobType: ProvingRequestType[job.type],
+      });
       const type = job.type;
       const inputs = await this.proofInputsDatabase.getProofInput(job.inputsUri);
       const [time, result] = await elapsed(this.getProof(inputs));

package/src/prover-client/prover-client.ts

@@ -4,7 +4,6 @@ import {
   type EpochProver,
   type EpochProverManager,
   type ForkMerkleTreeOperations,
-  type ProverCache,
   type ProvingJobBroker,
   type ProvingJobConsumer,
   type ProvingJobProducer,
@@ -12,17 +11,14 @@ import {
 } from '@aztec/circuit-types/interfaces';
 import { Fr } from '@aztec/circuits.js';
 import { times } from '@aztec/foundation/collection';
-import { createDebugLogger } from '@aztec/foundation/log';
+import { createLogger } from '@aztec/foundation/log';
 import { NativeACVMSimulator } from '@aztec/simulator';
 import { type TelemetryClient } from '@aztec/telemetry-client';
 
-import { join } from 'path';
-
 import { type ProverClientConfig } from '../config.js';
 import { ProvingOrchestrator } from '../orchestrator/orchestrator.js';
-import { CachingBrokerFacade } from '../proving_broker/caching_broker_facade.js';
+import { BrokerCircuitProverFacade } from '../proving_broker/broker_prover_facade.js';
 import { InlineProofStore } from '../proving_broker/proof_store.js';
-import { InMemoryProverCache } from '../proving_broker/prover_cache/memory.js';
 import { ProvingAgent } from '../proving_broker/proving_agent.js';
 
 /** Manages proving of epochs by orchestrating the proving of individual blocks relying on a pool of prover agents. */
@@ -30,25 +26,22 @@ export class ProverClient implements EpochProverManager {
   private running = false;
   private agents: ProvingAgent[] = [];
 
-  private cacheDir?: string;
-
   private constructor(
     private config: ProverClientConfig,
     private worldState: ForkMerkleTreeOperations,
     private telemetry: TelemetryClient,
     private orchestratorClient: ProvingJobProducer,
     private agentClient?: ProvingJobConsumer,
-    private log = createDebugLogger('aztec:prover-client:tx-prover'),
+    private log = createLogger('prover-client:tx-prover'),
   ) {
     // TODO(palla/prover-node): Cache the paddingTx here, and not in each proving orchestrator,
     // so it can be reused across multiple ones and not recomputed every time.
-    this.cacheDir = this.config.cacheDir ? join(this.config.cacheDir, `tx_prover_${this.config.proverId}`) : undefined;
   }
 
-  public createEpochProver(cache: ProverCache = new InMemoryProverCache()): EpochProver {
+  public createEpochProver(): EpochProver {
     return new ProvingOrchestrator(
       this.worldState,
-      new CachingBrokerFacade(this.orchestratorClient, cache),
+      new BrokerCircuitProverFacade(this.orchestratorClient),
       this.telemetry,
       this.config.proverId,
     );