@aztec/prover-client 0.0.1-commit.d3ec352c → 0.0.1-commit.f295ac2

package/src/orchestrator/epoch-proving-state.ts

@@ -1,14 +1,20 @@
 import { BatchedBlob, BatchedBlobAccumulator, type FinalBlobBatchingChallenges } from '@aztec/blob-lib';
-import type {
-  ARCHIVE_HEIGHT,
-  L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
-  NESTED_RECURSIVE_PROOF_LENGTH,
-  NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
+import {
+  type ARCHIVE_HEIGHT,
+  type L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
+  type NESTED_RECURSIVE_PROOF_LENGTH,
+  type NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
+  OUT_HASH_TREE_HEIGHT,
 } from '@aztec/constants';
 import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
-import type { Fr } from '@aztec/foundation/fields';
+import { Fr } from '@aztec/foundation/curves/bn254';
 import type { Tuple } from '@aztec/foundation/serialize';
-import { type TreeNodeLocation, UnbalancedTreeStore } from '@aztec/foundation/trees';
+import {
+  MerkleTreeCalculator,
+  type TreeNodeLocation,
+  UnbalancedTreeStore,
+  shaMerkleHash,
+} from '@aztec/foundation/trees';
 import type { PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
 import type { Proof } from '@aztec/stdlib/proofs';
 import {
@@ -20,7 +26,7 @@ import {
   RootRollupPrivateInputs,
   type RootRollupPublicInputs,
 } from '@aztec/stdlib/rollup';
-import type { AppendOnlyTreeSnapshot, MerkleTreeId } from '@aztec/stdlib/trees';
+import { AppendOnlyTreeSnapshot, type MerkleTreeId } from '@aztec/stdlib/trees';
 import type { BlockHeader } from '@aztec/stdlib/tx';
 
 import { toProofData } from './block-building-helpers.js';
@@ -212,6 +218,48 @@ export class EpochProvingState {
     this.checkpointPaddingProof = { provingOutput };
   }
 
+  public async accumulateCheckpointOutHashes() {
+    const treeCalculator = await MerkleTreeCalculator.create(OUT_HASH_TREE_HEIGHT, undefined, (left, right) =>
+      Promise.resolve(shaMerkleHash(left, right)),
+    );
+
+    const computeOutHashHint = async (leaves: Fr[]) => {
+      const tree = await treeCalculator.computeTree(leaves.map(l => l.toBuffer()));
+      const nextAvailableLeafIndex = leaves.length;
+      return {
+        treeSnapshot: new AppendOnlyTreeSnapshot(Fr.fromBuffer(tree.root), nextAvailableLeafIndex),
+        siblingPath: tree.getSiblingPath(nextAvailableLeafIndex).map(Fr.fromBuffer) as Tuple<
+          Fr,
+          typeof OUT_HASH_TREE_HEIGHT
+        >,
+      };
+    };
+
+    let hint = this.checkpoints[0]?.getOutHashHint();
+    const outHashes = [];
+    for (let i = 0; i < this.totalNumCheckpoints; i++) {
+      const checkpoint = this.checkpoints[i];
+      if (!checkpoint) {
+        break;
+      }
+
+      // If hints are not set yet, it must be the first checkpoint. Compute the hints with an empty tree.
+      hint ??= await computeOutHashHint([]);
+      checkpoint.setOutHashHint(hint);
+
+      // Get the out hash for this checkpoint.
+      const outHash = checkpoint.accumulateBlockOutHashes();
+      if (!outHash) {
+        break;
+      }
+      outHashes.push(outHash);
+
+      // Get or create hints for the next checkpoint.
+      hint = checkpoint.getOutHashHintForNextCheckpoint() ?? (await computeOutHashHint(outHashes));
+      checkpoint.setOutHashHintForNextCheckpoint(hint);
+    }
+  }
+
   public async setBlobAccumulators() {
     let previousAccumulator = this.startBlobAccumulator;
     // Accumulate blobs as far as we can for this epoch.

package/src/orchestrator/orchestrator.ts

@@ -8,21 +8,21 @@ import {
 } from '@aztec/constants';
 import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { padArrayEnd } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/curves/bn254';
 import { AbortError } from '@aztec/foundation/error';
-import { Fr } from '@aztec/foundation/fields';
 import { createLogger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { assertLength } from '@aztec/foundation/serialize';
 import { pushTestData } from '@aztec/foundation/testing';
 import { elapsed } from '@aztec/foundation/timer';
 import type { TreeNodeLocation } from '@aztec/foundation/trees';
-import { readAvmMinimalPublicTxInputsFromFile } from '@aztec/simulator/public/fixtures';
 import { EthAddress } from '@aztec/stdlib/block';
 import type {
   EpochProver,
   ForkMerkleTreeOperations,
   MerkleTreeWriteOperations,
   PublicInputsAndRecursiveProof,
+  ReadonlyWorldStateAccess,
   ServerCircuitProver,
 } from '@aztec/stdlib/interfaces/server';
 import type { Proof } from '@aztec/stdlib/proofs';
@@ -97,7 +97,7 @@ export class ProvingOrchestrator implements EpochProver {
   private dbs: Map<BlockNumber, MerkleTreeWriteOperations> = new Map();
 
   constructor(
-    private dbProvider: ForkMerkleTreeOperations,
+    private dbProvider: ReadonlyWorldStateAccess & ForkMerkleTreeOperations,
     private prover: ServerCircuitProver,
     private readonly proverId: EthAddress,
     telemetryClient: TelemetryClient = getTelemetryClient(),
@@ -113,6 +113,10 @@ export class ProvingOrchestrator implements EpochProver {
     return this.proverId;
   }
 
+  public getNumActiveForks() {
+    return this.dbs.size;
+  }
+
   public stop(): Promise<void> {
     this.cancel();
     return Promise.resolve();
@@ -143,6 +147,14 @@ export class ProvingOrchestrator implements EpochProver {
     this.provingPromise = promise;
   }
 
+  /**
+   * Starts a new checkpoint.
+   * @param checkpointIndex - The index of the checkpoint in the epoch.
+   * @param constants - The constants for this checkpoint.
+   * @param l1ToL2Messages - The set of L1 to L2 messages to be inserted at the beginning of this checkpoint.
+   * @param totalNumBlocks - The total number of blocks expected in the checkpoint (must be at least one).
+   * @param headerOfLastBlockInPreviousCheckpoint - The header of the last block in the previous checkpoint.
+   */
   public async startNewCheckpoint(
     checkpointIndex: number,
     constants: CheckpointConstantData,
@@ -255,7 +267,8 @@ export class ProvingOrchestrator implements EpochProver {
       await endSpongeBlob.absorb(blockEndBlobFields);
       blockProvingState.setEndSpongeBlob(endSpongeBlob);
 
-      // And also try to accumulate the blobs as far as we can:
+      // Try to accumulate the out hashes and blobs as far as we can:
+      await this.provingState.accumulateCheckpointOutHashes();
       await this.provingState.setBlobAccumulators();
     }
   }
@@ -310,7 +323,7 @@ export class ProvingOrchestrator implements EpochProver {
 
         validateTx(tx);
 
-        logger.info(`Received transaction: ${tx.hash}`);
+        logger.debug(`Received transaction: ${tx.hash}`);
 
         const startSpongeBlob = spongeBlobState.clone();
         const [hints, treeSnapshots] = await this.prepareBaseRollupInputs(
@@ -352,7 +365,8 @@ export class ProvingOrchestrator implements EpochProver {
 
     provingState.setEndSpongeBlob(spongeBlobState);
 
-    // Txs have been added to the block. Now try to accumulate the blobs as far as we can:
+    // Txs have been added to the block. Now try to accumulate the out hashes and blobs as far as we can:
+    await this.provingState.accumulateCheckpointOutHashes();
     await this.provingState.setBlobAccumulators();
   }
 
@@ -486,12 +500,7 @@ export class ProvingOrchestrator implements EpochProver {
     // is aborted and never reaches this point, it will leak the fork. We need to add a global cleanup,
     // but have to make sure it only runs once all operations are completed, otherwise some function here
     // will attempt to access the fork after it was closed.
-    logger.debug(`Cleaning up world state fork for ${blockNumber}`);
-    void this.dbs
-      .get(blockNumber)
-      ?.close()
-      .then(() => this.dbs.delete(blockNumber))
-      .catch(err => logger.error(`Error closing db for block ${blockNumber}`, err));
+    void this.cleanupDBFork(blockNumber);
   }
 
   /**
@@ -534,6 +543,21 @@ export class ProvingOrchestrator implements EpochProver {
     return epochProofResult;
   }
 
+  private async cleanupDBFork(blockNumber: BlockNumber): Promise<void> {
+    logger.debug(`Cleaning up world state fork for ${blockNumber}`);
+    const fork = this.dbs.get(blockNumber);
+    if (!fork) {
+      return;
+    }
+
+    try {
+      await fork.close();
+      this.dbs.delete(blockNumber);
+    } catch (err) {
+      logger.error(`Error closing db for block ${blockNumber}`, err);
+    }
+  }
+
   /**
    * Enqueue a job to be scheduled
    * @param provingState - The proving state object being operated on
@@ -851,19 +875,22 @@ export class ProvingOrchestrator implements EpochProver {
         },
       ),
       async result => {
-        // If the proofs were slower than the block header building, then we need to try validating the block header hashes here.
-        await this.verifyBuiltBlockAgainstSyncedState(provingState);
-
         logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
 
         const leafLocation = provingState.setBlockRootRollupProof(result);
         const checkpointProvingState = provingState.parentCheckpoint;
 
+        // If the proofs were slower than the block header building, then we need to try validating the block header hashes here.
+        await this.verifyBuiltBlockAgainstSyncedState(provingState);
+
         if (checkpointProvingState.totalNumBlocks === 1) {
           this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState);
         } else {
           this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation);
         }
+
+        // We are finished with the block at this point, ensure the fork is cleaned up
+        void this.cleanupDBFork(provingState.blockNumber);
       },
     );
   }
@@ -1207,8 +1234,6 @@ export class ProvingOrchestrator implements EpochProver {
 
     const txProvingState = provingState.getTxProvingState(txIndex);
 
-    // This function tries to do AVM proving. If there is a failure, it fakes the proof unless AVM_PROVING_STRICT is defined.
-    // Nothing downstream depends on the AVM proof yet. So having this mode lets us incrementally build the AVM circuit.
     const doAvmProving = wrapCallbackInSpan(
       this.tracer,
       'ProvingOrchestrator.prover.getAvmProof',
@@ -1217,36 +1242,13 @@ export class ProvingOrchestrator implements EpochProver {
       },
       async (signal: AbortSignal) => {
         const inputs = txProvingState.getAvmInputs();
-        try {
-          // TODO(#14234)[Unconditional PIs validation]: Remove the whole try-catch logic and
-          // just keep the next line but removing the second argument (false).
-          return await this.prover.getAvmProof(inputs, false, signal, provingState.epochNumber);
-        } catch (err) {
-          if (process.env.AVM_PROVING_STRICT) {
-            logger.error(`Error thrown when proving AVM circuit with AVM_PROVING_STRICT on`, err);
-            throw err;
-          } else {
-            logger.warn(
-              `Error thrown when proving AVM circuit but AVM_PROVING_STRICT is off. Use snapshotted
-               AVM inputs and carrying on. ${inspect(err)}.`,
-            );
-
-            try {
-              this.metrics.incAvmFallback();
-              const snapshotAvmPrivateInputs = readAvmMinimalPublicTxInputsFromFile();
-              return await this.prover.getAvmProof(snapshotAvmPrivateInputs, true, signal, provingState.epochNumber);
-            } catch (err) {
-              logger.error(`Error thrown when proving snapshotted AVM inputs.`, err);
-              throw err;
-            }
-          }
-        }
+        return await this.prover.getAvmProof(inputs, signal, provingState.epochNumber);
       },
     );
 
-    this.deferredProving(provingState, doAvmProving, proofAndVk => {
+    this.deferredProving(provingState, doAvmProving, proof => {
       logger.debug(`Proven VM for tx index: ${txIndex}`);
-      txProvingState.setAvmProof(proofAndVk);
+      txProvingState.setAvmProof(proof);
       this.checkAndEnqueueBaseRollup(provingState, txIndex);
     });
   }

package/src/orchestrator/orchestrator_metrics.ts

@@ -1,41 +1,18 @@
-import {
-  type Histogram,
-  Metrics,
-  type TelemetryClient,
-  type Tracer,
-  type UpDownCounter,
-  ValueType,
-} from '@aztec/telemetry-client';
+import { type Histogram, Metrics, type TelemetryClient, type Tracer } from '@aztec/telemetry-client';
 
 export class ProvingOrchestratorMetrics {
   public readonly tracer: Tracer;
 
   private baseRollupInputsDuration: Histogram;
-  private avmFallbackCount: UpDownCounter;
 
   constructor(client: TelemetryClient, name = 'ProvingOrchestrator') {
     this.tracer = client.getTracer(name);
     const meter = client.getMeter(name);
 
-    this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION, {
-      unit: 'ms',
-      description: 'Duration to build base rollup inputs',
-      valueType: ValueType.INT,
-    });
-
-    this.avmFallbackCount = meter.createUpDownCounter(Metrics.PROVING_ORCHESTRATOR_AVM_FALLBACK_COUNT, {
-      description: 'How many times the AVM fallback was used',
-      valueType: ValueType.INT,
-    });
-
-    this.avmFallbackCount.add(0);
+    this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION);
   }
 
   recordBaseRollupInputs(durationMs: number) {
     this.baseRollupInputsDuration.record(Math.ceil(durationMs));
   }
-
-  incAvmFallback() {
-    this.avmFallbackCount.add(1);
-  }
 }

package/src/orchestrator/tx-proving-state.ts

@@ -1,9 +1,9 @@
 import { AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED, NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH } from '@aztec/constants';
-import type { Fr } from '@aztec/foundation/fields';
-import { getAvmVkData, getVkData } from '@aztec/noir-protocol-circuits-types/server/vks';
+import type { Fr } from '@aztec/foundation/curves/bn254';
+import { getVkData } from '@aztec/noir-protocol-circuits-types/server/vks';
 import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
-import type { ProofAndVerificationKey, PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
-import { ProofData } from '@aztec/stdlib/proofs';
+import type { PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
+import { ProofData, ProofDataForFixedVk, RecursiveProof } from '@aztec/stdlib/proofs';
 import {
   type BaseRollupHints,
   PrivateBaseRollupHints,
@@ -32,7 +32,7 @@ export class TxProvingState {
     PublicChonkVerifierPublicInputs,
     typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH
   >;
-  private avm?: ProofAndVerificationKey<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>;
+  private avmProof?: RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>;
 
   constructor(
     public readonly processedTx: ProcessedTx,
@@ -46,7 +46,7 @@ export class TxProvingState {
   }
 
   public ready() {
-    return !this.requireAvmProof || (!!this.avm && !!this.publicChonkVerifier);
+    return !this.requireAvmProof || (!!this.avmProof && !!this.publicChonkVerifier);
   }
 
   public getAvmInputs(): AvmCircuitInputs {
@@ -80,8 +80,8 @@ export class TxProvingState {
     this.publicChonkVerifier = publicChonkVerifierProofAndVk;
   }
 
-  public setAvmProof(avmProofAndVk: ProofAndVerificationKey<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>) {
-    this.avm = avmProofAndVk;
+  public setAvmProof(avmProof: RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>) {
+    this.avmProof = avmProof;
   }
 
   #getPrivateBaseInputs() {
@@ -105,7 +105,7 @@ export class TxProvingState {
     if (!this.publicChonkVerifier) {
       throw new Error('Tx not ready for proving base rollup: public chonk verifier proof undefined');
     }
-    if (!this.avm) {
+    if (!this.avmProof) {
       throw new Error('Tx not ready for proving base rollup: avm proof undefined');
     }
     if (!(this.baseRollupHints instanceof PublicBaseRollupHints)) {
@@ -114,11 +114,7 @@ export class TxProvingState {
 
     const publicChonkVerifierProofData = toProofData(this.publicChonkVerifier);
 
-    const avmProofData = new ProofData(
-      this.processedTx.avmProvingRequest.inputs.publicInputs,
-      this.avm.proof,
-      getAvmVkData(),
-    );
+    const avmProofData = new ProofDataForFixedVk(this.processedTx.avmProvingRequest.inputs.publicInputs, this.avmProof);
 
     return new PublicTxBaseRollupPrivateInputs(publicChonkVerifierProofData, avmProofData, this.baseRollupHints);
   }

package/src/prover-client/factory.ts

@@ -1,4 +1,8 @@
-import type { ForkMerkleTreeOperations, ProvingJobBroker } from '@aztec/stdlib/interfaces/server';
+import type {
+  ForkMerkleTreeOperations,
+  ProvingJobBroker,
+  ReadonlyWorldStateAccess,
+} from '@aztec/stdlib/interfaces/server';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import type { ProverClientConfig } from '../config.js';
@@ -6,7 +10,7 @@ import { ProverClient } from './prover-client.js';
 
 export function createProverClient(
   config: ProverClientConfig,
-  worldState: ForkMerkleTreeOperations,
+  worldState: ForkMerkleTreeOperations & ReadonlyWorldStateAccess,
   broker: ProvingJobBroker,
   telemetry: TelemetryClient = getTelemetryClient(),
 ) {

package/src/prover-client/prover-client.ts

@@ -11,6 +11,7 @@ import {
   type ProvingJobBroker,
   type ProvingJobConsumer,
   type ProvingJobProducer,
+  type ReadonlyWorldStateAccess,
   type ServerCircuitProver,
   tryStop,
 } from '@aztec/stdlib/interfaces/server';
@@ -33,7 +34,7 @@ export class ProverClient implements EpochProverManager {
 
   private constructor(
     private config: ProverClientConfig,
-    private worldState: ForkMerkleTreeOperations,
+    private worldState: ForkMerkleTreeOperations & ReadonlyWorldStateAccess,
     private orchestratorClient: ProvingJobProducer,
     private agentClient?: ProvingJobConsumer,
     private telemetry: TelemetryClient = getTelemetryClient(),
@@ -99,7 +100,7 @@ export class ProverClient implements EpochProverManager {
    */
   public static async new(
     config: ProverClientConfig,
-    worldState: ForkMerkleTreeOperations,
+    worldState: ForkMerkleTreeOperations & ReadonlyWorldStateAccess,
     broker: ProvingJobBroker,
     telemetry: TelemetryClient = getTelemetryClient(),
   ) {
@@ -129,15 +130,7 @@ export class ProverClient implements EpochProverManager {
     const prover = await buildServerCircuitProver(this.config, this.telemetry);
     this.agents = times(
       this.config.proverAgentCount,
-      () =>
-        new ProvingAgent(
-          this.agentClient!,
-          proofStore,
-          prover,
-          [],
-          this.config.proverAgentPollIntervalMs,
-          this.telemetry,
-        ),
+      () => new ProvingAgent(this.agentClient!, proofStore, prover, [], this.config.proverAgentPollIntervalMs),
     );
 
     await Promise.all(this.agents.map(agent => agent.start()));

package/src/prover-client/server-epoch-prover.ts

@@ -1,6 +1,6 @@
 import type { BatchedBlob, FinalBlobBatchingChallenges } from '@aztec/blob-lib/types';
 import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
-import type { Fr } from '@aztec/foundation/fields';
+import type { Fr } from '@aztec/foundation/curves/bn254';
 import type { EthAddress } from '@aztec/stdlib/block';
 import type { EpochProver } from '@aztec/stdlib/interfaces/server';
 import type { Proof } from '@aztec/stdlib/proofs';

package/src/proving_broker/broker_prover_facade.ts

@@ -5,14 +5,12 @@ import type {
   RECURSIVE_PROOF_LENGTH,
 } from '@aztec/constants';
 import { EpochNumber } from '@aztec/foundation/branded-types';
-import { sha256 } from '@aztec/foundation/crypto';
-import { Fr } from '@aztec/foundation/fields';
+import { sha256 } from '@aztec/foundation/crypto/sha256';
 import { createLogger } from '@aztec/foundation/log';
 import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { truncate } from '@aztec/foundation/string';
 import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
 import {
-  type ProofAndVerificationKey,
   type ProofUri,
   type ProvingJobId,
   type ProvingJobInputsMap,
@@ -24,7 +22,7 @@ import {
   makeProvingJobId,
 } from '@aztec/stdlib/interfaces/server';
 import type { ParityBasePrivateInputs, ParityPublicInputs, ParityRootPrivateInputs } from '@aztec/stdlib/parity';
-import { ProvingRequestType } from '@aztec/stdlib/proofs';
+import { ProvingRequestType, RecursiveProof } from '@aztec/stdlib/proofs';
 import type {
   BlockMergeRollupPrivateInputs,
   BlockRollupPublicInputs,
@@ -398,24 +396,16 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver {
 
   getAvmProof(
     inputs: AvmCircuitInputs,
-    skipPublicInputsValidation?: boolean, // TODO(#14234)[Unconditional PIs validation]: remove this argument
     signal?: AbortSignal,
     epochNumber?: EpochNumber,
-  ): Promise<ProofAndVerificationKey<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>> {
-    this.log.info(`getAvmProof() called with skipPublicInputsValidation: ${skipPublicInputsValidation}`);
-
+  ): Promise<RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED>> {
     return this.enqueueJob(
       this.generateId(ProvingRequestType.PUBLIC_VM, inputs, epochNumber),
       ProvingRequestType.PUBLIC_VM,
       inputs,
       epochNumber,
       signal,
-    ).then(result => {
-      // TODO(#14234)[Unconditional PIs validation]: Remove ".then()".
-      // Override the default value of skipPublicInputsValidation potentially set in BBNativeRollupProver.getAvmProof().
-      result.proof.proof[0] = skipPublicInputsValidation ? new Fr(1) : new Fr(0);
-      return result;
-    });
+    );
   }
 
   getBaseParityProof(

package/src/proving_broker/config.ts

@@ -1,4 +1,4 @@
-import { type L1ReaderConfig, l1ReaderConfigMappings } from '@aztec/ethereum';
+import { type L1ReaderConfig, l1ReaderConfigMappings } from '@aztec/ethereum/l1-reader';
 import {
   type ConfigMappingsType,
   booleanConfigHelper,

package/src/proving_broker/fixtures.ts

@@ -1,5 +1,5 @@
 import { EpochNumber } from '@aztec/foundation/branded-types';
-import { randomBytes } from '@aztec/foundation/crypto';
+import { randomBytes } from '@aztec/foundation/crypto/random';
 import { type ProofUri, type ProvingJobId, makeProvingJobId } from '@aztec/stdlib/interfaces/server';
 import { ProvingRequestType } from '@aztec/stdlib/proofs';
 

package/src/proving_broker/proving_agent.ts

@@ -13,27 +13,16 @@ import type {
   ServerCircuitProver,
 } from '@aztec/stdlib/interfaces/server';
 import { ProvingRequestType } from '@aztec/stdlib/proofs';
-import {
-  type TelemetryClient,
-  type Traceable,
-  type Tracer,
-  getTelemetryClient,
-  trackSpan,
-} from '@aztec/telemetry-client';
 
 import type { ProofStore } from './proof_store/index.js';
-import { ProvingAgentInstrumentation } from './proving_agent_instrumentation.js';
 import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_controller.js';
 
 /**
  * A helper class that encapsulates a circuit prover and connects it to a job source.
  */
-export class ProvingAgent implements Traceable {
+export class ProvingAgent {
   private currentJobController?: ProvingJobController;
   private runningPromise: RunningPromise;
-  private instrumentation: ProvingAgentInstrumentation;
-
-  public readonly tracer: Tracer;
 
   constructor(
     /** The source of proving jobs */
@@ -46,12 +35,8 @@ export class ProvingAgent implements Traceable {
     private proofAllowList: Array<ProvingRequestType> = [],
     /** How long to wait between jobs */
     private pollIntervalMs = 1000,
-    /** A telemetry client through which to emit metrics */
-    client: TelemetryClient = getTelemetryClient(),
     private log = createLogger('prover-client:proving-agent'),
   ) {
-    this.tracer = client.getTracer('ProvingAgent');
-    this.instrumentation = new ProvingAgentInstrumentation(client);
     this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollIntervalMs);
   }
 
@@ -85,7 +70,6 @@ export class ProvingAgent implements Traceable {
     return this.runningPromise.isRunning() ? { status: 'running' } : { status: 'stopped' };
   }
 
-  @trackSpan('ProvingAgent.safeWork')
   private async work() {
     // every tick we need to take one of the following actions:
     // 1. send a hearbeat to the broker that we're working on some job

package/src/proving_broker/proving_broker.ts

@@ -16,13 +16,7 @@ import {
   tryStop,
 } from '@aztec/stdlib/interfaces/server';
 import { ProvingRequestType } from '@aztec/stdlib/proofs';
-import {
-  type TelemetryClient,
-  type Traceable,
-  type Tracer,
-  getTelemetryClient,
-  trackSpan,
-} from '@aztec/telemetry-client';
+import { type TelemetryClient, type Traceable, type Tracer, getTelemetryClient } from '@aztec/telemetry-client';
 
 import assert from 'assert';
 
@@ -302,6 +296,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
       this.resultsCache.delete(id);
       this.inProgress.delete(id);
       this.retries.delete(id);
+      this.enqueuedAt.delete(id);
     }
   }
 
@@ -354,6 +349,8 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
           const enqueuedAt = this.enqueuedAt.get(job.id);
           if (enqueuedAt) {
             this.instrumentation.recordJobWait(job.type, enqueuedAt);
+            // we can clear this flag now.
+            this.enqueuedAt.delete(job.id);
           }
 
           return { job, time };
@@ -562,7 +559,6 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Tr
     return this.#getProvingJob(filter);
   }
 
-  @trackSpan('ProvingBroker.cleanupPass')
   private async cleanupPass() {
     this.cleanupStaleJobs();
     this.reEnqueueExpiredJobs();

package/src/proving_broker/proving_broker_database/persisted.ts

@@ -11,7 +11,14 @@ import {
   ProvingJobSettledResult,
   getEpochFromProvingJobId,
 } from '@aztec/stdlib/interfaces/server';
-import { Attributes, LmdbMetrics, type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
+import {
+  Attributes,
+  LmdbMetrics,
+  type TelemetryClient,
+  type Tracer,
+  getTelemetryClient,
+  trackSpan,
+} from '@aztec/telemetry-client';
 
 import { mkdir, readdir } from 'fs/promises';
 import { join } from 'path';
@@ -78,6 +85,8 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
 
   private batchQueue: BatchQueue<ProvingJob | [ProvingJobId, ProvingJobSettledResult], number>;
 
+  public readonly tracer: Tracer;
+
   private constructor(
     private epochs: Map<number, SingleEpochDatabase>,
     private config: ProverBrokerConfig,
@@ -92,6 +101,8 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
       () => this.estimateSize(),
     );
 
+    this.tracer = client.getTracer('KVBrokerDatabase');
+
     this.batchQueue = new BatchQueue(
       (items, key) => this.commitWrites(items, key),
       config.proverBrokerBatchSize,
@@ -165,6 +176,9 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
     }
   }
 
+  @trackSpan('KVBrokerDatabase.deleteAllProvingJobsOlderThanEpoch', epochNumber => ({
+    [Attributes.EPOCH_NUMBER]: epochNumber,
+  }))
   async deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void> {
     const oldEpochs = Array.from(this.epochs.keys()).filter(e => e < Number(epochNumber));
     for (const old of oldEpochs) {