@aztec/prover-client 0.47.0 → 0.48.0

package/src/orchestrator/orchestrator_metrics.ts

@@ -0,0 +1,32 @@
+import {
+  type Histogram,
+  Metrics,
+  type TelemetryClient,
+  type Tracer,
+  ValueType,
+  millisecondBuckets,
+} from '@aztec/telemetry-client';
+
+export class ProvingOrchestratorMetrics {
+  public readonly tracer: Tracer;
+
+  private baseRollupInputsDuration: Histogram;
+
+  constructor(client: TelemetryClient, name = 'ProvingOrchestrator') {
+    this.tracer = client.getTracer(name);
+    const meter = client.getMeter(name);
+
+    this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION, {
+      unit: 'ms',
+      description: 'Duration to build base rollup inputs',
+      valueType: ValueType.INT,
+      advice: {
+        explicitBucketBoundaries: millisecondBuckets(1), // 10ms -> ~327s
+      },
+    });
+  }
+
+  recordBaseRollupInputs(durationMs: number) {
+    this.baseRollupInputsDuration.record(Math.ceil(durationMs));
+  }
+}

package/src/orchestrator/proving-state.ts

@@ -143,6 +143,11 @@ export class ProvingState {
     return this.txs;
   }
 
+  /** Returns the block number as an epoch number. Used for prioritizing proof requests. */
+  public get epochNumber(): number {
+    return this.globalVariables.blockNumber.toNumber();
+  }
+
   /**
    * Stores the inputs to a merge circuit and determines if the circuit is ready to be executed
    * @param mergeInputs - The inputs to store

package/src/orchestrator/tx-proving-state.ts

@@ -11,6 +11,7 @@ import {
   type BaseRollupInputs,
   type NESTED_RECURSIVE_PROOF_LENGTH,
   type Proof,
+  type RECURSIVE_PROOF_LENGTH,
   type RecursiveProof,
   type VerificationKeyData,
 } from '@aztec/circuits.js';
@@ -48,23 +49,17 @@ export class TxProvingState {
     public readonly processedTx: ProcessedTx,
     public readonly baseRollupInputs: BaseRollupInputs,
     public readonly treeSnapshots: Map<MerkleTreeId, AppendOnlyTreeSnapshot>,
-    privateKernelVk: VerificationKeyData,
   ) {
     let previousProofType = PublicKernelType.NON_PUBLIC;
     for (let i = 0; i < processedTx.publicProvingRequests.length; i++) {
       const provingRequest = processedTx.publicProvingRequests[i];
       const kernelRequest = provingRequest.type === AVM_REQUEST ? provingRequest.kernelRequest : provingRequest;
-      // the first circuit has a valid previous proof, it came from private
-      if (i === 0) {
-        kernelRequest.inputs.previousKernel.vk = privateKernelVk;
-        kernelRequest.inputs.previousKernel.clientIvcProof = processedTx.clientIvcProof;
-      }
       const vmRequest = provingRequest.type === AVM_REQUEST ? provingRequest : undefined;
       const publicFunction: PublicFunction = {
         vmRequest,
         vmProof: undefined,
         previousProofType,
-        previousKernelProven: i === 0,
+        previousKernelProven: false,
         publicKernelRequest: kernelRequest,
       };
       this.publicFunctions.push(publicFunction);
@@ -105,6 +100,37 @@ export class TxProvingState {
     return { code: TX_PROVING_CODE.READY, function: nextFunction };
   }
 
+  // Updates the transaction's proving state after completion of a tube proof
+  // Returns an instruction as to the next stage of tx proving
+  public getNextPublicKernelFromTubeProof(
+    proof: RecursiveProof<typeof RECURSIVE_PROOF_LENGTH>,
+    verificationKey: VerificationKeyData,
+  ): TxProvingInstruction {
+    const nextKernelIndex = 0;
+    if (nextKernelIndex >= this.publicFunctions.length) {
+      // The next kernel index is greater than our set of functions, we are done!
+      return { code: TX_PROVING_CODE.COMPLETED, function: undefined };
+    }
+
+    // There is more work to do, are we ready?
+    const nextFunction = this.publicFunctions[nextKernelIndex];
+
+    // pass both the proof and verification key forward to the next circuit
+    nextFunction.publicKernelRequest.inputs.previousKernel.proof = proof;
+    nextFunction.publicKernelRequest.inputs.previousKernel.vk = verificationKey;
+
+    // We need to update this so the state machine knows this proof is ready
+    nextFunction.previousKernelProven = true;
+    nextFunction.previousProofType = PublicKernelType.NON_PUBLIC;
+    if (nextFunction.vmProof === undefined) {
+      // The VM proof for the next function is not ready
+      return { code: TX_PROVING_CODE.NOT_READY, function: undefined };
+    }
+
+    // The VM proof is ready, we can continue
+    return { code: TX_PROVING_CODE.READY, function: nextFunction };
+  }
+
   // Updates the transaction's proving state after completion of a VM proof
   // Returns an instruction as to the next stage of tx proving
   public getNextPublicKernelFromVMProof(provenIndex: number, proof: Proof): TxProvingInstruction {

package/src/prover-agent/memory-proving-queue.ts

@@ -31,35 +31,43 @@ import type {
 } from '@aztec/circuits.js';
 import { randomBytes } from '@aztec/foundation/crypto';
 import { AbortError, TimeoutError } from '@aztec/foundation/error';
-import { MemoryFifo } from '@aztec/foundation/fifo';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
-
-type ProvingJobWithResolvers<T extends ProvingRequest = ProvingRequest> = {
-  id: string;
-  request: T;
-  signal?: AbortSignal;
-  attempts: number;
-  heartbeat: number;
-} & PromiseWithResolvers<ProvingRequestResult<T['type']>>;
+import { PriorityMemoryQueue } from '@aztec/foundation/queue';
+import { serializeToBuffer } from '@aztec/foundation/serialize';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+
+import { ProvingQueueMetrics } from './queue_metrics.js';
+
+type ProvingJobWithResolvers<T extends ProvingRequest = ProvingRequest> = ProvingJob<T> &
+  PromiseWithResolvers<ProvingRequestResult<T['type']>> & {
+    signal?: AbortSignal;
+    epochNumber?: number;
+    attempts: number;
+    heartbeat: number;
+  };
 
 const MAX_RETRIES = 3;
 
 const defaultIdGenerator = () => randomBytes(4).toString('hex');
 const defaultTimeSource = () => Date.now();
-
 /**
  * A helper class that sits in between services that need proofs created and agents that can create them.
- * The queue accumulates jobs and provides them to agents in FIFO order.
+ * The queue accumulates jobs and provides them to agents prioritized by block number.
  */
 export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource {
   private log = createDebugLogger('aztec:prover-client:prover-pool:queue');
-  private queue = new MemoryFifo<ProvingJobWithResolvers>();
+  private queue = new PriorityMemoryQueue<ProvingJobWithResolvers>(
+    (a, b) => (a.epochNumber ?? 0) - (b.epochNumber ?? 0),
+  );
   private jobsInProgress = new Map<string, ProvingJobWithResolvers>();
 
   private runningPromise: RunningPromise;
 
+  private metrics: ProvingQueueMetrics;
+
   constructor(
+    client: TelemetryClient,
     /** Timeout the job if an agent doesn't report back in this time */
     private jobTimeoutMs = 60 * 1000,
     /** How often to check for timed out jobs */
@@ -67,6 +75,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
     private generateId = defaultIdGenerator,
     private timeSource = defaultTimeSource,
   ) {
+    this.metrics = new ProvingQueueMetrics(client, 'MemoryProvingQueue');
     this.runningPromise = new RunningPromise(this.poll, pollingIntervalMs);
   }
 
@@ -156,7 +165,8 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
       return Promise.resolve();
     }
 
-    if (job.attempts < MAX_RETRIES) {
+    // every job should be retried with the exception of the public VM since its in development and can fail
+    if (job.attempts < MAX_RETRIES && job.request.type !== ProvingRequestType.PUBLIC_VM) {
       job.attempts++;
       this.log.warn(
         `Job id=${job.id} type=${ProvingRequestType[job.request.type]} failed with error: ${err}. Retry ${
@@ -190,6 +200,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
 
   private poll = () => {
     const now = this.timeSource();
+    this.metrics.recordQueueSize(this.queue.length());
 
     for (const job of this.jobsInProgress.values()) {
       if (job.signal?.aborted) {
@@ -210,6 +221,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   private enqueue<T extends ProvingRequest>(
     request: T,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<ProvingRequestResult<T['type']>> {
     if (!this.runningPromise.isRunning()) {
       return Promise.reject(new Error('Proving queue is not running.'));
@@ -225,6 +237,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
       reject,
       attempts: 1,
       heartbeat: 0,
+      epochNumber,
     };
 
     if (signal) {
@@ -239,28 +252,34 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
       throw new Error();
     }
 
+    const byteSize = serializeToBuffer(item.request.inputs).length;
+    this.metrics.recordNewJob(item.request.type, byteSize);
+
     return promise;
   }
 
   getEmptyPrivateKernelProof(
     inputs: PrivateKernelEmptyInputData,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<KernelCircuitPublicInputs>> {
-    return this.enqueue({ type: ProvingRequestType.PRIVATE_KERNEL_EMPTY, inputs }, signal);
+    return this.enqueue({ type: ProvingRequestType.PRIVATE_KERNEL_EMPTY, inputs }, signal, epochNumber);
   }
 
   getTubeProof(
     inputs: TubeInputs,
-    signal?: AbortSignal | undefined,
+    signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<{ tubeVK: VerificationKeyData; tubeProof: RecursiveProof<typeof RECURSIVE_PROOF_LENGTH> }> {
-    return this.enqueue({ type: ProvingRequestType.TUBE_PROOF, inputs }, signal);
+    return this.enqueue({ type: ProvingRequestType.TUBE_PROOF, inputs }, signal, epochNumber);
   }
 
   getEmptyTubeProof(
     inputs: PrivateKernelEmptyInputData,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<KernelCircuitPublicInputs>> {
-    return this.enqueue({ type: ProvingRequestType.PRIVATE_KERNEL_EMPTY, inputs }, signal);
+    return this.enqueue({ type: ProvingRequestType.PRIVATE_KERNEL_EMPTY, inputs }, signal, epochNumber);
   }
 
   /**
@@ -270,14 +289,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getBaseParityProof(
     inputs: BaseParityInputs,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<RootParityInput<typeof RECURSIVE_PROOF_LENGTH>> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.BASE_PARITY,
-        inputs,
-      },
-      signal,
-    );
+    return this.enqueue({ type: ProvingRequestType.BASE_PARITY, inputs }, signal, epochNumber);
   }
 
   /**
@@ -287,14 +301,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getRootParityProof(
     inputs: RootParityInputs,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<RootParityInput<typeof NESTED_RECURSIVE_PROOF_LENGTH>> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.ROOT_PARITY,
-        inputs,
-      },
-      signal,
-    );
+    return this.enqueue({ type: ProvingRequestType.ROOT_PARITY, inputs }, signal, epochNumber);
   }
 
   /**
@@ -304,14 +313,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getBaseRollupProof(
     baseRollupInput: BaseRollupInputs,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<BaseOrMergeRollupPublicInputs>> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.BASE_ROLLUP,
-        inputs: baseRollupInput,
-      },
-      signal,
-    );
+    return this.enqueue({ type: ProvingRequestType.BASE_ROLLUP, inputs: baseRollupInput }, signal, epochNumber);
   }
 
   /**
@@ -321,14 +325,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getMergeRollupProof(
     input: MergeRollupInputs,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<BaseOrMergeRollupPublicInputs>> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.MERGE_ROLLUP,
-        inputs: input,
-      },
-      signal,
-    );
+    return this.enqueue({ type: ProvingRequestType.MERGE_ROLLUP, inputs: input }, signal, epochNumber);
   }
 
   /**
@@ -338,14 +337,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getRootRollupProof(
     input: RootRollupInputs,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<RootRollupPublicInputs>> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.ROOT_ROLLUP,
-        inputs: input,
-      },
-      signal,
-    );
+    return this.enqueue({ type: ProvingRequestType.ROOT_ROLLUP, inputs: input }, signal, epochNumber);
   }
 
   /**
@@ -355,14 +349,12 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getPublicKernelProof(
     kernelRequest: PublicKernelNonTailRequest,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<PublicKernelCircuitPublicInputs>> {
     return this.enqueue(
-      {
-        type: ProvingRequestType.PUBLIC_KERNEL_NON_TAIL,
-        kernelType: kernelRequest.type,
-        inputs: kernelRequest.inputs,
-      },
+      { type: ProvingRequestType.PUBLIC_KERNEL_NON_TAIL, kernelType: kernelRequest.type, inputs: kernelRequest.inputs },
       signal,
+      epochNumber,
     );
   }
 
@@ -373,28 +365,20 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getPublicTailProof(
     kernelRequest: PublicKernelTailRequest,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<KernelCircuitPublicInputs>> {
     return this.enqueue(
-      {
-        type: ProvingRequestType.PUBLIC_KERNEL_TAIL,
-        kernelType: kernelRequest.type,
-        inputs: kernelRequest.inputs,
-      },
+      { type: ProvingRequestType.PUBLIC_KERNEL_TAIL, kernelType: kernelRequest.type, inputs: kernelRequest.inputs },
       signal,
+      epochNumber,
     );
   }
 
   /**
    * Creates an AVM proof.
    */
-  getAvmProof(inputs: AvmCircuitInputs, signal?: AbortSignal | undefined): Promise<ProofAndVerificationKey> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.PUBLIC_VM,
-        inputs,
-      },
-      signal,
-    );
+  getAvmProof(inputs: AvmCircuitInputs, signal?: AbortSignal, epochNumber?: number): Promise<ProofAndVerificationKey> {
+    return this.enqueue({ type: ProvingRequestType.PUBLIC_VM, inputs }, signal, epochNumber);
   }
 
   /**

package/src/prover-agent/prover-agent.ts

@@ -12,11 +12,20 @@ import { elapsed } from '@aztec/foundation/timer';
 
 import { ProvingError } from './proving-error.js';
 
+const PRINT_THRESHOLD_NS = 6e10; // 60 seconds
+
 /**
  * A helper class that encapsulates a circuit prover and connects it to a job source.
  */
 export class ProverAgent {
-  private inFlightPromises = new Map<string, Promise<any>>();
+  private inFlightPromises = new Map<
+    string,
+    {
+      id: string;
+      type: ProvingRequestType;
+      promise: Promise<any>;
+    }
+  >();
   private runningPromise?: RunningPromise;
 
   constructor(
@@ -49,11 +58,26 @@ export class ProverAgent {
       throw new Error('Agent is already running');
     }
 
+    let lastPrint = process.hrtime.bigint();
+
     this.runningPromise = new RunningPromise(async () => {
       for (const jobId of this.inFlightPromises.keys()) {
         await jobSource.heartbeat(jobId);
       }
 
+      const now = process.hrtime.bigint();
+
+      if (now - lastPrint >= PRINT_THRESHOLD_NS) {
+        // only log if we're actually doing work
+        if (this.inFlightPromises.size > 0) {
+          const jobs = Array.from(this.inFlightPromises.values())
+            .map(job => `id=${job.id},type=${ProvingRequestType[job.type]}`)
+            .join(' ');
+          this.log.info(`Agent is running with ${this.inFlightPromises.size} in-flight jobs: ${jobs}`);
+        }
+        lastPrint = now;
+      }
+
       while (this.inFlightPromises.size < this.maxConcurrency) {
         try {
           const job = await jobSource.getProvingJob();
@@ -62,10 +86,20 @@ export class ProverAgent {
             return;
           }
 
-          const promise = this.work(jobSource, job).finally(() => this.inFlightPromises.delete(job.id));
-          this.inFlightPromises.set(job.id, promise);
+          try {
+            const promise = this.work(jobSource, job).finally(() => this.inFlightPromises.delete(job.id));
+            this.inFlightPromises.set(job.id, {
+              id: job.id,
+              type: job.request.type,
+              promise,
+            });
+          } catch (err) {
+            this.log.warn(
+              `Error processing job! type=${ProvingRequestType[job.request.type]}: ${err}. ${(err as Error).stack}`,
+            );
+          }
         } catch (err) {
-          this.log.warn(`Error processing job: ${err}`);
+          this.log.error(`Error fetching job`, err);
         }
       }
     }, this.pollIntervalMs);
@@ -90,12 +124,12 @@ export class ProverAgent {
       this.log.debug(`Picked up proving job id=${job.id} type=${ProvingRequestType[job.request.type]}`);
       const [time, result] = await elapsed(this.getProof(job.request));
       if (this.isRunning()) {
-        this.log.debug(
+        this.log.verbose(
           `Processed proving job id=${job.id} type=${ProvingRequestType[job.request.type]} duration=${time}ms`,
         );
         await jobSource.resolveProvingJob(job.id, result);
       } else {
-        this.log.debug(
+        this.log.verbose(
           `Dropping proving job id=${job.id} type=${
             ProvingRequestType[job.request.type]
           } duration=${time}ms: agent stopped`,
@@ -107,10 +141,11 @@ export class ProverAgent {
           `Error processing proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: ${
             (err as any).stack || err
           }`,
+          err,
         );
         await jobSource.rejectProvingJob(job.id, new ProvingError((err as any)?.message ?? String(err)));
       } else {
-        this.log.debug(
+        this.log.verbose(
           `Dropping proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: agent stopped: ${
             (err as any).stack || err
           }`,

package/src/prover-agent/queue_metrics.ts

@@ -0,0 +1,29 @@
+import { ProvingRequestType } from '@aztec/circuit-types';
+import { Attributes, type Gauge, type Histogram, Metrics, type TelemetryClient } from '@aztec/telemetry-client';
+
+export class ProvingQueueMetrics {
+  private jobSize: Histogram;
+  private queueSize: Gauge;
+
+  constructor(client: TelemetryClient, name = 'ProvingQueueMetrics') {
+    const meter = client.getMeter(name);
+    this.jobSize = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_SIZE, {
+      description: 'Size of proving job',
+      unit: 'by',
+    });
+
+    this.queueSize = meter.createGauge(Metrics.PROVING_QUEUE_SIZE, {
+      description: 'Size of proving queue',
+    });
+  }
+
+  recordNewJob(type: ProvingRequestType, size: number) {
+    this.jobSize.record(size, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[type],
+    });
+  }
+
+  recordQueueSize(size: number) {
+    this.queueSize.record(size);
+  }
+}

package/src/prover-agent/rpc.ts

@@ -9,6 +9,7 @@ import {
   KernelCircuitPublicInputs,
   MergeRollupInputs,
   ParityPublicInputs,
+  PrivateKernelEmptyInputData,
   Proof,
   PublicKernelCircuitPrivateInputs,
   PublicKernelCircuitPublicInputs,
@@ -41,6 +42,7 @@ export function createProvingJobSourceServer(queue: ProvingJobSource): JsonRpcSe
       ParityPublicInputs,
       Proof,
       ProvingError,
+      PrivateKernelEmptyInputData,
       PublicKernelCircuitPrivateInputs,
       PublicKernelCircuitPublicInputs,
       PublicKernelTailCircuitPrivateInputs,
@@ -75,6 +77,7 @@ export function createProvingJobSourceClient(
       ParityPublicInputs,
       Proof,
       ProvingError,
+      PrivateKernelEmptyInputData,
       PublicKernelCircuitPrivateInputs,
       PublicKernelCircuitPublicInputs,
       PublicKernelTailCircuitPrivateInputs,

package/src/tx-prover/factory.ts

@@ -1,16 +1,9 @@
-import { type L2BlockSource } from '@aztec/circuit-types';
 import { type TelemetryClient } from '@aztec/telemetry-client';
 import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
-import { type WorldStateSynchronizer } from '@aztec/world-state';
 
 import { type ProverClientConfig } from '../config.js';
 import { TxProver } from './tx-prover.js';
 
-export function createProverClient(
-  config: ProverClientConfig,
-  worldStateSynchronizer: WorldStateSynchronizer,
-  blockSource: L2BlockSource,
-  telemetry: TelemetryClient = new NoopTelemetryClient(),
-) {
-  return config.disableProver ? undefined : TxProver.new(config, worldStateSynchronizer, blockSource, telemetry);
+export function createProverClient(config: ProverClientConfig, telemetry: TelemetryClient = new NoopTelemetryClient()) {
+  return config.disableProver ? undefined : TxProver.new(config, telemetry);
 }