@aztec/prover-client 0.47.1 → 0.49.2

package/src/prover-agent/memory-proving-queue.ts

@@ -31,35 +31,43 @@ import type {
 } from '@aztec/circuits.js';
 import { randomBytes } from '@aztec/foundation/crypto';
 import { AbortError, TimeoutError } from '@aztec/foundation/error';
-import { MemoryFifo } from '@aztec/foundation/fifo';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
-
-type ProvingJobWithResolvers<T extends ProvingRequest = ProvingRequest> = {
-  id: string;
-  request: T;
-  signal?: AbortSignal;
-  attempts: number;
-  heartbeat: number;
-} & PromiseWithResolvers<ProvingRequestResult<T['type']>>;
+import { PriorityMemoryQueue } from '@aztec/foundation/queue';
+import { serializeToBuffer } from '@aztec/foundation/serialize';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+
+import { ProvingQueueMetrics } from './queue_metrics.js';
+
+type ProvingJobWithResolvers<T extends ProvingRequest = ProvingRequest> = ProvingJob<T> &
+  PromiseWithResolvers<ProvingRequestResult<T['type']>> & {
+    signal?: AbortSignal;
+    epochNumber?: number;
+    attempts: number;
+    heartbeat: number;
+  };
 
 const MAX_RETRIES = 3;
 
 const defaultIdGenerator = () => randomBytes(4).toString('hex');
 const defaultTimeSource = () => Date.now();
-
 /**
  * A helper class that sits in between services that need proofs created and agents that can create them.
- * The queue accumulates jobs and provides them to agents in FIFO order.
+ * The queue accumulates jobs and provides them to agents prioritized by block number.
  */
 export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource {
   private log = createDebugLogger('aztec:prover-client:prover-pool:queue');
-  private queue = new MemoryFifo<ProvingJobWithResolvers>();
+  private queue = new PriorityMemoryQueue<ProvingJobWithResolvers>(
+    (a, b) => (a.epochNumber ?? 0) - (b.epochNumber ?? 0),
+  );
   private jobsInProgress = new Map<string, ProvingJobWithResolvers>();
 
   private runningPromise: RunningPromise;
 
+  private metrics: ProvingQueueMetrics;
+
   constructor(
+    client: TelemetryClient,
     /** Timeout the job if an agent doesn't report back in this time */
     private jobTimeoutMs = 60 * 1000,
     /** How often to check for timed out jobs */
@@ -67,6 +75,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
     private generateId = defaultIdGenerator,
     private timeSource = defaultTimeSource,
   ) {
+    this.metrics = new ProvingQueueMetrics(client, 'MemoryProvingQueue');
     this.runningPromise = new RunningPromise(this.poll, pollingIntervalMs);
   }
 
@@ -156,7 +165,8 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
       return Promise.resolve();
     }
 
-    if (job.attempts < MAX_RETRIES) {
+    // every job should be retried with the exception of the public VM since its in development and can fail
+    if (job.attempts < MAX_RETRIES && job.request.type !== ProvingRequestType.PUBLIC_VM) {
       job.attempts++;
       this.log.warn(
         `Job id=${job.id} type=${ProvingRequestType[job.request.type]} failed with error: ${err}. Retry ${
@@ -190,6 +200,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
 
   private poll = () => {
     const now = this.timeSource();
+    this.metrics.recordQueueSize(this.queue.length());
 
     for (const job of this.jobsInProgress.values()) {
       if (job.signal?.aborted) {
@@ -210,6 +221,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   private enqueue<T extends ProvingRequest>(
     request: T,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<ProvingRequestResult<T['type']>> {
     if (!this.runningPromise.isRunning()) {
       return Promise.reject(new Error('Proving queue is not running.'));
@@ -225,6 +237,7 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
       reject,
       attempts: 1,
       heartbeat: 0,
+      epochNumber,
     };
 
     if (signal) {
@@ -239,28 +252,34 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
       throw new Error();
     }
 
+    const byteSize = serializeToBuffer(item.request.inputs).length;
+    this.metrics.recordNewJob(item.request.type, byteSize);
+
     return promise;
   }
 
   getEmptyPrivateKernelProof(
     inputs: PrivateKernelEmptyInputData,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<KernelCircuitPublicInputs>> {
-    return this.enqueue({ type: ProvingRequestType.PRIVATE_KERNEL_EMPTY, inputs }, signal);
+    return this.enqueue({ type: ProvingRequestType.PRIVATE_KERNEL_EMPTY, inputs }, signal, epochNumber);
   }
 
   getTubeProof(
     inputs: TubeInputs,
-    signal?: AbortSignal | undefined,
+    signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<{ tubeVK: VerificationKeyData; tubeProof: RecursiveProof<typeof RECURSIVE_PROOF_LENGTH> }> {
-    return this.enqueue({ type: ProvingRequestType.TUBE_PROOF, inputs }, signal);
+    return this.enqueue({ type: ProvingRequestType.TUBE_PROOF, inputs }, signal, epochNumber);
   }
 
   getEmptyTubeProof(
     inputs: PrivateKernelEmptyInputData,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<KernelCircuitPublicInputs>> {
-    return this.enqueue({ type: ProvingRequestType.PRIVATE_KERNEL_EMPTY, inputs }, signal);
+    return this.enqueue({ type: ProvingRequestType.PRIVATE_KERNEL_EMPTY, inputs }, signal, epochNumber);
   }
 
   /**
@@ -270,14 +289,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getBaseParityProof(
     inputs: BaseParityInputs,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<RootParityInput<typeof RECURSIVE_PROOF_LENGTH>> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.BASE_PARITY,
-        inputs,
-      },
-      signal,
-    );
+    return this.enqueue({ type: ProvingRequestType.BASE_PARITY, inputs }, signal, epochNumber);
   }
 
   /**
@@ -287,14 +301,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getRootParityProof(
     inputs: RootParityInputs,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<RootParityInput<typeof NESTED_RECURSIVE_PROOF_LENGTH>> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.ROOT_PARITY,
-        inputs,
-      },
-      signal,
-    );
+    return this.enqueue({ type: ProvingRequestType.ROOT_PARITY, inputs }, signal, epochNumber);
   }
 
   /**
@@ -304,14 +313,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getBaseRollupProof(
     baseRollupInput: BaseRollupInputs,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<BaseOrMergeRollupPublicInputs>> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.BASE_ROLLUP,
-        inputs: baseRollupInput,
-      },
-      signal,
-    );
+    return this.enqueue({ type: ProvingRequestType.BASE_ROLLUP, inputs: baseRollupInput }, signal, epochNumber);
   }
 
   /**
@@ -321,14 +325,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getMergeRollupProof(
     input: MergeRollupInputs,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<BaseOrMergeRollupPublicInputs>> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.MERGE_ROLLUP,
-        inputs: input,
-      },
-      signal,
-    );
+    return this.enqueue({ type: ProvingRequestType.MERGE_ROLLUP, inputs: input }, signal, epochNumber);
   }
 
   /**
@@ -338,14 +337,9 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getRootRollupProof(
     input: RootRollupInputs,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<RootRollupPublicInputs>> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.ROOT_ROLLUP,
-        inputs: input,
-      },
-      signal,
-    );
+    return this.enqueue({ type: ProvingRequestType.ROOT_ROLLUP, inputs: input }, signal, epochNumber);
   }
 
   /**
@@ -355,14 +349,12 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getPublicKernelProof(
     kernelRequest: PublicKernelNonTailRequest,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<PublicKernelCircuitPublicInputs>> {
     return this.enqueue(
-      {
-        type: ProvingRequestType.PUBLIC_KERNEL_NON_TAIL,
-        kernelType: kernelRequest.type,
-        inputs: kernelRequest.inputs,
-      },
+      { type: ProvingRequestType.PUBLIC_KERNEL_NON_TAIL, kernelType: kernelRequest.type, inputs: kernelRequest.inputs },
       signal,
+      epochNumber,
     );
   }
 
@@ -373,28 +365,20 @@ export class MemoryProvingQueue implements ServerCircuitProver, ProvingJobSource
   getPublicTailProof(
     kernelRequest: PublicKernelTailRequest,
     signal?: AbortSignal,
+    epochNumber?: number,
   ): Promise<PublicInputsAndRecursiveProof<KernelCircuitPublicInputs>> {
     return this.enqueue(
-      {
-        type: ProvingRequestType.PUBLIC_KERNEL_TAIL,
-        kernelType: kernelRequest.type,
-        inputs: kernelRequest.inputs,
-      },
+      { type: ProvingRequestType.PUBLIC_KERNEL_TAIL, kernelType: kernelRequest.type, inputs: kernelRequest.inputs },
       signal,
+      epochNumber,
     );
   }
 
   /**
    * Creates an AVM proof.
    */
-  getAvmProof(inputs: AvmCircuitInputs, signal?: AbortSignal | undefined): Promise<ProofAndVerificationKey> {
-    return this.enqueue(
-      {
-        type: ProvingRequestType.PUBLIC_VM,
-        inputs,
-      },
-      signal,
-    );
+  getAvmProof(inputs: AvmCircuitInputs, signal?: AbortSignal, epochNumber?: number): Promise<ProofAndVerificationKey> {
+    return this.enqueue({ type: ProvingRequestType.PUBLIC_VM, inputs }, signal, epochNumber);
   }
 
   /**

package/src/prover-agent/prover-agent.ts

@@ -12,11 +12,20 @@ import { elapsed } from '@aztec/foundation/timer';
 
 import { ProvingError } from './proving-error.js';
 
+const PRINT_THRESHOLD_NS = 6e10; // 60 seconds
+
 /**
  * A helper class that encapsulates a circuit prover and connects it to a job source.
  */
 export class ProverAgent {
-  private inFlightPromises = new Map<string, Promise<any>>();
+  private inFlightPromises = new Map<
+    string,
+    {
+      id: string;
+      type: ProvingRequestType;
+      promise: Promise<any>;
+    }
+  >();
   private runningPromise?: RunningPromise;
 
   constructor(
@@ -49,11 +58,26 @@ export class ProverAgent {
       throw new Error('Agent is already running');
     }
 
+    let lastPrint = process.hrtime.bigint();
+
     this.runningPromise = new RunningPromise(async () => {
       for (const jobId of this.inFlightPromises.keys()) {
         await jobSource.heartbeat(jobId);
       }
 
+      const now = process.hrtime.bigint();
+
+      if (now - lastPrint >= PRINT_THRESHOLD_NS) {
+        // only log if we're actually doing work
+        if (this.inFlightPromises.size > 0) {
+          const jobs = Array.from(this.inFlightPromises.values())
+            .map(job => `id=${job.id},type=${ProvingRequestType[job.type]}`)
+            .join(' ');
+          this.log.info(`Agent is running with ${this.inFlightPromises.size} in-flight jobs: ${jobs}`);
+        }
+        lastPrint = now;
+      }
+
       while (this.inFlightPromises.size < this.maxConcurrency) {
         try {
           const job = await jobSource.getProvingJob();
@@ -64,14 +88,18 @@ export class ProverAgent {
 
           try {
             const promise = this.work(jobSource, job).finally(() => this.inFlightPromises.delete(job.id));
-            this.inFlightPromises.set(job.id, promise);
+            this.inFlightPromises.set(job.id, {
+              id: job.id,
+              type: job.request.type,
+              promise,
+            });
           } catch (err) {
             this.log.warn(
               `Error processing job! type=${ProvingRequestType[job.request.type]}: ${err}. ${(err as Error).stack}`,
             );
           }
         } catch (err) {
-          // no-op
+          this.log.error(`Error fetching job`, err);
         }
       }
     }, this.pollIntervalMs);
@@ -96,12 +124,12 @@ export class ProverAgent {
       this.log.debug(`Picked up proving job id=${job.id} type=${ProvingRequestType[job.request.type]}`);
       const [time, result] = await elapsed(this.getProof(job.request));
       if (this.isRunning()) {
-        this.log.debug(
+        this.log.verbose(
           `Processed proving job id=${job.id} type=${ProvingRequestType[job.request.type]} duration=${time}ms`,
         );
         await jobSource.resolveProvingJob(job.id, result);
       } else {
-        this.log.debug(
+        this.log.verbose(
           `Dropping proving job id=${job.id} type=${
             ProvingRequestType[job.request.type]
           } duration=${time}ms: agent stopped`,
@@ -113,10 +141,11 @@ export class ProverAgent {
           `Error processing proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: ${
             (err as any).stack || err
           }`,
+          err,
         );
         await jobSource.rejectProvingJob(job.id, new ProvingError((err as any)?.message ?? String(err)));
       } else {
-        this.log.debug(
+        this.log.verbose(
           `Dropping proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: agent stopped: ${
             (err as any).stack || err
           }`,

package/src/prover-agent/queue_metrics.ts

@@ -0,0 +1,29 @@
+import { ProvingRequestType } from '@aztec/circuit-types';
+import { Attributes, type Gauge, type Histogram, Metrics, type TelemetryClient } from '@aztec/telemetry-client';
+
+export class ProvingQueueMetrics {
+  private jobSize: Histogram;
+  private queueSize: Gauge;
+
+  constructor(client: TelemetryClient, name = 'ProvingQueueMetrics') {
+    const meter = client.getMeter(name);
+    this.jobSize = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_SIZE, {
+      description: 'Size of proving job',
+      unit: 'by',
+    });
+
+    this.queueSize = meter.createGauge(Metrics.PROVING_QUEUE_SIZE, {
+      description: 'Size of proving queue',
+    });
+  }
+
+  recordNewJob(type: ProvingRequestType, size: number) {
+    this.jobSize.record(size, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[type],
+    });
+  }
+
+  recordQueueSize(size: number) {
+    this.queueSize.record(size);
+  }
+}

package/src/prover-agent/rpc.ts

@@ -9,6 +9,7 @@ import {
   KernelCircuitPublicInputs,
   MergeRollupInputs,
   ParityPublicInputs,
+  PrivateKernelEmptyInputData,
   Proof,
   PublicKernelCircuitPrivateInputs,
   PublicKernelCircuitPublicInputs,
@@ -41,6 +42,7 @@ export function createProvingJobSourceServer(queue: ProvingJobSource): JsonRpcSe
       ParityPublicInputs,
       Proof,
       ProvingError,
+      PrivateKernelEmptyInputData,
       PublicKernelCircuitPrivateInputs,
       PublicKernelCircuitPublicInputs,
       PublicKernelTailCircuitPrivateInputs,
@@ -75,6 +77,7 @@ export function createProvingJobSourceClient(
       ParityPublicInputs,
       Proof,
       ProvingError,
+      PrivateKernelEmptyInputData,
       PublicKernelCircuitPrivateInputs,
       PublicKernelCircuitPublicInputs,
       PublicKernelTailCircuitPrivateInputs,

package/src/tx-prover/factory.ts

@@ -1,16 +1,9 @@
-import { type L2BlockSource } from '@aztec/circuit-types';
 import { type TelemetryClient } from '@aztec/telemetry-client';
 import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
-import { type WorldStateSynchronizer } from '@aztec/world-state';
 
 import { type ProverClientConfig } from '../config.js';
 import { TxProver } from './tx-prover.js';
 
-export function createProverClient(
-  config: ProverClientConfig,
-  worldStateSynchronizer: WorldStateSynchronizer,
-  blockSource: L2BlockSource,
-  telemetry: TelemetryClient = new NoopTelemetryClient(),
-) {
-  return config.disableProver ? undefined : TxProver.new(config, worldStateSynchronizer, blockSource, telemetry);
+export function createProverClient(config: ProverClientConfig, telemetry: TelemetryClient = new NoopTelemetryClient()) {
+  return TxProver.new(config, telemetry);
 }

package/src/tx-prover/tx-prover.ts

@@ -1,16 +1,14 @@
 import { BBNativeRollupProver, TestCircuitProver } from '@aztec/bb-prover';
-import { type L2BlockSource, type ProcessedTx } from '@aztec/circuit-types';
 import {
-  type BlockResult,
+  type BlockProver,
   type ProverClient,
   type ProvingJobSource,
-  type ProvingTicket,
   type ServerCircuitProver,
 } from '@aztec/circuit-types/interfaces';
-import { type Fr, type GlobalVariables } from '@aztec/circuits.js';
+import { Fr } from '@aztec/circuits.js';
 import { NativeACVMSimulator } from '@aztec/simulator';
 import { type TelemetryClient } from '@aztec/telemetry-client';
-import { type WorldStateSynchronizer } from '@aztec/world-state';
+import { type MerkleTreeOperations } from '@aztec/world-state';
 
 import { type ProverClientConfig } from '../config.js';
 import { ProvingOrchestrator } from '../orchestrator/orchestrator.js';
@@ -18,21 +16,29 @@ import { MemoryProvingQueue } from '../prover-agent/memory-proving-queue.js';
 import { ProverAgent } from '../prover-agent/prover-agent.js';
 
 /**
- * A prover accepting individual transaction requests
+ * A prover factory.
+ * TODO(palla/prover-node): Rename this class
  */
 export class TxProver implements ProverClient {
-  private orchestrator: ProvingOrchestrator;
   private queue: MemoryProvingQueue;
   private running = false;
 
   private constructor(
     private config: ProverClientConfig,
-    private worldStateSynchronizer: WorldStateSynchronizer,
     private telemetry: TelemetryClient,
     private agent?: ProverAgent,
   ) {
-    this.queue = new MemoryProvingQueue(config.proverJobTimeoutMs, config.proverJobPollIntervalMs);
-    this.orchestrator = new ProvingOrchestrator(worldStateSynchronizer.getLatest(), this.queue, telemetry);
+    // TODO(palla/prover-node): Cache the paddingTx here, and not in each proving orchestrator,
+    // so it can be reused across multiple ones and not recomputed every time.
+    this.queue = new MemoryProvingQueue(telemetry, config.proverJobTimeoutMs, config.proverJobPollIntervalMs);
+  }
+
+  public createBlockProver(db: MerkleTreeOperations): BlockProver {
+    return new ProvingOrchestrator(db, this.queue, this.telemetry, this.config.proverId);
+  }
+
+  public getProverId(): Fr {
+    return this.config.proverId ?? Fr.ZERO;
   }
 
   async updateProverConfig(config: Partial<ProverClientConfig>): Promise<void> {
@@ -48,7 +54,7 @@ export class TxProver implements ProverClient {
     }
 
     if (!this.config.realProofs && newConfig.realProofs) {
-      this.orchestrator.reset();
+      // TODO(palla/prover-node): Reset padding tx here once we cache it at this class
     }
 
     this.config = newConfig;
@@ -76,6 +82,8 @@ export class TxProver implements ProverClient {
       return;
     }
     this.running = false;
+
+    // TODO(palla/prover-node): Keep a reference to all proving orchestrators that are alive and stop them?
     await this.agent?.stop();
     await this.queue.stop();
   }
@@ -87,12 +95,7 @@ export class TxProver implements ProverClient {
    * @param worldStateSynchronizer - An instance of the world state
    * @returns An instance of the prover, constructed and started.
    */
-  public static async new(
-    config: ProverClientConfig,
-    worldStateSynchronizer: WorldStateSynchronizer,
-    blockSource: L2BlockSource,
-    telemetry: TelemetryClient,
-  ) {
+  public static async new(config: ProverClientConfig, telemetry: TelemetryClient) {
     const agent = config.proverAgentEnabled
       ? new ProverAgent(
           await TxProver.buildCircuitProver(config, telemetry),
@@ -101,7 +104,7 @@ export class TxProver implements ProverClient {
         )
       : undefined;
 
-    const prover = new TxProver(config, worldStateSynchronizer, telemetry, agent);
+    const prover = new TxProver(config, telemetry, agent);
     await prover.start();
     return prover;
   }
@@ -121,52 +124,6 @@ export class TxProver implements ProverClient {
     return new TestCircuitProver(telemetry, simulationProvider);
   }
 
-  /**
-   * Cancels any block that is currently being built and prepares for a new one to be built
-   * @param numTxs - The complete size of the block, must be a power of 2
-   * @param globalVariables - The global variables for this block
-   * @param l1ToL2Messages - The set of L1 to L2 messages to be included in this block
-   */
-  public async startNewBlock(
-    numTxs: number,
-    globalVariables: GlobalVariables,
-    newL1ToL2Messages: Fr[],
-  ): Promise<ProvingTicket> {
-    const previousBlockNumber = globalVariables.blockNumber.toNumber() - 1;
-    await this.worldStateSynchronizer.syncImmediate(previousBlockNumber);
-    return this.orchestrator.startNewBlock(numTxs, globalVariables, newL1ToL2Messages);
-  }
-
-  /**
-   * Add a processed transaction to the current block
-   * @param tx - The transaction to be added
-   */
-  public addNewTx(tx: ProcessedTx): Promise<void> {
-    return this.orchestrator.addNewTx(tx);
-  }
-
-  /**
-   * Cancels the block currently being proven. Proofs already bring built may continue but further proofs should not be started.
-   */
-  public cancelBlock(): void {
-    this.orchestrator.cancelBlock();
-  }
-
-  /**
-   * Performs the final archive tree insertion for this block and returns the L2Block and Proof instances
-   */
-  public finaliseBlock(): Promise<BlockResult> {
-    return this.orchestrator.finaliseBlock();
-  }
-
-  /**
-   * Mark the block as having all the transactions it is going to contain.
-   * Will pad the block to it's complete size with empty transactions and prove all the way to the root rollup.
-   */
-  public setBlockCompleted(): Promise<void> {
-    return this.orchestrator.setBlockCompleted();
-  }
-
   public getProvingJobSource(): ProvingJobSource {
     return this.queue;
   }