@aztec-labs/bb-prover 6.0.0-rc.1 → 6.0.0-rc.2

package/src/verifier/bb_verifier.ts

@@ -1,3 +1,4 @@
+import { isRetryableError } from '@aztec-labs/foundation/error';
 import { type Logger, createLogger } from '@aztec-labs/foundation/log';
 import { Timer } from '@aztec-labs/foundation/timer';
 import { ProtocolCircuitVks } from '@aztec-labs/noir-protocol-circuits-types/server/vks';
@@ -14,24 +15,43 @@ import { Tx } from '@aztec-labs/stdlib/tx';
 import type { VerificationKeyData } from '@aztec-labs/stdlib/vks';
 import { promises as fs } from 'fs';
 
-import { BBJsFactory } from '../bb/bb_js_backend.js';
+import { type BBJsApi, BBJsFactory } from '../bb/bb_js_backend.js';
 import type { BBConfig } from '../config.js';
 import { getUltraHonkFlavorForCircuit } from '../honk.js';
 
+/** Thrown when no live bb process could check a proof, so the proof was neither accepted nor rejected. */
+export class ProofVerifierUnavailableError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = 'ProofVerifierUnavailableError';
+  }
+}
+
 export class BBCircuitVerifier implements ClientProtocolCircuitVerifier {
+  /** bb instances a Chonk verification tries, while each one's bb dies under it, before the verifier is unavailable. */
+  private static readonly MAX_CHONK_VERIFY_ATTEMPTS = 2;
+
   private bbJsFactory: BBJsFactory;
 
-  private constructor(
+  protected constructor(
     private config: BBConfig,
     private logger: Logger,
+    bbJsFactory?: BBJsFactory,
   ) {
     // BB_NUM_IVC_VERIFIERS bounds the number of long-lived bb processes the pool keeps alive.
     // If 0, fall back to spawning a fresh bb per verification.
-    this.bbJsFactory = new BBJsFactory(config.bbBinaryPath, {
-      poolSize: config.numConcurrentIVCVerifiers > 0 ? config.numConcurrentIVCVerifiers : undefined,
-      logger,
-      debugDir: config.bbDebugOutputDir,
-    });
+    const poolSize = config.numConcurrentIVCVerifiers > 0 ? config.numConcurrentIVCVerifiers : undefined;
+    this.bbJsFactory =
+      bbJsFactory ??
+      new BBJsFactory(config.bbBinaryPath, {
+        poolSize,
+        // A pooled instance outlives the call that borrowed it, so it replaces a bb process that dies
+        // under it and the next borrower gets a working one. Each verification stands alone, so a
+        // replacement has nothing to carry over. A fresh-per-call instance has nothing to heal.
+        respawn: poolSize !== undefined,
+        logger,
+        debugDir: config.bbDebugOutputDir,
+      });
   }
 
   public stop(): Promise<void> {
@@ -85,9 +105,13 @@ export class BBCircuitVerifier implements ClientProtocolCircuitVerifier {
     } satisfies CircuitVerificationStats);
   }
 
-  /** Verify a Chonk (IVC) proof from a transaction via bb.js API. */
+  /**
+   * Verify a Chonk (IVC) proof from a transaction via bb.js API. Throws {@link ProofVerifierUnavailableError} when no
+   * live bb process could check the proof; any other failure returns `valid: false`.
+   */
   public async verifyProof(tx: Tx): Promise<IVCProofVerificationResult> {
     const proofType = 'Chonk';
+    const txHash = tx.getTxHash().toString();
     try {
       const totalTimer = new Timer();
 
@@ -98,8 +122,11 @@ export class BBCircuitVerifier implements ClientProtocolCircuitVerifier {
       const proofWithPubInputs = tx.chonkProof.attachPublicInputs(tx.data.publicInputs().toFields());
       const fieldsAsBuffers = proofWithPubInputs.fieldsWithPublicInputs.map(f => new Uint8Array(f.toBuffer()));
 
-      await using instance = await this.bbJsFactory.getInstance();
-      const { verified, durationMs } = await instance.verifyChonkProof(fieldsAsBuffers, verificationKey.keyAsBytes);
+      const { verified, durationMs } = await this.verifyChonkProofOnLiveInstance(
+        fieldsAsBuffers,
+        verificationKey.keyAsBytes,
+        txHash,
+      );
 
       if (!verified) {
         throw new Error(`Failed to verify ${proofType} proof for ${circuit}!`);
@@ -114,10 +141,59 @@ export class BBCircuitVerifier implements ClientProtocolCircuitVerifier {
 
       return { valid: true, durationMs, totalDurationMs: totalTimer.ms() };
     } catch (err) {
-      this.logger.warn(`Failed to verify ${proofType} proof for tx ${tx.getTxHash().toString()}: ${String(err)}`);
+      if (err instanceof ProofVerifierUnavailableError) {
+        throw err;
+      }
+      this.logger.warn(`Failed to verify ${proofType} proof`, { txHash, err });
       return { valid: false, durationMs: 0, totalDurationMs: 0 };
     }
   }
+
+  /**
+   * Runs a Chonk verification on a pooled bb instance. A call that failed for environmental reasons — its bb process
+   * died, or could not be started — is retried; any other failure is the verification's own verdict and is rethrown.
+   *
+   * The error says so itself, through the `retry` property bb.js sets. Asking the instance whether it is still alive
+   * would be a guess: the process can die between the answer and the next call.
+   */
+  private async verifyChonkProofOnLiveInstance(
+    fieldsWithPublicInputs: Uint8Array[],
+    verificationKey: Uint8Array,
+    txHash: string,
+  ): Promise<{ verified: boolean; durationMs: number }> {
+    for (let attempt = 1; ; attempt++) {
+      let borrowed: BBJsApi & AsyncDisposable;
+      try {
+        borrowed = await this.bbJsFactory.getInstance();
+      } catch (err) {
+        // A bb that could not be started is worth another go, within the same budget a death gets.
+        // Anything else — the factory destroyed under us — will not improve by asking again. Either
+        // way no proof was checked, so this is never the proof's fault.
+        if (isRetryableError(err) && attempt < BBCircuitVerifier.MAX_CHONK_VERIFY_ATTEMPTS) {
+          this.logger.warn('no bb instance available to verify a proof; retrying', { txHash, attempt });
+          continue;
+        }
+        throw new ProofVerifierUnavailableError('No bb instance available to verify the proof', { cause: err });
+      }
+
+      await using instance = borrowed;
+      try {
+        return await instance.verifyChonkProof(fieldsWithPublicInputs, verificationKey);
+      } catch (err) {
+        // Only an environmental failure is worth retrying; anything else is the verification's own
+        // verdict and belongs to the caller.
+        if (!isRetryableError(err)) {
+          throw err;
+        }
+        if (attempt >= BBCircuitVerifier.MAX_CHONK_VERIFY_ATTEMPTS) {
+          throw new ProofVerifierUnavailableError(`bb died while verifying the proof, on ${attempt} attempts`, {
+            cause: err,
+          });
+        }
+        this.logger.warn('bb died while verifying a proof; retrying', { txHash, attempt });
+      }
+    }
+  }
 }
 
 /** Split a buffer into 32-byte Uint8Array field elements. */

package/src/verifier/queued_chonk_verifier.ts

@@ -102,7 +102,12 @@ export class QueuedIVCVerifier implements ClientProtocolCircuitVerifier {
   }
 
   async stop(): Promise<void> {
-    await this.queue.end();
-    await this.verifier.stop();
+    // Stopped first so that verifications waiting for a bb instance fail rather than keep the queue from draining. Queued
+    // verifications that have not started yet fail too.
+    try {
+      await this.verifier.stop();
+    } finally {
+      await this.queue.end();
+    }
   }
 }