@aztec/prover-client 0.42.0

package/package.json

@@ -0,0 +1,87 @@
+{
+  "name": "@aztec/prover-client",
+  "version": "0.42.0",
+  "type": "module",
+  "exports": {
+    ".": "./dest/index.js",
+    "./prover-agent": "./dest/prover-agent/index.js"
+  },
+  "typedocOptions": {
+    "entryPoints": [
+      "./src/index.ts"
+    ],
+    "name": "Prover Client",
+    "tsconfig": "./tsconfig.json"
+  },
+  "inherits": [
+    "../package.common.json",
+    "./package.local.json"
+  ],
+  "scripts": {
+    "build": "yarn clean && tsc -b",
+    "build:dev": "tsc -b --watch",
+    "clean": "rm -rf ./dest .tsbuildinfo",
+    "formatting": "run -T prettier --check ./src && run -T eslint ./src",
+    "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
+    "bb": "node --no-warnings ./dest/bb/index.js",
+    "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=1500000 --forceExit"
+  },
+  "jest": {
+    "moduleNameMapper": {
+      "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+    },
+    "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
+    "rootDir": "./src",
+    "transform": {
+      "^.+\\.tsx?$": [
+        "@swc/jest"
+      ]
+    },
+    "extensionsToTreatAsEsm": [
+      ".ts"
+    ],
+    "reporters": [
+      [
+        "default",
+        {
+          "summaryThreshold": 9999
+        }
+      ]
+    ]
+  },
+  "dependencies": {
+    "@aztec/bb-prover": "0.42.0",
+    "@aztec/circuit-types": "0.42.0",
+    "@aztec/circuits.js": "0.42.0",
+    "@aztec/foundation": "0.42.0",
+    "@aztec/kv-store": "0.42.0",
+    "@aztec/noir-protocol-circuits-types": "0.42.0",
+    "@aztec/simulator": "0.42.0",
+    "@aztec/world-state": "0.42.0",
+    "@noir-lang/types": "portal:../../noir/packages/types",
+    "commander": "^9.0.0",
+    "lodash.chunk": "^4.2.0",
+    "source-map-support": "^0.5.21",
+    "tslib": "^2.4.0"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.5.0",
+    "@types/jest": "^29.5.0",
+    "@types/memdown": "^3.0.0",
+    "@types/node": "^18.7.23",
+    "@types/source-map-support": "^0.5.10",
+    "jest": "^29.5.0",
+    "jest-mock-extended": "^3.0.3",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4"
+  },
+  "files": [
+    "dest",
+    "src",
+    "!*.test.*"
+  ],
+  "types": "./dest/index.d.ts",
+  "engines": {
+    "node": ">=18"
+  }
+}

package/src/config.ts

@@ -0,0 +1,59 @@
+import { type ProverConfig } from '@aztec/circuit-types';
+
+import { tmpdir } from 'os';
+
+/**
+ * The prover configuration.
+ */
+export type ProverClientConfig = ProverConfig & {
+  /** The working directory to use for simulation/proving */
+  acvmWorkingDirectory: string;
+  /** The path to the ACVM binary */
+  acvmBinaryPath: string;
+  /** The working directory to for proving */
+  bbWorkingDirectory: string;
+  /** The path to the bb binary */
+  bbBinaryPath: string;
+};
+
+/**
+ * Returns the prover configuration from the environment variables.
+ * Note: If an environment variable is not set, the default value is used.
+ * @returns The prover configuration.
+ */
+export function getProverEnvVars(): ProverClientConfig {
+  const {
+    AZTEC_NODE_URL,
+    ACVM_WORKING_DIRECTORY = tmpdir(),
+    ACVM_BINARY_PATH = '',
+    BB_WORKING_DIRECTORY = tmpdir(),
+    BB_BINARY_PATH = '',
+    /** @deprecated */
+    PROVER_AGENTS = '1',
+    PROVER_AGENT_ENABLED = '1',
+    PROVER_AGENT_CONCURRENCY = PROVER_AGENTS,
+    PROVER_AGENT_POLL_INTERVAL_MS = '100',
+    PROVER_REAL_PROOFS = '',
+  } = process.env;
+
+  const realProofs = ['1', 'true'].includes(PROVER_REAL_PROOFS);
+  const proverAgentEnabled = ['1', 'true'].includes(PROVER_AGENT_ENABLED);
+  const parsedProverConcurrency = parseInt(PROVER_AGENT_CONCURRENCY, 10);
+  const proverAgentConcurrency = Number.isSafeInteger(parsedProverConcurrency) ? parsedProverConcurrency : 1;
+  const parsedProverAgentPollInterval = parseInt(PROVER_AGENT_POLL_INTERVAL_MS, 10);
+  const proverAgentPollInterval = Number.isSafeInteger(parsedProverAgentPollInterval)
+    ? parsedProverAgentPollInterval
+    : 100;
+
+  return {
+    acvmWorkingDirectory: ACVM_WORKING_DIRECTORY,
+    acvmBinaryPath: ACVM_BINARY_PATH,
+    bbBinaryPath: BB_BINARY_PATH,
+    bbWorkingDirectory: BB_WORKING_DIRECTORY,
+    realProofs,
+    proverAgentEnabled,
+    proverAgentPollInterval,
+    proverAgentConcurrency,
+    nodeUrl: AZTEC_NODE_URL,
+  };
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export { ProverClient } from '@aztec/circuit-types';
+
+export * from './tx-prover/tx-prover.js';
+export * from './config.js';

package/src/mocks/fixtures.ts

@@ -0,0 +1,182 @@
+import {
+  MerkleTreeId,
+  type ProcessedTx,
+  makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots,
+  makeProcessedTx,
+  mockTx,
+} from '@aztec/circuit-types';
+import {
+  AztecAddress,
+  EthAddress,
+  Fr,
+  GasFees,
+  GlobalVariables,
+  KernelCircuitPublicInputs,
+  MAX_NEW_L2_TO_L1_MSGS_PER_TX,
+  MAX_NEW_NOTE_HASHES_PER_TX,
+  MAX_NEW_NULLIFIERS_PER_TX,
+  MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
+  NULLIFIER_TREE_HEIGHT,
+  PUBLIC_DATA_SUBTREE_HEIGHT,
+  PublicDataTreeLeaf,
+  PublicDataUpdateRequest,
+} from '@aztec/circuits.js';
+import { fr, makeProof } from '@aztec/circuits.js/testing';
+import { makeTuple } from '@aztec/foundation/array';
+import { padArrayEnd } from '@aztec/foundation/collection';
+import { randomBytes } from '@aztec/foundation/crypto';
+import { type DebugLogger } from '@aztec/foundation/log';
+import { fileURLToPath } from '@aztec/foundation/url';
+import { NativeACVMSimulator, type SimulationProvider, WASMSimulator } from '@aztec/simulator';
+import { type MerkleTreeOperations } from '@aztec/world-state';
+
+import * as fs from 'fs/promises';
+import path from 'path';
+
+const {
+  BB_RELEASE_DIR = 'cpp/build/bin',
+  TEMP_DIR = '/tmp',
+  BB_BINARY_PATH = '',
+  BB_WORKING_DIRECTORY = '',
+  NOIR_RELEASE_DIR = 'noir-repo/target/release',
+  ACVM_BINARY_PATH = '',
+  ACVM_WORKING_DIRECTORY = '',
+} = process.env;
+
+// Determines if we have access to the bb binary and a tmp folder for temp files
+export const getEnvironmentConfig = async (logger: DebugLogger) => {
+  try {
+    const expectedBBPath = BB_BINARY_PATH
+      ? BB_BINARY_PATH
+      : `${path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../barretenberg/', BB_RELEASE_DIR)}/bb`;
+    await fs.access(expectedBBPath, fs.constants.R_OK);
+    const tempWorkingDirectory = `${TEMP_DIR}/${randomBytes(4).toString('hex')}`;
+    const bbWorkingDirectory = BB_WORKING_DIRECTORY ? BB_WORKING_DIRECTORY : `${tempWorkingDirectory}/bb`;
+    await fs.mkdir(bbWorkingDirectory, { recursive: true });
+    logger.verbose(`Using native BB binary at ${expectedBBPath} with working directory ${bbWorkingDirectory}`);
+
+    const expectedAcvmPath = ACVM_BINARY_PATH
+      ? ACVM_BINARY_PATH
+      : `${path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../noir/', NOIR_RELEASE_DIR)}/acvm`;
+    await fs.access(expectedAcvmPath, fs.constants.R_OK);
+    const acvmWorkingDirectory = ACVM_WORKING_DIRECTORY ? ACVM_WORKING_DIRECTORY : `${tempWorkingDirectory}/acvm`;
+    await fs.mkdir(acvmWorkingDirectory, { recursive: true });
+    logger.verbose(`Using native ACVM binary at ${expectedAcvmPath} with working directory ${acvmWorkingDirectory}`);
+    return {
+      acvmWorkingDirectory,
+      bbWorkingDirectory,
+      expectedAcvmPath,
+      expectedBBPath,
+      directoryToCleanup: ACVM_WORKING_DIRECTORY && BB_WORKING_DIRECTORY ? undefined : tempWorkingDirectory,
+    };
+  } catch (err) {
+    logger.verbose(`Native BB not available, error: ${err}`);
+    return undefined;
+  }
+};
+
+export async function getSimulationProvider(
+  config: { acvmWorkingDirectory: string | undefined; acvmBinaryPath: string | undefined },
+  logger?: DebugLogger,
+): Promise<SimulationProvider> {
+  if (config.acvmBinaryPath && config.acvmWorkingDirectory) {
+    try {
+      await fs.access(config.acvmBinaryPath, fs.constants.R_OK);
+      await fs.mkdir(config.acvmWorkingDirectory, { recursive: true });
+      logger?.info(
+        `Using native ACVM at ${config.acvmBinaryPath} and working directory ${config.acvmWorkingDirectory}`,
+      );
+      return new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath);
+    } catch {
+      logger?.warn(`Failed to access ACVM at ${config.acvmBinaryPath}, falling back to WASM`);
+    }
+  }
+  logger?.info('Using WASM ACVM simulation');
+  return new WASMSimulator();
+}
+
+export const makeBloatedProcessedTx = async (builderDb: MerkleTreeOperations, seed = 0x1) => {
+  seed *= MAX_NEW_NULLIFIERS_PER_TX; // Ensure no clashing given incremental seeds
+  const tx = mockTx(seed);
+  const kernelOutput = KernelCircuitPublicInputs.empty();
+  kernelOutput.constants.historicalHeader = await builderDb.buildInitialHeader();
+  kernelOutput.end.publicDataUpdateRequests = makeTuple(
+    MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
+    i => new PublicDataUpdateRequest(fr(i), fr(i + 10), i + 20),
+    seed + 0x500,
+  );
+  kernelOutput.end.publicDataUpdateRequests = makeTuple(
+    MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
+    i => new PublicDataUpdateRequest(fr(i), fr(i + 10), i + 20),
+    seed + 0x600,
+  );
+
+  const processedTx = makeProcessedTx(tx, kernelOutput, makeProof(), []);
+
+  processedTx.data.end.newNoteHashes = makeTuple(MAX_NEW_NOTE_HASHES_PER_TX, fr, seed + 0x100);
+  processedTx.data.end.newNullifiers = makeTuple(MAX_NEW_NULLIFIERS_PER_TX, fr, seed + 0x100000);
+
+  processedTx.data.end.newNullifiers[tx.data.forPublic!.end.newNullifiers.length - 1] = Fr.zero();
+
+  processedTx.data.end.newL2ToL1Msgs = makeTuple(MAX_NEW_L2_TO_L1_MSGS_PER_TX, fr, seed + 0x300);
+  processedTx.data.end.noteEncryptedLogsHash = Fr.fromBuffer(processedTx.noteEncryptedLogs.hash());
+  processedTx.data.end.encryptedLogsHash = Fr.fromBuffer(processedTx.encryptedLogs.hash());
+  processedTx.data.end.unencryptedLogsHash = Fr.fromBuffer(processedTx.unencryptedLogs.hash());
+
+  return processedTx;
+};
+
+export const makeEmptyProcessedTx = async (builderDb: MerkleTreeOperations, chainId: Fr, version: Fr) => {
+  const header = await builderDb.buildInitialHeader();
+  return makeEmptyProcessedTxFromHistoricalTreeRoots(header, chainId, version);
+};
+
+// Updates the expectedDb trees based on the new note hashes, contracts, and nullifiers from these txs
+export const updateExpectedTreesFromTxs = async (db: MerkleTreeOperations, txs: ProcessedTx[]) => {
+  await db.appendLeaves(
+    MerkleTreeId.NOTE_HASH_TREE,
+    txs.flatMap(tx =>
+      padArrayEnd(
+        tx.data.end.newNoteHashes.filter(x => !x.isZero()),
+        Fr.zero(),
+        MAX_NEW_NOTE_HASHES_PER_TX,
+      ),
+    ),
+  );
+  await db.batchInsert(
+    MerkleTreeId.NULLIFIER_TREE,
+    txs.flatMap(tx =>
+      padArrayEnd(
+        tx.data.end.newNullifiers.filter(x => !x.isZero()),
+        Fr.zero(),
+        MAX_NEW_NULLIFIERS_PER_TX,
+      ).map(x => x.toBuffer()),
+    ),
+    NULLIFIER_TREE_HEIGHT,
+  );
+  for (const tx of txs) {
+    await db.batchInsert(
+      MerkleTreeId.PUBLIC_DATA_TREE,
+      tx.data.end.publicDataUpdateRequests.map(write => {
+        return new PublicDataTreeLeaf(write.leafSlot, write.newValue).toBuffer();
+      }),
+      PUBLIC_DATA_SUBTREE_HEIGHT,
+    );
+  }
+};
+
+export const makeGlobals = (blockNumber: number) => {
+  return new GlobalVariables(
+    Fr.ZERO,
+    Fr.ZERO,
+    new Fr(blockNumber),
+    Fr.ZERO,
+    EthAddress.ZERO,
+    AztecAddress.ZERO,
+    GasFees.empty(),
+  );
+};
+
+export const makeEmptyProcessedTestTx = (builderDb: MerkleTreeOperations): Promise<ProcessedTx> => {
+  return makeEmptyProcessedTx(builderDb, Fr.ZERO, Fr.ZERO);
+};

package/src/mocks/test_context.ts

@@ -0,0 +1,217 @@
+import { type BBProverConfig } from '@aztec/bb-prover';
+import {
+  type BlockProver,
+  type BlockResult,
+  type ProcessedTx,
+  type ProvingTicket,
+  type ServerCircuitProver,
+  type Tx,
+  type TxValidator,
+} from '@aztec/circuit-types';
+import {
+  type Gas,
+  GlobalVariables,
+  Header,
+  type Nullifier,
+  type TxContext,
+  getMockVerificationKeys,
+} from '@aztec/circuits.js';
+import { type Fr } from '@aztec/foundation/fields';
+import { type DebugLogger } from '@aztec/foundation/log';
+import { openTmpStore } from '@aztec/kv-store/utils';
+import {
+  type ContractsDataSourcePublicDB,
+  type PublicExecution,
+  type PublicExecutionResult,
+  PublicExecutionResultBuilder,
+  type PublicExecutor,
+  PublicProcessor,
+  RealPublicKernelCircuitSimulator,
+  type SimulationProvider,
+  WASMSimulator,
+  type WorldStatePublicDB,
+} from '@aztec/simulator';
+import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state';
+
+import * as fs from 'fs/promises';
+import { type MockProxy, mock } from 'jest-mock-extended';
+
+import { TestCircuitProver } from '../../../bb-prover/src/test/test_circuit_prover.js';
+import { ProvingOrchestrator } from '../orchestrator/orchestrator.js';
+import { MemoryProvingQueue } from '../prover-agent/memory-proving-queue.js';
+import { ProverAgent } from '../prover-agent/prover-agent.js';
+import { getEnvironmentConfig, getSimulationProvider, makeGlobals } from './fixtures.js';
+
+class DummyProverClient implements BlockProver {
+  constructor(private orchestrator: ProvingOrchestrator, private verificationKeys = getMockVerificationKeys()) {}
+  startNewBlock(numTxs: number, globalVariables: GlobalVariables, l1ToL2Messages: Fr[]): Promise<ProvingTicket> {
+    return this.orchestrator.startNewBlock(numTxs, globalVariables, l1ToL2Messages, this.verificationKeys);
+  }
+  addNewTx(tx: ProcessedTx): Promise<void> {
+    return this.orchestrator.addNewTx(tx);
+  }
+  cancelBlock(): void {
+    return this.orchestrator.cancelBlock();
+  }
+  finaliseBlock(): Promise<BlockResult> {
+    return this.orchestrator.finaliseBlock();
+  }
+  setBlockCompleted(): Promise<void> {
+    return this.orchestrator.setBlockCompleted();
+  }
+}
+
+export class TestContext {
+  public blockProver: BlockProver;
+  constructor(
+    public publicExecutor: MockProxy<PublicExecutor>,
+    public publicContractsDB: MockProxy<ContractsDataSourcePublicDB>,
+    public publicWorldStateDB: MockProxy<WorldStatePublicDB>,
+    public publicProcessor: PublicProcessor,
+    public simulationProvider: SimulationProvider,
+    public globalVariables: GlobalVariables,
+    public actualDb: MerkleTreeOperations,
+    public prover: ServerCircuitProver,
+    public proverAgent: ProverAgent,
+    public orchestrator: ProvingOrchestrator,
+    public blockNumber: number,
+    public directoriesToCleanup: string[],
+    public logger: DebugLogger,
+  ) {
+    this.blockProver = new DummyProverClient(this.orchestrator);
+  }
+
+  static async new(
+    logger: DebugLogger,
+    proverCount = 4,
+    createProver: (bbConfig: BBProverConfig) => Promise<ServerCircuitProver> = _ =>
+      Promise.resolve(new TestCircuitProver(new WASMSimulator())),
+    blockNumber = 3,
+  ) {
+    const globalVariables = makeGlobals(blockNumber);
+
+    const publicExecutor = mock<PublicExecutor>();
+    const publicContractsDB = mock<ContractsDataSourcePublicDB>();
+    const publicWorldStateDB = mock<WorldStatePublicDB>();
+    const publicKernel = new RealPublicKernelCircuitSimulator(new WASMSimulator());
+    const actualDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest());
+    const processor = new PublicProcessor(
+      actualDb,
+      publicExecutor,
+      publicKernel,
+      GlobalVariables.empty(),
+      Header.empty(),
+      publicContractsDB,
+      publicWorldStateDB,
+    );
+
+    let localProver: ServerCircuitProver;
+    const config = await getEnvironmentConfig(logger);
+    const simulationProvider = await getSimulationProvider({
+      acvmWorkingDirectory: config?.acvmWorkingDirectory,
+      acvmBinaryPath: config?.expectedAcvmPath,
+    });
+    if (!config) {
+      localProver = new TestCircuitProver(simulationProvider);
+    } else {
+      const bbConfig: BBProverConfig = {
+        acvmBinaryPath: config.expectedAcvmPath,
+        acvmWorkingDirectory: config.acvmWorkingDirectory,
+        bbBinaryPath: config.expectedBBPath,
+        bbWorkingDirectory: config.bbWorkingDirectory,
+      };
+      localProver = await createProver(bbConfig);
+    }
+
+    const queue = new MemoryProvingQueue();
+    const orchestrator = new ProvingOrchestrator(actualDb, queue);
+    const agent = new ProverAgent(localProver, proverCount);
+
+    agent.start(queue);
+
+    return new this(
+      publicExecutor,
+      publicContractsDB,
+      publicWorldStateDB,
+      processor,
+      simulationProvider,
+      globalVariables,
+      actualDb,
+      localProver,
+      agent,
+      orchestrator,
+      blockNumber,
+      [config?.directoryToCleanup ?? ''],
+      logger,
+    );
+  }
+
+  async cleanup() {
+    await this.proverAgent.stop();
+    for (const dir of this.directoriesToCleanup.filter(x => x !== '')) {
+      await fs.rm(dir, { recursive: true, force: true });
+    }
+  }
+
+  public async processPublicFunctions(
+    txs: Tx[],
+    maxTransactions: number,
+    blockProver?: BlockProver,
+    txValidator?: TxValidator<ProcessedTx>,
+  ) {
+    const defaultExecutorImplementation = (
+      execution: PublicExecution,
+      _globalVariables: GlobalVariables,
+      availableGas: Gas,
+      _txContext: TxContext,
+      _pendingNullifiers: Nullifier[],
+      transactionFee?: Fr,
+      _sideEffectCounter?: number,
+    ) => {
+      for (const tx of txs) {
+        const allCalls = tx.publicTeardownFunctionCall.isEmpty()
+          ? tx.enqueuedPublicFunctionCalls
+          : [...tx.enqueuedPublicFunctionCalls, tx.publicTeardownFunctionCall];
+        for (const request of allCalls) {
+          if (execution.contractAddress.equals(request.contractAddress)) {
+            const result = PublicExecutionResultBuilder.fromPublicCallRequest({ request }).build({
+              startGasLeft: availableGas,
+              endGasLeft: availableGas,
+              transactionFee,
+            });
+            return Promise.resolve(result);
+          }
+        }
+      }
+      throw new Error(`Unexpected execution request: ${execution}`);
+    };
+    return await this.processPublicFunctionsWithMockExecutorImplementation(
+      txs,
+      maxTransactions,
+      blockProver,
+      txValidator,
+      defaultExecutorImplementation,
+    );
+  }
+
+  public async processPublicFunctionsWithMockExecutorImplementation(
+    txs: Tx[],
+    maxTransactions: number,
+    blockProver?: BlockProver,
+    txValidator?: TxValidator<ProcessedTx>,
+    executorMock?: (
+      execution: PublicExecution,
+      globalVariables: GlobalVariables,
+      availableGas: Gas,
+      txContext: TxContext,
+      pendingNullifiers: Nullifier[],
+      transactionFee?: Fr,
+      sideEffectCounter?: number,
+    ) => Promise<PublicExecutionResult>,
+  ) {
+    if (executorMock) {
+      this.publicExecutor.simulate.mockImplementation(executorMock);
+    }
+    return await this.publicProcessor.process(txs, maxTransactions, blockProver, txValidator);
+  }
+}