@aztec/prover-client 0.0.1-commit.b655e406 → 0.0.1-commit.b9865e97

package/src/mocks/test_context.ts

@@ -1,62 +1,67 @@
 import type { BBProverConfig } from '@aztec/bb-prover';
 import { TestCircuitProver } from '@aztec/bb-prover';
-import { SpongeBlob } from '@aztec/blob-lib';
 import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
-import { padArrayEnd, times, timesParallel } from '@aztec/foundation/collection';
-import { Fr } from '@aztec/foundation/fields';
+import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
+import { padArrayEnd, times, timesAsync } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/curves/bn254';
 import type { Logger } from '@aztec/foundation/log';
-import { TestDateProvider } from '@aztec/foundation/timer';
 import type { FieldsOf } from '@aztec/foundation/types';
 import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
 import { ProtocolContractsList } from '@aztec/protocol-contracts';
 import { computeFeePayerBalanceLeafSlot } from '@aztec/protocol-contracts/fee-juice';
-import { SimpleContractDataSource } from '@aztec/simulator/public/fixtures';
-import { PublicProcessorFactory } from '@aztec/simulator/server';
 import { PublicDataWrite } from '@aztec/stdlib/avm';
 import { AztecAddress } from '@aztec/stdlib/aztec-address';
 import { EthAddress } from '@aztec/stdlib/block';
-import { getCheckpointBlobFields } from '@aztec/stdlib/checkpoint';
-import type { ServerCircuitProver } from '@aztec/stdlib/interfaces/server';
+import type { Checkpoint } from '@aztec/stdlib/checkpoint';
+import type { MerkleTreeWriteOperations, ServerCircuitProver } from '@aztec/stdlib/interfaces/server';
 import type { CheckpointConstantData } from '@aztec/stdlib/rollup';
-import { makeBloatedProcessedTx } from '@aztec/stdlib/testing';
+import { mockProcessedTx } from '@aztec/stdlib/testing';
 import { MerkleTreeId, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
-import { type BlockHeader, type GlobalVariables, type ProcessedTx, TreeSnapshots, type Tx } from '@aztec/stdlib/tx';
+import {
+  type BlockHeader,
+  type GlobalVariables,
+  type ProcessedTx,
+  StateReference,
+  TreeSnapshots,
+} from '@aztec/stdlib/tx';
+import type { GenesisData } from '@aztec/stdlib/world-state';
 import type { MerkleTreeAdminDatabase } from '@aztec/world-state';
 import { NativeWorldStateService } from '@aztec/world-state/native';
 
 import { promises as fs } from 'fs';
 
-import { buildBlockWithCleanDB } from '../block-factory/light.js';
-import { getTreeSnapshot } from '../orchestrator/block-building-helpers.js';
+import { LightweightCheckpointBuilder } from '../light/lightweight_checkpoint_builder.js';
+import {
+  buildFinalBlobChallenges,
+  getTreeSnapshot,
+  insertSideEffects,
+} from '../orchestrator/block-building-helpers.js';
 import type { BlockProvingState } from '../orchestrator/block-proving-state.js';
 import { ProvingOrchestrator } from '../orchestrator/index.js';
 import { BrokerCircuitProverFacade } from '../proving_broker/broker_prover_facade.js';
 import { TestBroker } from '../test/mock_prover.js';
-import {
-  getEnvironmentConfig,
-  getSimulator,
-  makeCheckpointConstants,
-  makeGlobals,
-  updateExpectedTreesFromTxs,
-} from './fixtures.js';
+import { getEnvironmentConfig, getSimulator, makeCheckpointConstants, makeGlobals } from './fixtures.js';
 
 export class TestContext {
   private headers: Map<number, BlockHeader> = new Map();
+  private checkpoints: Checkpoint[] = [];
+  private checkpointOutHashes: Fr[] = [];
+  private nextCheckpointIndex = 0;
+  private nextCheckpointNumber = CheckpointNumber(1);
+  private nextBlockNumber = 1;
+  private epochNumber = 1;
   private feePayerBalance: Fr;
 
   constructor(
     public worldState: MerkleTreeAdminDatabase,
-    public firstCheckpointNumber: Fr,
-    public globalVariables: GlobalVariables,
     public prover: ServerCircuitProver,
     public broker: TestBroker,
     public brokerProverFacade: BrokerCircuitProverFacade,
     public orchestrator: TestProvingOrchestrator,
-    public blockNumber: number,
-    public feePayer: AztecAddress,
+    private feePayer: AztecAddress,
     initialFeePayerBalance: Fr,
-    public directoriesToCleanup: string[],
-    public logger: Logger,
+    private directoriesToCleanup: string[],
+    private logger: Logger,
   ) {
     this.feePayerBalance = initialFeePayerBalance;
   }
@@ -65,39 +70,28 @@ export class TestContext {
     return this.orchestrator;
   }
 
-  public getCheckpointConstants(checkpointIndex = 0): CheckpointConstantData {
-    return makeCheckpointConstants(this.firstCheckpointNumber.toNumber() + checkpointIndex);
-  }
-
   static async new(
     logger: Logger,
     {
       proverCount = 4,
       createProver = async (bbConfig: BBProverConfig) => new TestCircuitProver(await getSimulator(bbConfig, logger)),
-      slotNumber = 1,
-      blockNumber = 1,
     }: {
       proverCount?: number;
       createProver?: (bbConfig: BBProverConfig) => Promise<ServerCircuitProver>;
-      slotNumber?: number;
-      blockNumber?: number;
     } = {},
   ) {
     const directoriesToCleanup: string[] = [];
-    const firstCheckpointNumber = new Fr(slotNumber);
-    const globalVariables = makeGlobals(blockNumber, slotNumber);
 
     const feePayer = AztecAddress.fromNumber(42222);
     const initialFeePayerBalance = new Fr(10n ** 20n);
     const feePayerSlot = await computeFeePayerBalanceLeafSlot(feePayer);
-    const prefilledPublicData = [new PublicDataTreeLeaf(feePayerSlot, initialFeePayerBalance)];
+    const genesis: GenesisData = {
+      prefilledPublicData: [new PublicDataTreeLeaf(feePayerSlot, initialFeePayerBalance)],
+      genesisTimestamp: 0n,
+    };
 
     // Separated dbs for public processor and prover - see public_processor for context
-    const ws = await NativeWorldStateService.tmp(
-      /*rollupAddress=*/ undefined,
-      /*cleanupTmpDir=*/ true,
-      prefilledPublicData,
-    );
+    const ws = await NativeWorldStateService.tmp(/*cleanupTmpDir=*/ true, genesis);
 
     let localProver: ServerCircuitProver;
     const config = await getEnvironmentConfig(logger);
@@ -110,8 +104,10 @@ export class TestContext {
         bbBinaryPath: config.expectedBBPath,
         bbWorkingDirectory: config.bbWorkingDirectory,
         bbSkipCleanup: config.bbSkipCleanup,
-        numConcurrentIVCVerifiers: 2,
+        numConcurrentIVCVerifiers: 8,
         bbIVCConcurrency: 1,
+        bbChonkVerifyMaxBatch: 16,
+        bbChonkVerifyConcurrency: 6,
       };
       localProver = await createProver(bbConfig);
     }
@@ -122,20 +118,17 @@ export class TestContext {
 
     const broker = new TestBroker(proverCount, localProver);
     const facade = new BrokerCircuitProverFacade(broker);
-    const orchestrator = new TestProvingOrchestrator(ws, facade, EthAddress.ZERO);
+    const orchestrator = new TestProvingOrchestrator(ws, facade, EthAddress.ZERO, false, 10);
 
     await broker.start();
     facade.start();
 
     return new this(
       ws,
-      firstCheckpointNumber,
-      globalVariables,
       localProver,
       broker,
       facade,
       orchestrator,
-      blockNumber,
       feePayer,
       initialFeePayerBalance,
       directoriesToCleanup,
@@ -147,19 +140,10 @@ export class TestContext {
     return this.worldState.fork();
   }
 
-  public getBlockHeader(blockNumber: 0): BlockHeader;
-  public getBlockHeader(blockNumber: number): BlockHeader | undefined;
-  public getBlockHeader(blockNumber = 0): BlockHeader | undefined {
-    return blockNumber === 0 ? this.worldState.getCommitted().getInitialHeader() : this.headers.get(blockNumber);
-  }
-
-  public getPreviousBlockHeader(currentBlockNumber = this.blockNumber): BlockHeader {
-    return this.getBlockHeader(currentBlockNumber - 1)!;
-  }
-
   async cleanup() {
     await this.brokerProverFacade.stop();
     await this.broker.stop();
+    await this.worldState.close();
     for (const dir of this.directoriesToCleanup.filter(x => x !== '')) {
       try {
         await fs.rm(dir, { recursive: true, force: true, maxRetries: 3 });
@@ -169,188 +153,184 @@ export class TestContext {
     }
   }
 
-  async makeProcessedTx(opts?: Parameters<typeof makeBloatedProcessedTx>[0]): Promise<ProcessedTx> {
-    const globalVariables = opts?.globalVariables ?? this.globalVariables;
-    const blockNumber = globalVariables.blockNumber;
-    const header = opts?.header ?? this.getBlockHeader(blockNumber - 1);
-    const tx = await makeBloatedProcessedTx({
-      header,
-      vkTreeRoot: getVKTreeRoot(),
-      protocolContracts: ProtocolContractsList,
-      globalVariables,
-      feePayer: this.feePayer,
-      ...opts,
-    });
-    this.feePayerBalance = new Fr(this.feePayerBalance.toBigInt() - tx.txEffect.transactionFee.toBigInt());
-    if (opts?.privateOnly) {
-      const feePayerSlot = await computeFeePayerBalanceLeafSlot(this.feePayer);
-      tx.txEffect.publicDataWrites[0] = new PublicDataWrite(feePayerSlot, this.feePayerBalance);
-    }
-    return tx;
+  public startNewEpoch() {
+    this.checkpoints = [];
+    this.checkpointOutHashes = [];
+    this.nextCheckpointIndex = 0;
+    this.epochNumber++;
   }
 
-  /** Creates a block with the given number of txs and adds it to world-state */
-  public async makePendingBlock(
-    numTxs: number,
-    {
-      checkpointIndex = 0,
-      numL1ToL2Messages = 0,
-      blockNumber = this.blockNumber,
-      makeProcessedTxOpts = () => ({}),
-    }: {
-      checkpointIndex?: number;
-      numL1ToL2Messages?: number;
-      blockNumber?: number;
-      makeProcessedTxOpts?: (index: number) => Partial<Parameters<typeof makeBloatedProcessedTx>[0]>;
-    } = {},
-  ) {
-    const slotNumber = this.firstCheckpointNumber.toNumber() + checkpointIndex;
-    const globalVariables = makeGlobals(blockNumber, slotNumber);
-    const blockNum = globalVariables.blockNumber;
-    const db = await this.worldState.fork();
-    const l1ToL2Messages = times(numL1ToL2Messages, i => new Fr(blockNum * 100 + i));
-    const merkleTrees = await this.worldState.fork();
-    await merkleTrees.appendLeaves(
-      MerkleTreeId.L1_TO_L2_MESSAGE_TREE,
-      padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP),
-    );
-    const newL1ToL2Snapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, merkleTrees);
-    const txs = await timesParallel(numTxs, i =>
-      this.makeProcessedTx({
-        seed: i + blockNum * 1000,
-        globalVariables,
-        newL1ToL2Snapshot,
-        ...makeProcessedTxOpts(i),
-      }),
-    );
-    await this.setTreeRoots(txs);
+  // Return blob fields of all checkpoints in the epoch.
+  public getBlobFields() {
+    return this.checkpoints.map(checkpoint => checkpoint.toBlobFields());
+  }
 
-    const block = await buildBlockWithCleanDB(txs, globalVariables, l1ToL2Messages, db);
-    this.headers.set(blockNum, block.getBlockHeader());
-    await this.worldState.handleL2BlockAndMessages(block, l1ToL2Messages);
-    return { block, txs, l1ToL2Messages };
+  public async getFinalBlobChallenges() {
+    const blobFields = this.getBlobFields();
+    return await buildFinalBlobChallenges(blobFields);
   }
 
-  public async makePendingBlocksInCheckpoint(
+  public async makeCheckpoint(
     numBlocks: number,
     {
-      checkpointIndex = 0,
-      numTxsPerBlock = 1,
+      numTxsPerBlock = 0,
       numL1ToL2Messages = 0,
-      firstBlockNumber = this.blockNumber + checkpointIndex * numBlocks,
-      makeGlobalVariablesOpts = () => ({}),
       makeProcessedTxOpts = () => ({}),
+      ...constantOpts
     }: {
-      checkpointIndex?: number;
       numTxsPerBlock?: number | number[];
       numL1ToL2Messages?: number;
-      firstBlockNumber?: number;
-      makeGlobalVariablesOpts?: (
-        blockNumber: number,
-        checkpointIndex: number,
-      ) => Partial<FieldsOf<GlobalVariables> & FieldsOf<CheckpointConstantData>>;
       makeProcessedTxOpts?: (
         blockGlobalVariables: GlobalVariables,
         txIndex: number,
-      ) => Partial<Parameters<typeof makeBloatedProcessedTx>[0]>;
-    } = {},
+      ) => Partial<Parameters<typeof mockProcessedTx>[0]>;
+    } & Partial<FieldsOf<CheckpointConstantData>> = {},
   ) {
-    const slotNumber = this.firstCheckpointNumber.toNumber() + checkpointIndex;
+    if (numBlocks === 0) {
+      throw new Error(
+        'Cannot make a checkpoint with 0 blocks. Crate an empty block (numTxsPerBlock = 0) if there are no txs.',
+      );
+    }
+
+    const checkpointIndex = this.nextCheckpointIndex++;
+    const checkpointNumber = this.nextCheckpointNumber;
+    this.nextCheckpointNumber++;
+    const slotNumber = checkpointNumber * 15; // times an arbitrary number to make it different to the checkpoint number
+
+    const constants = makeCheckpointConstants(slotNumber, constantOpts);
+
+    const fork = await this.worldState.fork();
+
+    // Build l1 to l2 messages.
     const l1ToL2Messages = times(numL1ToL2Messages, i => new Fr(slotNumber * 100 + i));
-    const merkleTrees = await this.worldState.fork();
-    await merkleTrees.appendLeaves(
+    await fork.appendLeaves(
       MerkleTreeId.L1_TO_L2_MESSAGE_TREE,
       padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP),
     );
-    const newL1ToL2Snapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, merkleTrees);
+    const newL1ToL2Snapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, fork);
+
+    const startBlockNumber = this.nextBlockNumber;
+    const previousBlockHeader = this.getBlockHeader(BlockNumber(startBlockNumber - 1));
+    // All blocks in the same slot/checkpoint share the same timestamp.
+    const timestamp = BigInt(slotNumber * 26);
 
+    // Build global variables.
     const blockGlobalVariables = times(numBlocks, i =>
-      makeGlobals(firstBlockNumber + i, slotNumber, makeGlobalVariablesOpts(firstBlockNumber + i, checkpointIndex)),
+      makeGlobals(startBlockNumber + i, slotNumber, {
+        coinbase: constants.coinbase,
+        feeRecipient: constants.feeRecipient,
+        gasFees: constants.gasFees,
+        timestamp,
+      }),
     );
+    this.nextBlockNumber += numBlocks;
+
+    // Build txs.
     let totalTxs = 0;
-    const blockTxs = await timesParallel(numBlocks, blockIndex => {
+    const blockEndStates: StateReference[] = [];
+    const blockTxs = await timesAsync(numBlocks, async blockIndex => {
       const txIndexOffset = totalTxs;
       const numTxs = typeof numTxsPerBlock === 'number' ? numTxsPerBlock : numTxsPerBlock[blockIndex];
       totalTxs += numTxs;
-      return timesParallel(numTxs, txIndex =>
+      const txs = await timesAsync(numTxs, txIndex =>
         this.makeProcessedTx({
-          seed: (txIndexOffset + txIndex + 1) * 321 + (checkpointIndex + 1) * 123456,
+          seed: (txIndexOffset + txIndex + 1) * 321 + (checkpointIndex + 1) * 123456 + this.epochNumber * 0x99999,
           globalVariables: blockGlobalVariables[blockIndex],
-          header: this.getBlockHeader(firstBlockNumber - 1),
+          anchorBlockHeader: previousBlockHeader,
           newL1ToL2Snapshot,
           ...makeProcessedTxOpts(blockGlobalVariables[blockIndex], txIndexOffset + txIndex),
         }),
       );
+
+      // Insert side effects into the trees.
+      const endState = await this.updateTrees(txs, fork);
+      blockEndStates.push(endState);
+
+      return txs;
     });
 
-    const blobFields = getCheckpointBlobFields(blockTxs.map(txs => txs.map(tx => tx.txEffect)));
-    const spongeBlobState = await SpongeBlob.init(blobFields.length);
+    const cleanFork = await this.worldState.fork();
+    const previousCheckpointOutHashes = this.checkpointOutHashes;
+    const builder = await LightweightCheckpointBuilder.startNewCheckpoint(
+      checkpointNumber,
+      { ...constants, timestamp },
+      l1ToL2Messages,
+      previousCheckpointOutHashes,
+      cleanFork,
+    );
 
-    const blocks: { header: BlockHeader; txs: ProcessedTx[] }[] = [];
+    // Add tx effects to db and build block headers.
+    const blocks = [];
     for (let i = 0; i < numBlocks; i++) {
-      const isFirstBlock = i === 0;
-      const blockNumber = firstBlockNumber + i;
-      const globalVariables = blockGlobalVariables[i];
       const txs = blockTxs[i];
+      const state = blockEndStates[i];
 
-      await this.setTreeRoots(txs);
-
-      const fork = await this.worldState.fork();
-      const blockMsgs = isFirstBlock ? l1ToL2Messages : [];
-      const block = await buildBlockWithCleanDB(txs, globalVariables, blockMsgs, fork, spongeBlobState, isFirstBlock);
-
-      const header = block.getBlockHeader();
-      this.headers.set(blockNumber, header);
+      const { block } = await builder.addBlock(blockGlobalVariables[i], txs, {
+        expectedEndState: state,
+        insertTxsEffects: true,
+      });
 
-      await this.worldState.handleL2BlockAndMessages(block, blockMsgs, isFirstBlock);
+      const header = block.header;
+      this.headers.set(block.number, header);
 
-      const blockBlobFields = block.body.toBlobFields();
-      await spongeBlobState.absorb(blockBlobFields);
+      const blockMsgs = block.indexWithinCheckpoint === 0 ? l1ToL2Messages : [];
+      await this.worldState.handleL2BlockAndMessages(block, blockMsgs);
 
       blocks.push({ header, txs });
     }
 
-    return { blocks, l1ToL2Messages, blobFields };
+    const checkpoint = await builder.completeCheckpoint();
+    this.checkpoints.push(checkpoint);
+    this.checkpointOutHashes.push(checkpoint.getCheckpointOutHash());
+
+    return {
+      constants,
+      checkpoint,
+      header: checkpoint.header,
+      blocks,
+      l1ToL2Messages,
+      previousBlockHeader,
+    };
   }
 
-  public async processPublicFunctions(
-    txs: Tx[],
-    {
-      maxTransactions = txs.length,
-      numL1ToL2Messages = 0,
-      contractDataSource,
-    }: {
-      maxTransactions?: number;
-      numL1ToL2Messages?: number;
-      contractDataSource?: SimpleContractDataSource;
-    } = {},
-  ) {
-    const l1ToL2Messages = times(numL1ToL2Messages, i => new Fr(this.blockNumber * 100 + i));
-    const merkleTrees = await this.worldState.fork();
-    await merkleTrees.appendLeaves(
-      MerkleTreeId.L1_TO_L2_MESSAGE_TREE,
-      padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP),
-    );
-
-    const processorFactory = new PublicProcessorFactory(
-      contractDataSource ?? new SimpleContractDataSource(),
-      new TestDateProvider(),
-    );
-    const publicProcessor = processorFactory.create(merkleTrees, this.globalVariables, {
-      skipFeeEnforcement: false,
-      clientInitiatedSimulation: false,
+  private async makeProcessedTx(opts: Parameters<typeof mockProcessedTx>[0] = {}): Promise<ProcessedTx> {
+    const tx = await mockProcessedTx({
+      vkTreeRoot: getVKTreeRoot(),
+      protocolContracts: ProtocolContractsList,
+      feePayer: this.feePayer,
+      ...opts,
     });
 
-    return await publicProcessor.process(txs, { maxTransactions });
+    this.feePayerBalance = new Fr(this.feePayerBalance.toBigInt() - tx.txEffect.transactionFee.toBigInt());
+
+    const feePayerSlot = await computeFeePayerBalanceLeafSlot(this.feePayer);
+    const feePaymentPublicDataWrite = new PublicDataWrite(feePayerSlot, this.feePayerBalance);
+    tx.txEffect.publicDataWrites[0] = feePaymentPublicDataWrite;
+    if (tx.avmProvingRequest) {
+      tx.avmProvingRequest.inputs.publicInputs.accumulatedData.publicDataWrites[0] = feePaymentPublicDataWrite;
+    }
+
+    return tx;
+  }
+
+  private getBlockHeader(blockNumber: BlockNumber): BlockHeader {
+    if (Number(blockNumber) > 0 && Number(blockNumber) >= this.nextBlockNumber) {
+      throw new Error(`Block header not built for block number ${blockNumber}.`);
+    }
+    return Number(blockNumber) === 0
+      ? this.worldState.getCommitted().getInitialHeader()
+      : this.headers.get(Number(blockNumber))!;
   }
 
-  private async setTreeRoots(txs: ProcessedTx[]) {
-    const db = await this.worldState.fork();
+  private async updateTrees(txs: ProcessedTx[], fork: MerkleTreeWriteOperations) {
+    let startStateReference = await fork.getStateReference();
+    let endStateReference = startStateReference;
     for (const tx of txs) {
-      const startStateReference = await db.getStateReference();
-      await updateExpectedTreesFromTxs(db, [tx]);
-      const endStateReference = await db.getStateReference();
+      await insertSideEffects(tx, fork);
+      endStateReference = await fork.getStateReference();
+
       if (tx.avmProvingRequest) {
+        // Update the trees in the avm public inputs so that the proof won't fail.
         const l1ToL2MessageTree = tx.avmProvingRequest.inputs.publicInputs.startTreeSnapshots.l1ToL2MessageTree;
         tx.avmProvingRequest.inputs.publicInputs.startTreeSnapshots = new TreeSnapshots(
           l1ToL2MessageTree,
@@ -358,6 +338,7 @@ export class TestContext {
           startStateReference.partial.nullifierTree,
           startStateReference.partial.publicDataTree,
         );
+
         tx.avmProvingRequest.inputs.publicInputs.endTreeSnapshots = new TreeSnapshots(
           l1ToL2MessageTree,
           endStateReference.partial.noteHashTree,
@@ -365,7 +346,11 @@ export class TestContext {
           endStateReference.partial.publicDataTree,
         );
       }
+
+      startStateReference = endStateReference;
     }
+
+    return endStateReference;
   }
 }