@aztec/prover-client 0.0.1-commit.d431d1c → 0.0.1-commit.d939eb5aa

package/src/light/lightweight_checkpoint_builder.ts

@@ -3,8 +3,9 @@ import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
 import { type CheckpointNumber, IndexWithinCheckpoint } from '@aztec/foundation/branded-types';
 import { padArrayEnd } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
-import { createLogger } from '@aztec/foundation/log';
-import { L2BlockNew } from '@aztec/stdlib/block';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
+import { elapsed } from '@aztec/foundation/timer';
+import { L2Block } from '@aztec/stdlib/block';
 import { Checkpoint } from '@aztec/stdlib/checkpoint';
 import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
 import {
@@ -34,22 +35,28 @@ import {
  * Finally completes the checkpoint by computing its header.
  */
 export class LightweightCheckpointBuilder {
-  private readonly logger = createLogger('lightweight-checkpoint-builder');
+  private readonly logger: Logger;
 
   private lastArchives: AppendOnlyTreeSnapshot[] = [];
   private spongeBlob: SpongeBlob;
-  private blocks: L2BlockNew[] = [];
+  private blocks: L2Block[] = [];
   private blobFields: Fr[] = [];
 
   constructor(
     public readonly checkpointNumber: CheckpointNumber,
     public readonly constants: CheckpointGlobalVariables,
+    public feeAssetPriceModifier: bigint,
     public readonly l1ToL2Messages: Fr[],
     private readonly previousCheckpointOutHashes: Fr[],
     public readonly db: MerkleTreeWriteOperations,
+    bindings?: LoggerBindings,
   ) {
+    this.logger = createLogger('checkpoint-builder', {
+      ...bindings,
+      instanceId: `checkpoint-${checkpointNumber}`,
+    });
     this.spongeBlob = SpongeBlob.init();
-    this.logger.debug('Starting new checkpoint', { constants, l1ToL2Messages });
+    this.logger.debug('Starting new checkpoint', { constants, l1ToL2Messages, feeAssetPriceModifier });
   }
 
   static async startNewCheckpoint(
@@ -58,6 +65,8 @@ export class LightweightCheckpointBuilder {
     l1ToL2Messages: Fr[],
     previousCheckpointOutHashes: Fr[],
     db: MerkleTreeWriteOperations,
+    bindings?: LoggerBindings,
+    feeAssetPriceModifier: bigint = 0n,
   ): Promise<LightweightCheckpointBuilder> {
     // Insert l1-to-l2 messages into the tree.
     await db.appendLeaves(
@@ -68,9 +77,11 @@ export class LightweightCheckpointBuilder {
     return new LightweightCheckpointBuilder(
       checkpointNumber,
       constants,
+      feeAssetPriceModifier,
       l1ToL2Messages,
       previousCheckpointOutHashes,
       db,
+      bindings,
     );
   }
 
@@ -83,17 +94,21 @@ export class LightweightCheckpointBuilder {
   static async resumeCheckpoint(
     checkpointNumber: CheckpointNumber,
     constants: CheckpointGlobalVariables,
+    feeAssetPriceModifier: bigint,
     l1ToL2Messages: Fr[],
     previousCheckpointOutHashes: Fr[],
     db: MerkleTreeWriteOperations,
-    existingBlocks: L2BlockNew[],
+    existingBlocks: L2Block[],
+    bindings?: LoggerBindings,
   ): Promise<LightweightCheckpointBuilder> {
     const builder = new LightweightCheckpointBuilder(
       checkpointNumber,
       constants,
+      feeAssetPriceModifier,
       l1ToL2Messages,
       previousCheckpointOutHashes,
       db,
+      bindings,
     );
 
     builder.logger.debug('Resuming checkpoint from existing blocks', {
@@ -134,6 +149,15 @@ export class LightweightCheckpointBuilder {
     return builder;
   }
 
+  /** Returns how many blocks have been added to this checkpoint so far */
+  public getBlockCount() {
+    return this.blocks.length;
+  }
+
+  public getBlocks() {
+    return this.blocks;
+  }
+
   /**
    * Adds a new block to the checkpoint. The tx effects must have already been inserted into the db if
    * this is called after tx processing, if that's not the case, then set `insertTxsEffects` to true.
@@ -142,7 +166,8 @@ export class LightweightCheckpointBuilder {
     globalVariables: GlobalVariables,
     txs: ProcessedTx[],
     opts: { insertTxsEffects?: boolean; expectedEndState?: StateReference } = {},
-  ): Promise<L2BlockNew> {
+  ): Promise<{ block: L2Block; timings: Record<string, number> }> {
+    const timings: Record<string, number> = {};
     const isFirstBlock = this.blocks.length === 0;
 
     // Empty blocks are only allowed as the first block in a checkpoint
@@ -151,7 +176,9 @@ export class LightweightCheckpointBuilder {
     }
 
     if (isFirstBlock) {
-      this.lastArchives.push(await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db));
+      const [msGetInitialArchive, initialArchive] = await elapsed(() => getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db));
+      this.lastArchives.push(initialArchive);
+      timings.getInitialArchive = msGetInitialArchive;
     }
 
     const lastArchive = this.lastArchives.at(-1)!;
@@ -161,12 +188,17 @@ export class LightweightCheckpointBuilder {
         `Inserting side effects for ${txs.length} txs for block ${globalVariables.blockNumber} into db`,
         { txs: txs.map(tx => tx.hash.toString()) },
       );
+      let msInsertSideEffects = 0;
       for (const tx of txs) {
-        await insertSideEffects(tx, this.db);
+        const [ms] = await elapsed(() => insertSideEffects(tx, this.db));
+        msInsertSideEffects += ms;
       }
+      timings.insertSideEffects = msInsertSideEffects;
     }
 
-    const endState = await this.db.getStateReference();
+    const [msGetEndState, endState] = await elapsed(() => this.db.getStateReference());
+    timings.getEndState = msGetEndState;
+
     if (opts.expectedEndState && !endState.equals(opts.expectedEndState)) {
       this.logger.error('End state after processing txs does not match expected end state', {
         globalVariables: globalVariables.toInspect(),
@@ -176,26 +208,31 @@ export class LightweightCheckpointBuilder {
       throw new Error(`End state does not match expected end state when building block ${globalVariables.blockNumber}`);
     }
 
-    const { header, body, blockBlobFields } = await buildHeaderAndBodyFromTxs(
-      txs,
-      lastArchive,
-      endState,
-      globalVariables,
-      this.spongeBlob,
-      isFirstBlock,
+    const [msBuildHeaderAndBody, { header, body, blockBlobFields }] = await elapsed(() =>
+      buildHeaderAndBodyFromTxs(txs, lastArchive, endState, globalVariables, this.spongeBlob, isFirstBlock),
     );
+    timings.buildHeaderAndBody = msBuildHeaderAndBody;
 
     header.state.validate();
 
     await this.db.updateArchive(header);
-    const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db);
+    const [msUpdateArchive, newArchive] = await elapsed(() => getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db));
+    timings.updateArchive = msUpdateArchive;
     this.lastArchives.push(newArchive);
 
+    const expectedNextLeafIndex = Number(globalVariables.blockNumber) + 1;
+    if (newArchive.nextAvailableLeafIndex !== expectedNextLeafIndex) {
+      throw new Error(
+        `Archive tree next leaf index mismatch after building block ${globalVariables.blockNumber} (expected ${expectedNextLeafIndex} but got ${newArchive.nextAvailableLeafIndex})`,
+      );
+    }
+
     const indexWithinCheckpoint = IndexWithinCheckpoint(this.blocks.length);
-    const block = new L2BlockNew(newArchive, header, body, this.checkpointNumber, indexWithinCheckpoint);
+    const block = new L2Block(newArchive, header, body, this.checkpointNumber, indexWithinCheckpoint);
     this.blocks.push(block);
 
-    await this.spongeBlob.absorb(blockBlobFields);
+    const [msSpongeAbsorb] = await elapsed(() => this.spongeBlob.absorb(blockBlobFields));
+    timings.spongeAbsorb = msSpongeAbsorb;
     this.blobFields.push(...blockBlobFields);
 
     this.logger.debug(`Built block ${header.getBlockNumber()}`, {
@@ -206,7 +243,7 @@ export class LightweightCheckpointBuilder {
       txs: block.body.txEffects.map(tx => tx.txHash.toString()),
     });
 
-    return block;
+    return { block, timings };
   }
 
   async completeCheckpoint(): Promise<Checkpoint> {
@@ -223,7 +260,7 @@ export class LightweightCheckpointBuilder {
 
     const newArchive = this.lastArchives[this.lastArchives.length - 1];
 
-    const blobs = getBlobsPerL1Block(this.blobFields);
+    const blobs = await getBlobsPerL1Block(this.blobFields);
     const blobsHash = computeBlobsHashFromBlobs(blobs);
 
     const inHash = computeInHashFromL1ToL2Messages(this.l1ToL2Messages);
@@ -234,8 +271,7 @@ export class LightweightCheckpointBuilder {
     );
     const epochOutHash = accumulateCheckpointOutHashes([...this.previousCheckpointOutHashes, checkpointOutHash]);
 
-    // TODO(palla/mbps): Should we source this from the constants instead?
-    // timestamp of a checkpoint is the timestamp of the last block in the checkpoint.
+    // All blocks in the checkpoint have the same timestamp
     const timestamp = blocks[blocks.length - 1].timestamp;
 
     const totalManaUsed = blocks.reduce((acc, block) => acc.add(block.header.totalManaUsed), Fr.ZERO);
@@ -254,16 +290,18 @@ export class LightweightCheckpointBuilder {
       totalManaUsed,
     });
 
-    return new Checkpoint(newArchive, header, blocks, this.checkpointNumber);
+    return new Checkpoint(newArchive, header, blocks, this.checkpointNumber, this.feeAssetPriceModifier);
   }
 
   clone() {
     const clone = new LightweightCheckpointBuilder(
       this.checkpointNumber,
       this.constants,
+      this.feeAssetPriceModifier,
       [...this.l1ToL2Messages],
       [...this.previousCheckpointOutHashes],
       this.db,
+      this.logger.getBindings(),
     );
     clone.lastArchives = [...this.lastArchives];
     clone.spongeBlob = this.spongeBlob.clone();

package/src/mocks/fixtures.ts

@@ -75,7 +75,8 @@ export async function getSimulator(
       logger?.info(
         `Using native ACVM at ${config.acvmBinaryPath} and working directory ${config.acvmWorkingDirectory}`,
       );
-      return new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath);
+      const acvmLogger = logger?.createChild('acvm-native');
+      return new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath, undefined, acvmLogger);
     } catch {
       logger?.warn(`Failed to access ACVM at ${config.acvmBinaryPath}, falling back to WASM`);
     }

package/src/mocks/test_context.ts

@@ -24,6 +24,7 @@ import {
   StateReference,
   TreeSnapshots,
 } from '@aztec/stdlib/tx';
+import type { GenesisData } from '@aztec/stdlib/world-state';
 import type { MerkleTreeAdminDatabase } from '@aztec/world-state';
 import { NativeWorldStateService } from '@aztec/world-state/native';
 
@@ -84,14 +85,13 @@ export class TestContext {
     const feePayer = AztecAddress.fromNumber(42222);
     const initialFeePayerBalance = new Fr(10n ** 20n);
     const feePayerSlot = await computeFeePayerBalanceLeafSlot(feePayer);
-    const prefilledPublicData = [new PublicDataTreeLeaf(feePayerSlot, initialFeePayerBalance)];
+    const genesis: GenesisData = {
+      prefilledPublicData: [new PublicDataTreeLeaf(feePayerSlot, initialFeePayerBalance)],
+      genesisTimestamp: 0n,
+    };
 
     // Separated dbs for public processor and prover - see public_processor for context
-    const ws = await NativeWorldStateService.tmp(
-      /*rollupAddress=*/ undefined,
-      /*cleanupTmpDir=*/ true,
-      prefilledPublicData,
-    );
+    const ws = await NativeWorldStateService.tmp(/*rollupAddress=*/ undefined, /*cleanupTmpDir=*/ true, genesis);
 
     let localProver: ServerCircuitProver;
     const config = await getEnvironmentConfig(logger);
@@ -104,8 +104,10 @@ export class TestContext {
         bbBinaryPath: config.expectedBBPath,
         bbWorkingDirectory: config.bbWorkingDirectory,
         bbSkipCleanup: config.bbSkipCleanup,
-        numConcurrentIVCVerifiers: 2,
+        numConcurrentIVCVerifiers: 8,
         bbIVCConcurrency: 1,
+        bbChonkVerifyMaxBatch: 16,
+        bbChonkVerifyConcurrency: 6,
       };
       localProver = await createProver(bbConfig);
     }
@@ -116,7 +118,7 @@ export class TestContext {
 
     const broker = new TestBroker(proverCount, localProver);
     const facade = new BrokerCircuitProverFacade(broker);
-    const orchestrator = new TestProvingOrchestrator(ws, facade, EthAddress.ZERO);
+    const orchestrator = new TestProvingOrchestrator(ws, facade, EthAddress.ZERO, false, 10);
 
     await broker.start();
     facade.start();
@@ -250,7 +252,7 @@ export class TestContext {
     const previousCheckpointOutHashes = this.checkpointOutHashes;
     const builder = await LightweightCheckpointBuilder.startNewCheckpoint(
       checkpointNumber,
-      constants,
+      { ...constants, timestamp },
       l1ToL2Messages,
       previousCheckpointOutHashes,
       cleanFork,
@@ -262,7 +264,7 @@ export class TestContext {
       const txs = blockTxs[i];
       const state = blockEndStates[i];
 
-      const block = await builder.addBlock(blockGlobalVariables[i], txs, {
+      const { block } = await builder.addBlock(blockGlobalVariables[i], txs, {
         expectedEndState: state,
         insertTxsEffects: true,
       });

package/src/orchestrator/block-building-helpers.ts

@@ -99,7 +99,7 @@ export const insertSideEffectsAndBuildBaseRollupHints = runInSpan(
 
     const blockHash = await tx.data.constants.anchorBlockHeader.hash();
     const anchorBlockArchiveSiblingPath = (
-      await getMembershipWitnessFor(blockHash, MerkleTreeId.ARCHIVE, ARCHIVE_HEIGHT, db)
+      await getMembershipWitnessFor(blockHash.toFr(), MerkleTreeId.ARCHIVE, ARCHIVE_HEIGHT, db)
     ).siblingPath;
 
     const contractClassLogsFields = makeTuple(
@@ -253,8 +253,8 @@ export function getPublicChonkVerifierPrivateInputsFromTx(tx: Tx | ProcessedTx,
 // Build "hints" as the private inputs for the checkpoint root rollup circuit.
 // The `blobCommitments` will be accumulated and checked in the root rollup against the `finalBlobChallenges`.
 // The `blobsHash` will be validated on L1 against the submitted blob data.
-export const buildBlobHints = (blobFields: Fr[]) => {
-  const blobs = getBlobsPerL1Block(blobFields);
+export const buildBlobHints = async (blobFields: Fr[]) => {
+  const blobs = await getBlobsPerL1Block(blobFields);
   const blobCommitments = getBlobCommitmentsFromBlobs(blobs);
   const blobsHash = computeBlobsHashFromBlobs(blobs);
   return { blobCommitments, blobs, blobsHash };

package/src/orchestrator/block-proving-state.ts

@@ -55,6 +55,7 @@ export class BlockProvingState {
     | ProofState<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
     | undefined;
   private builtBlockHeader: BlockHeader | undefined;
+  private builtArchive: AppendOnlyTreeSnapshot | undefined;
   private endState: StateReference | undefined;
   private endSpongeBlob: SpongeBlob | undefined;
   private txs: TxProvingState[] = [];
@@ -232,6 +233,14 @@ export class BlockProvingState {
     return this.builtBlockHeader;
   }
 
+  public setBuiltArchive(archive: AppendOnlyTreeSnapshot) {
+    this.builtArchive = archive;
+  }
+
+  public getBuiltArchive() {
+    return this.builtArchive;
+  }
+
   public getStartSpongeBlob() {
     return this.startSpongeBlob;
   }

package/src/orchestrator/checkpoint-proving-state.ts

@@ -85,7 +85,7 @@ export class CheckpointProvingState {
       typeof L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH
     >,
     public parentEpoch: EpochProvingState,
-    private onBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => void,
+    private onBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => Promise<void>,
   ) {
     this.blockProofs = new UnbalancedTreeStore(totalNumBlocks);
     this.firstBlockNumber = BlockNumber(headerOfLastBlockInPreviousCheckpoint.globalVariables.blockNumber + 1);
@@ -245,7 +245,7 @@ export class CheckpointProvingState {
     this.endBlobAccumulator = await accumulateBlobs(this.blobFields!, startBlobAccumulator);
     this.startBlobAccumulator = startBlobAccumulator;
 
-    this.onBlobAccumulatorSet(this);
+    await this.onBlobAccumulatorSet(this);
 
     return this.endBlobAccumulator;
   }
@@ -271,7 +271,7 @@ export class CheckpointProvingState {
     return this.totalNumBlocks === 1 ? 'rollup-checkpoint-root-single-block' : 'rollup-checkpoint-root';
   }
 
-  public getCheckpointRootRollupInputs() {
+  public async getCheckpointRootRollupInputs() {
     const proofs = this.#getChildProofsForRoot();
     const nonEmptyProofs = proofs.filter(p => !!p);
     if (proofs.length !== nonEmptyProofs.length) {
@@ -287,7 +287,7 @@ export class CheckpointProvingState {
     // `blobFields` must've been set if `startBlobAccumulator` is set (in `accumulateBlobs`).
     const blobFields = this.blobFields!;
 
-    const { blobCommitments, blobsHash } = buildBlobHints(blobFields);
+    const { blobCommitments, blobsHash } = await buildBlobHints(blobFields);
 
     const hints = CheckpointRootRollupHints.from({
       previousBlockHeader: this.headerOfLastBlockInPreviousCheckpoint,

package/src/orchestrator/epoch-proving-state.ts

@@ -76,7 +76,7 @@ export class EpochProvingState {
     public readonly epochNumber: EpochNumber,
     public readonly totalNumCheckpoints: number,
     private readonly finalBlobBatchingChallenges: FinalBlobBatchingChallenges,
-    private onCheckpointBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => void,
+    private onCheckpointBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => Promise<void>,
     private completionCallback: (result: ProvingResult) => void,
     private rejectionCallback: (reason: string) => void,
   ) {
@@ -254,9 +254,11 @@ export class EpochProvingState {
       }
       outHashes.push(outHash);
 
-      // Get or create hints for the next checkpoint.
-      hint = checkpoint.getOutHashHintForNextCheckpoint() ?? (await computeOutHashHint(outHashes));
-      checkpoint.setOutHashHintForNextCheckpoint(hint);
+      // If this is NOT the last checkpoint, get or create the hint for the next checkpoint.
+      if (i !== this.totalNumCheckpoints - 1) {
+        hint = checkpoint.getOutHashHintForNextCheckpoint() ?? (await computeOutHashHint(outHashes));
+        checkpoint.setOutHashHintForNextCheckpoint(hint);
+      }
     }
   }