@aztec/prover-client 0.0.1-commit.f504929 → 0.0.1-commit.f5a9928

package/src/light/lightweight_checkpoint_builder.ts

@@ -1,7 +1,5 @@
 import { SpongeBlob, computeBlobsHashFromBlobs, encodeCheckpointEndMarker, getBlobsPerL1Block } from '@aztec/blob-lib';
-import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
 import { type CheckpointNumber, IndexWithinCheckpoint } from '@aztec/foundation/branded-types';
-import { padArrayEnd } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { elapsed } from '@aztec/foundation/timer';
@@ -10,6 +8,7 @@ import { Checkpoint } from '@aztec/stdlib/checkpoint';
 import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
 import {
   accumulateCheckpointOutHashes,
+  appendL1ToL2MessagesToTree,
   computeCheckpointOutHash,
   computeInHashFromL1ToL2Messages,
 } from '@aztec/stdlib/messaging';
@@ -69,10 +68,7 @@ export class LightweightCheckpointBuilder {
     feeAssetPriceModifier: bigint = 0n,
   ): Promise<LightweightCheckpointBuilder> {
     // Insert l1-to-l2 messages into the tree.
-    await db.appendLeaves(
-      MerkleTreeId.L1_TO_L2_MESSAGE_TREE,
-      padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP),
-    );
+    await appendL1ToL2MessagesToTree(db, l1ToL2Messages);
 
     return new LightweightCheckpointBuilder(
       checkpointNumber,
@@ -117,6 +113,10 @@ export class LightweightCheckpointBuilder {
       blockNumbers: existingBlocks.map(b => b.header.getBlockNumber()),
     });
 
+    if (existingBlocks.length === 0) {
+      throw new Error(`Cannot resume checkpoint ${checkpointNumber} with no existing blocks`);
+    }
+
     // Validate block order and consistency
     for (let i = 1; i < existingBlocks.length; i++) {
       const prev = existingBlocks[i - 1];
@@ -154,6 +154,10 @@ export class LightweightCheckpointBuilder {
     return this.blocks.length;
   }
 
+  public getBlocks() {
+    return this.blocks;
+  }
+
   /**
    * Adds a new block to the checkpoint. The tx effects must have already been inserted into the db if
    * this is called after tx processing, if that's not the case, then set `insertTxsEffects` to true.
@@ -216,6 +220,13 @@ export class LightweightCheckpointBuilder {
     timings.updateArchive = msUpdateArchive;
     this.lastArchives.push(newArchive);
 
+    const expectedNextLeafIndex = Number(globalVariables.blockNumber) + 1;
+    if (newArchive.nextAvailableLeafIndex !== expectedNextLeafIndex) {
+      throw new Error(
+        `Archive tree next leaf index mismatch after building block ${globalVariables.blockNumber} (expected ${expectedNextLeafIndex} but got ${newArchive.nextAvailableLeafIndex})`,
+      );
+    }
+
     const indexWithinCheckpoint = IndexWithinCheckpoint(this.blocks.length);
     const block = new L2Block(newArchive, header, body, this.checkpointNumber, indexWithinCheckpoint);
     this.blocks.push(block);
@@ -264,6 +275,7 @@ export class LightweightCheckpointBuilder {
     const timestamp = blocks[blocks.length - 1].timestamp;
 
     const totalManaUsed = blocks.reduce((acc, block) => acc.add(block.header.totalManaUsed), Fr.ZERO);
+    const accumulatedFees = blocks.reduce((acc, block) => acc.add(block.header.totalFees), Fr.ZERO);
 
     const header = CheckpointHeader.from({
       lastArchiveRoot: this.lastArchives[0].root,
@@ -277,6 +289,15 @@ export class LightweightCheckpointBuilder {
       feeRecipient,
       gasFees,
       totalManaUsed,
+      accumulatedFees,
+    });
+
+    this.logger.debug(`Completed checkpoint ${this.checkpointNumber}`, {
+      checkpointNumber: this.checkpointNumber,
+      headerHash: header.hash().toString(),
+      checkpointOutHash: checkpointOutHash.toString(),
+      numPreviousCheckpointOutHashes: this.previousCheckpointOutHashes.length,
+      ...header.toInspect(),
     });
 
     return new Checkpoint(newArchive, header, blocks, this.checkpointNumber, this.feeAssetPriceModifier);

package/src/mocks/test_context.ts

@@ -5,13 +5,13 @@ import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
 import { padArrayEnd, times, timesAsync } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import type { Logger } from '@aztec/foundation/log';
+import { SerialQueue } from '@aztec/foundation/queue';
 import type { FieldsOf } from '@aztec/foundation/types';
 import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
 import { ProtocolContractsList } from '@aztec/protocol-contracts';
 import { computeFeePayerBalanceLeafSlot } from '@aztec/protocol-contracts/fee-juice';
 import { PublicDataWrite } from '@aztec/stdlib/avm';
 import { AztecAddress } from '@aztec/stdlib/aztec-address';
-import { EthAddress } from '@aztec/stdlib/block';
 import type { Checkpoint } from '@aztec/stdlib/checkpoint';
 import type { MerkleTreeWriteOperations, ServerCircuitProver } from '@aztec/stdlib/interfaces/server';
 import type { CheckpointConstantData } from '@aztec/stdlib/rollup';
@@ -24,6 +24,7 @@ import {
   StateReference,
   TreeSnapshots,
 } from '@aztec/stdlib/tx';
+import type { GenesisData } from '@aztec/stdlib/world-state';
 import type { MerkleTreeAdminDatabase } from '@aztec/world-state';
 import { NativeWorldStateService } from '@aztec/world-state/native';
 
@@ -35,12 +36,21 @@ import {
   getTreeSnapshot,
   insertSideEffects,
 } from '../orchestrator/block-building-helpers.js';
-import type { BlockProvingState } from '../orchestrator/block-proving-state.js';
-import { ProvingOrchestrator } from '../orchestrator/index.js';
 import { BrokerCircuitProverFacade } from '../proving_broker/broker_prover_facade.js';
 import { TestBroker } from '../test/mock_prover.js';
 import { getEnvironmentConfig, getSimulator, makeCheckpointConstants, makeGlobals } from './fixtures.js';
 
+/**
+ * Builds a started `SerialQueue` for use as an orchestrator's deferred-job queue in
+ * tests. Production wires a single shared queue via `ProverClient`; tests that construct
+ * orchestrators directly use one of these per orchestrator (or share one).
+ */
+export function makeTestDeferredJobQueue(concurrency = 10): SerialQueue {
+  const queue = new SerialQueue();
+  queue.start(concurrency);
+  return queue;
+}
+
 export class TestContext {
   private headers: Map<number, BlockHeader> = new Map();
   private checkpoints: Checkpoint[] = [];
@@ -56,7 +66,6 @@ export class TestContext {
     public prover: ServerCircuitProver,
     public broker: TestBroker,
     public brokerProverFacade: BrokerCircuitProverFacade,
-    public orchestrator: TestProvingOrchestrator,
     private feePayer: AztecAddress,
     initialFeePayerBalance: Fr,
     private directoriesToCleanup: string[],
@@ -65,10 +74,6 @@ export class TestContext {
     this.feePayerBalance = initialFeePayerBalance;
   }
 
-  public get epochProver() {
-    return this.orchestrator;
-  }
-
   static async new(
     logger: Logger,
     {
@@ -81,17 +86,16 @@ export class TestContext {
   ) {
     const directoriesToCleanup: string[] = [];
 
-    const feePayer = AztecAddress.fromNumber(42222);
+    const feePayer = AztecAddress.fromNumberUnsafe(42222);
     const initialFeePayerBalance = new Fr(10n ** 20n);
     const feePayerSlot = await computeFeePayerBalanceLeafSlot(feePayer);
-    const prefilledPublicData = [new PublicDataTreeLeaf(feePayerSlot, initialFeePayerBalance)];
+    const genesis: GenesisData = {
+      prefilledPublicData: [new PublicDataTreeLeaf(feePayerSlot, initialFeePayerBalance)],
+      genesisTimestamp: 0n,
+    };
 
     // Separated dbs for public processor and prover - see public_processor for context
-    const ws = await NativeWorldStateService.tmp(
-      /*rollupAddress=*/ undefined,
-      /*cleanupTmpDir=*/ true,
-      prefilledPublicData,
-    );
+    const ws = await NativeWorldStateService.tmp(/*rollupAddress=*/ undefined, /*cleanupTmpDir=*/ true, genesis);
 
     let localProver: ServerCircuitProver;
     const config = await getEnvironmentConfig(logger);
@@ -104,8 +108,10 @@ export class TestContext {
         bbBinaryPath: config.expectedBBPath,
         bbWorkingDirectory: config.bbWorkingDirectory,
         bbSkipCleanup: config.bbSkipCleanup,
-        numConcurrentIVCVerifiers: 2,
+        numConcurrentIVCVerifiers: 8,
         bbIVCConcurrency: 1,
+        bbChonkVerifyMaxBatch: 16,
+        bbChonkVerifyConcurrency: 6,
       };
       localProver = await createProver(bbConfig);
     }
@@ -116,22 +122,11 @@ export class TestContext {
 
     const broker = new TestBroker(proverCount, localProver);
     const facade = new BrokerCircuitProverFacade(broker);
-    const orchestrator = new TestProvingOrchestrator(ws, facade, EthAddress.ZERO);
 
     await broker.start();
     facade.start();
 
-    return new this(
-      ws,
-      localProver,
-      broker,
-      facade,
-      orchestrator,
-      feePayer,
-      initialFeePayerBalance,
-      directoriesToCleanup,
-      logger,
-    );
+    return new this(ws, localProver, broker, facade, feePayer, initialFeePayerBalance, directoriesToCleanup, logger);
   }
 
   public getFork() {
@@ -282,6 +277,7 @@ export class TestContext {
 
     return {
       constants,
+      checkpoint,
       header: checkpoint.header,
       blocks,
       l1ToL2Messages,
@@ -349,16 +345,3 @@ export class TestContext {
     return endStateReference;
   }
 }
-
-class TestProvingOrchestrator extends ProvingOrchestrator {
-  public isVerifyBuiltBlockAgainstSyncedStateEnabled = false;
-
-  // Disable this check by default, since it requires seeding world state with the block being built
-  // This is only enabled in some tests with multiple blocks that populate the pending chain via makePendingBlock
-  protected override verifyBuiltBlockAgainstSyncedState(provingState: BlockProvingState): Promise<void> {
-    if (this.isVerifyBuiltBlockAgainstSyncedStateEnabled) {
-      return super.verifyBuiltBlockAgainstSyncedState(provingState);
-    }
-    return Promise.resolve();
-  }
-}

package/src/orchestrator/checkpoint-proving-state.ts

@@ -1,66 +1,30 @@
-import {
-  BatchedBlobAccumulator,
-  type FinalBlobBatchingChallenges,
-  SpongeBlob,
-  encodeCheckpointBlobDataFromBlocks,
-} from '@aztec/blob-lib';
+import { SpongeBlob } from '@aztec/blob-lib';
 import {
   type ARCHIVE_HEIGHT,
-  BLOBS_PER_CHECKPOINT,
-  FIELDS_PER_BLOB,
   type L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
   type NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
   NUM_MSGS_PER_BASE_PARITY,
-  OUT_HASH_TREE_HEIGHT,
 } from '@aztec/constants';
 import { BlockNumber } from '@aztec/foundation/branded-types';
 import { padArrayEnd } from '@aztec/foundation/collection';
-import { BLS12Point } from '@aztec/foundation/curves/bls12';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import type { Tuple } from '@aztec/foundation/serialize';
 import { type TreeNodeLocation, UnbalancedTreeStore } from '@aztec/foundation/trees';
 import type { PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
-import { computeCheckpointOutHash } from '@aztec/stdlib/messaging';
 import { ParityBasePrivateInputs } from '@aztec/stdlib/parity';
-import {
-  BlockMergeRollupPrivateInputs,
-  BlockRollupPublicInputs,
-  CheckpointConstantData,
-  CheckpointRollupPublicInputs,
-  CheckpointRootRollupHints,
-  CheckpointRootRollupPrivateInputs,
-  CheckpointRootSingleBlockRollupPrivateInputs,
-} from '@aztec/stdlib/rollup';
-import type { CircuitName } from '@aztec/stdlib/stats';
+import { BlockMergeRollupPrivateInputs, BlockRollupPublicInputs, CheckpointConstantData } from '@aztec/stdlib/rollup';
 import type { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees';
 import type { BlockHeader } from '@aztec/stdlib/tx';
 import type { UInt64 } from '@aztec/stdlib/types';
 
-import { accumulateBlobs, buildBlobHints, toProofData } from './block-building-helpers.js';
+import { toProofData } from './block-building-helpers.js';
 import { BlockProvingState, type ProofState } from './block-proving-state.js';
-import type { EpochProvingState } from './epoch-proving-state.js';
-
-type OutHashHint = {
-  treeSnapshot: AppendOnlyTreeSnapshot;
-  siblingPath: Tuple<Fr, typeof OUT_HASH_TREE_HEIGHT>;
-};
 
 export class CheckpointProvingState {
   private blockProofs: UnbalancedTreeStore<
     ProofState<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
   >;
-  private checkpointRootProof:
-    | ProofState<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
-    | undefined;
   private blocks: (BlockProvingState | undefined)[] = [];
-  private previousOutHashHint: OutHashHint | undefined;
-  private outHash: Fr | undefined;
-  // The snapshot and sibling path after the checkpoint's out hash is inserted.
-  // Stored here to be retrieved for the next checkpoint when it's added.
-  private newOutHashHint: OutHashHint | undefined;
-  private startBlobAccumulator: BatchedBlobAccumulator | undefined;
-  private endBlobAccumulator: BatchedBlobAccumulator | undefined;
-  private blobFields: Fr[] | undefined;
   private error: string | undefined;
   public readonly firstBlockNumber: BlockNumber;
 
@@ -68,7 +32,6 @@ export class CheckpointProvingState {
     public readonly index: number,
     public readonly constants: CheckpointConstantData,
     public readonly totalNumBlocks: number,
-    private readonly finalBlobBatchingChallenges: FinalBlobBatchingChallenges,
     private readonly headerOfLastBlockInPreviousCheckpoint: BlockHeader,
     private readonly lastArchiveSiblingPath: Tuple<Fr, typeof ARCHIVE_HEIGHT>,
     private readonly l1ToL2Messages: Fr[],
@@ -84,17 +47,16 @@ export class CheckpointProvingState {
       Fr,
       typeof L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH
     >,
-    public parentEpoch: EpochProvingState,
-    private onBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => Promise<void>,
+    public readonly epochNumber: number,
+    /** Owner's liveness check. `verifyState()` returns false once this returns false. */
+    private readonly isAlive: () => boolean,
+    /** Owner's failure callback. Invoked from `reject` to surface the error upward. */
+    private readonly onReject: (reason: string) => void,
   ) {
     this.blockProofs = new UnbalancedTreeStore(totalNumBlocks);
     this.firstBlockNumber = BlockNumber(headerOfLastBlockInPreviousCheckpoint.globalVariables.blockNumber + 1);
   }
 
-  public get epochNumber(): number {
-    return this.parentEpoch.epochNumber;
-  }
-
   public startNewBlock(
     blockNumber: BlockNumber,
     timestamp: UInt64,
@@ -176,25 +138,6 @@ export class CheckpointProvingState {
     this.blockProofs.setNode(location, { provingOutput });
   }
 
-  public tryStartProvingCheckpointRoot() {
-    if (this.checkpointRootProof?.isProving) {
-      return false;
-    } else {
-      this.checkpointRootProof = { isProving: true };
-      return true;
-    }
-  }
-
-  public setCheckpointRootRollupProof(
-    provingOutput: PublicInputsAndRecursiveProof<
-      CheckpointRollupPublicInputs,
-      typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH
-    >,
-  ): TreeNodeLocation {
-    this.checkpointRootProof = { provingOutput };
-    return this.parentEpoch.setCheckpointRootRollupProof(this.index, provingOutput);
-  }
-
   public getBaseParityInputs(baseParityIndex: number) {
     const messages = padArrayEnd(
       this.l1ToL2Messages.slice(
@@ -204,54 +147,7 @@ export class CheckpointProvingState {
       Fr.ZERO,
       NUM_MSGS_PER_BASE_PARITY,
     );
-    return new ParityBasePrivateInputs(messages, this.constants.vkTreeRoot);
-  }
-
-  public setOutHashHint(hint: OutHashHint) {
-    this.previousOutHashHint = hint;
-  }
-
-  public getOutHashHint() {
-    return this.previousOutHashHint;
-  }
-
-  public accumulateBlockOutHashes() {
-    if (this.isAcceptingBlocks() || this.blocks.some(b => !b?.hasEndState())) {
-      return;
-    }
-
-    if (!this.outHash) {
-      const messagesPerBlock = this.blocks.map(b => b!.getTxEffects().map(tx => tx.l2ToL1Msgs));
-      this.outHash = computeCheckpointOutHash(messagesPerBlock);
-    }
-
-    return this.outHash;
-  }
-
-  public setOutHashHintForNextCheckpoint(hint: OutHashHint) {
-    this.newOutHashHint = hint;
-  }
-
-  public getOutHashHintForNextCheckpoint() {
-    return this.newOutHashHint;
-  }
-
-  public async accumulateBlobs(startBlobAccumulator: BatchedBlobAccumulator) {
-    if (this.isAcceptingBlocks() || this.blocks.some(b => !b?.hasEndState())) {
-      return;
-    }
-
-    this.blobFields = encodeCheckpointBlobDataFromBlocks(this.blocks.map(b => b!.getBlockBlobData()));
-    this.endBlobAccumulator = await accumulateBlobs(this.blobFields!, startBlobAccumulator);
-    this.startBlobAccumulator = startBlobAccumulator;
-
-    await this.onBlobAccumulatorSet(this);
-
-    return this.endBlobAccumulator;
-  }
-
-  public getEndBlobAccumulator() {
-    return this.endBlobAccumulator;
+    return new ParityBasePrivateInputs(messages, this.constants.vkTreeRoot, this.constants.proverId);
   }
 
   public getParentLocation(location: TreeNodeLocation) {
@@ -267,47 +163,6 @@ export class CheckpointProvingState {
     return new BlockMergeRollupPrivateInputs([toProofData(left), toProofData(right)]);
   }
 
-  public getCheckpointRootRollupType(): CircuitName {
-    return this.totalNumBlocks === 1 ? 'rollup-checkpoint-root-single-block' : 'rollup-checkpoint-root';
-  }
-
-  public async getCheckpointRootRollupInputs() {
-    const proofs = this.#getChildProofsForRoot();
-    const nonEmptyProofs = proofs.filter(p => !!p);
-    if (proofs.length !== nonEmptyProofs.length) {
-      throw new Error('At least one child is not ready for the checkpoint root rollup.');
-    }
-    if (!this.previousOutHashHint) {
-      throw new Error('Out hash hint is not set.');
-    }
-    if (!this.startBlobAccumulator) {
-      throw new Error('Start blob accumulator is not set.');
-    }
-
-    // `blobFields` must've been set if `startBlobAccumulator` is set (in `accumulateBlobs`).
-    const blobFields = this.blobFields!;
-
-    const { blobCommitments, blobsHash } = await buildBlobHints(blobFields);
-
-    const hints = CheckpointRootRollupHints.from({
-      previousBlockHeader: this.headerOfLastBlockInPreviousCheckpoint,
-      previousArchiveSiblingPath: this.lastArchiveSiblingPath,
-      previousOutHash: this.previousOutHashHint.treeSnapshot,
-      newOutHashSiblingPath: this.previousOutHashHint.siblingPath,
-      startBlobAccumulator: this.startBlobAccumulator.toBlobAccumulator(),
-      finalBlobChallenges: this.finalBlobBatchingChallenges,
-      blobFields: padArrayEnd(blobFields, Fr.ZERO, FIELDS_PER_BLOB * BLOBS_PER_CHECKPOINT),
-      blobCommitments: padArrayEnd(blobCommitments, BLS12Point.ZERO, BLOBS_PER_CHECKPOINT),
-      blobsHash,
-    });
-
-    const [left, right] = nonEmptyProofs.map(p => toProofData(p));
-
-    return !right
-      ? new CheckpointRootSingleBlockRollupPrivateInputs(left, hints)
-      : new CheckpointRootRollupPrivateInputs([left, right], hints);
-  }
-
   public getBlockProvingStateByBlockNumber(blockNumber: BlockNumber) {
     const index = Number(blockNumber) - Number(this.firstBlockNumber);
     return this.blocks[index];
@@ -317,13 +172,8 @@ export class CheckpointProvingState {
     return !!this.blockProofs.getSibling(location)?.provingOutput;
   }
 
-  public isReadyForCheckpointRoot() {
-    const allChildProofsReady = this.#getChildProofsForRoot().every(p => !!p);
-    return allChildProofsReady && !!this.previousOutHashHint && !!this.startBlobAccumulator;
-  }
-
   public verifyState() {
-    return this.parentEpoch.verifyState();
+    return this.isAlive();
   }
 
   public getError() {
@@ -337,13 +187,22 @@ export class CheckpointProvingState {
 
   public reject(reason: string) {
     this.error = reason;
-    this.parentEpoch.reject(reason);
+    this.onReject(reason);
   }
 
-  #getChildProofsForRoot() {
+  /**
+   * Returns the block-level proof outputs that feed into the checkpoint root rollup.
+   * Used by `CheckpointSubTreeOrchestrator` to surface its sub-tree result.
+   */
+  public getSubTreeOutputProofs() {
     const rootLocation = { level: 0, index: 0 };
     return this.totalNumBlocks === 1
       ? [this.blockProofs.getNode(rootLocation)?.provingOutput] // If there's only 1 block, its proof will be stored at the root.
       : this.blockProofs.getChildren(rootLocation).map(c => c?.provingOutput);
   }
+
+  /** Sibling path of the archive tree captured before any block in this checkpoint landed. */
+  public getLastArchiveSiblingPath() {
+    return this.lastArchiveSiblingPath;
+  }
 }