@aztec-labs/prover-client 6.0.0-nightly.20260829

package/src/light/lightweight_checkpoint_builder.ts

@@ -0,0 +1,342 @@
+import {
+  SpongeBlob,
+  computeBlobsHashFromBlobs,
+  encodeCheckpointEndMarker,
+  getBlobsPerL1Block,
+} from '@aztec-labs/blob-lib';
+import { type CheckpointNumber, IndexWithinCheckpoint } from '@aztec-labs/foundation/branded-types';
+import { Fr } from '@aztec-labs/foundation/curves/bn254';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec-labs/foundation/log';
+import { elapsed } from '@aztec-labs/foundation/timer';
+import { L2Block } from '@aztec-labs/stdlib/block';
+import { Checkpoint } from '@aztec-labs/stdlib/checkpoint';
+import type { MerkleTreeWriteOperations } from '@aztec-labs/stdlib/interfaces/server';
+import {
+  accumulateCheckpointOutHashes,
+  accumulateInboxRollingHash,
+  appendL1ToL2MessagesToTree,
+  computeCheckpointOutHash,
+} from '@aztec-labs/stdlib/messaging';
+import { CheckpointHeader, computeBlockHeadersHash } from '@aztec-labs/stdlib/rollup';
+import { AppendOnlyTreeSnapshot, MerkleTreeId } from '@aztec-labs/stdlib/trees';
+import {
+  type CheckpointGlobalVariables,
+  type GlobalVariables,
+  type ProcessedTx,
+  StateReference,
+} from '@aztec-labs/stdlib/tx';
+
+import {
+  buildHeaderAndBodyFromTxs,
+  getTreeSnapshot,
+  insertSideEffects,
+} from '../orchestrator/block-building-helpers.js';
+
+/**
+ * Builds a checkpoint and its header and the blocks in it from a set of processed tx without running any circuits.
+ *
+ * Each added block inserts its own L1-to-L2 message bundle into the message tree and then updates the archive tree.
+ * Finally completes the checkpoint by computing its header.
+ */
+export class LightweightCheckpointBuilder {
+  private readonly logger: Logger;
+
+  private lastArchives: AppendOnlyTreeSnapshot[] = [];
+  private spongeBlob: SpongeBlob;
+  private blocks: L2Block[] = [];
+  private blobFields: Fr[] = [];
+
+  constructor(
+    public readonly checkpointNumber: CheckpointNumber,
+    public readonly constants: CheckpointGlobalVariables,
+    public feeAssetPriceModifier: bigint,
+    public readonly l1ToL2Messages: Fr[],
+    private readonly previousCheckpointOutHashes: Fr[],
+    // Inbox rolling hash of the previous checkpoint (this checkpoint's chain start); genesis is zero.
+    private readonly previousInboxRollingHash: Fr,
+    public readonly db: MerkleTreeWriteOperations,
+    bindings?: LoggerBindings,
+  ) {
+    this.logger = createLogger('checkpoint-builder', {
+      ...bindings,
+      instanceId: `checkpoint-${checkpointNumber}`,
+    });
+    this.spongeBlob = SpongeBlob.init();
+    this.logger.debug('Starting new checkpoint', { constants, l1ToL2Messages, feeAssetPriceModifier });
+  }
+
+  /**
+   * Starts a fresh checkpoint. The checkpoint's L1-to-L2 messages are not supplied here: every block brings its own
+   * bundle to {@link addBlock}, which inserts it into the tree and accumulates it into the checkpoint's message list.
+   */
+  static startNewCheckpoint(
+    checkpointNumber: CheckpointNumber,
+    constants: CheckpointGlobalVariables,
+    previousCheckpointOutHashes: Fr[],
+    previousInboxRollingHash: Fr,
+    db: MerkleTreeWriteOperations,
+    bindings?: LoggerBindings,
+    feeAssetPriceModifier: bigint = 0n,
+  ): LightweightCheckpointBuilder {
+    return new LightweightCheckpointBuilder(
+      checkpointNumber,
+      constants,
+      feeAssetPriceModifier,
+      [],
+      previousCheckpointOutHashes,
+      previousInboxRollingHash,
+      db,
+      bindings,
+    );
+  }
+
+  /**
+   * Resumes building a checkpoint from existing blocks. This is used for validator re-execution
+   * where blocks have already been built and their effects are already in the database.
+   * `l1ToL2Messages` is the whole checkpoint's message list as consumed by the existing blocks: it seeds the
+   * checkpoint's rolling hash and is not inserted into the tree, since the blocks already inserted it.
+   */
+  static async resumeCheckpoint(
+    checkpointNumber: CheckpointNumber,
+    constants: CheckpointGlobalVariables,
+    feeAssetPriceModifier: bigint,
+    l1ToL2Messages: Fr[],
+    previousCheckpointOutHashes: Fr[],
+    previousInboxRollingHash: Fr,
+    db: MerkleTreeWriteOperations,
+    existingBlocks: L2Block[],
+    bindings?: LoggerBindings,
+  ): Promise<LightweightCheckpointBuilder> {
+    const builder = new LightweightCheckpointBuilder(
+      checkpointNumber,
+      constants,
+      feeAssetPriceModifier,
+      l1ToL2Messages,
+      previousCheckpointOutHashes,
+      previousInboxRollingHash,
+      db,
+      bindings,
+    );
+
+    builder.logger.debug('Resuming checkpoint from existing blocks', {
+      checkpointNumber,
+      numExistingBlocks: existingBlocks.length,
+      blockNumbers: existingBlocks.map(b => b.header.getBlockNumber()),
+    });
+
+    if (existingBlocks.length === 0) {
+      throw new Error(`Cannot resume checkpoint ${checkpointNumber} with no existing blocks`);
+    }
+
+    // Validate block order and consistency
+    for (let i = 1; i < existingBlocks.length; i++) {
+      const prev = existingBlocks[i - 1];
+      const curr = existingBlocks[i];
+      if (curr.number !== prev.number + 1) {
+        throw new Error(`Non-sequential block numbers in resumeCheckpoint: ${prev.number} -> ${curr.number}`);
+      }
+      if (!prev.archive.root.equals(curr.header.lastArchive.root)) {
+        throw new Error(`Archive root mismatch between blocks ${prev.number} and ${curr.number}`);
+      }
+    }
+
+    for (let i = 0; i < existingBlocks.length; i++) {
+      const block = existingBlocks[i];
+      const isFirstBlock = i === 0;
+
+      if (isFirstBlock) {
+        builder.lastArchives.push(block.header.lastArchive);
+      }
+
+      builder.lastArchives.push(block.archive);
+
+      const blockBlobFields = block.toBlobFields();
+      await builder.spongeBlob.absorb(blockBlobFields);
+      builder.blobFields.push(...blockBlobFields);
+
+      builder.blocks.push(block);
+    }
+
+    return builder;
+  }
+
+  /** Returns how many blocks have been added to this checkpoint so far */
+  public getBlockCount() {
+    return this.blocks.length;
+  }
+
+  public getBlocks() {
+    return this.blocks;
+  }
+
+  /**
+   * Adds a new block to the checkpoint. The tx effects must have already been inserted into the db if
+   * this is called after tx processing, if that's not the case, then set `insertTxsEffects` to true.
+   * @param l1ToL2Messages - The message leaves this block consumes from the Inbox, in insertion order.
+   */
+  public async addBlock(
+    globalVariables: GlobalVariables,
+    txs: ProcessedTx[],
+    l1ToL2Messages: Fr[],
+    opts: { insertTxsEffects?: boolean; expectedEndState?: StateReference } = {},
+  ): Promise<{ block: L2Block; timings: Record<string, number> }> {
+    const timings: Record<string, number> = {};
+    const isFirstBlock = this.blocks.length === 0;
+
+    if (isFirstBlock) {
+      const [msGetInitialArchive, initialArchive] = await elapsed(() => getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db));
+      this.lastArchives.push(initialArchive);
+      timings.getInitialArchive = msGetInitialArchive;
+    }
+
+    const lastArchive = this.lastArchives.at(-1)!;
+
+    if (opts.insertTxsEffects) {
+      this.logger.debug(
+        `Inserting side effects for ${txs.length} txs for block ${globalVariables.blockNumber} into db`,
+        { txs: txs.map(tx => tx.hash.toString()) },
+      );
+      let msInsertSideEffects = 0;
+      for (const tx of txs) {
+        const [ms] = await elapsed(() => insertSideEffects(tx, this.db));
+        msInsertSideEffects += ms;
+      }
+      timings.insertSideEffects = msInsertSideEffects;
+    }
+
+    // Streaming Inbox: insert this block's L1-to-L2 message bundle before reading the end state,
+    // so the block header's L1-to-L2 tree snapshot reflects it. Bundles are appended compactly (unpadded, at the
+    // tree's current next-available index). The logical messages are accumulated only once the block is fully built
+    // (below), so a mid-build failure does not pollute the checkpoint's rolling hash; the rolling hash is recomputed
+    // over them at checkpoint completion.
+    await appendL1ToL2MessagesToTree(this.db, l1ToL2Messages);
+
+    const [msGetEndState, endState] = await elapsed(() => this.db.getStateReference());
+    timings.getEndState = msGetEndState;
+
+    if (opts.expectedEndState && !endState.equals(opts.expectedEndState)) {
+      this.logger.error('End state after processing txs does not match expected end state', {
+        globalVariables: globalVariables.toInspect(),
+        expectedEndState: opts.expectedEndState.toInspect(),
+        actualEndState: endState.toInspect(),
+      });
+      throw new Error(`End state does not match expected end state when building block ${globalVariables.blockNumber}`);
+    }
+
+    const [msBuildHeaderAndBody, { header, body, blockBlobFields }] = await elapsed(() =>
+      buildHeaderAndBodyFromTxs(txs, lastArchive, endState, globalVariables, this.spongeBlob),
+    );
+    timings.buildHeaderAndBody = msBuildHeaderAndBody;
+
+    header.state.validate();
+
+    await this.db.updateArchive(header);
+    const [msUpdateArchive, newArchive] = await elapsed(() => getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db));
+    timings.updateArchive = msUpdateArchive;
+    this.lastArchives.push(newArchive);
+
+    const expectedNextLeafIndex = Number(globalVariables.blockNumber) + 1;
+    if (newArchive.nextAvailableLeafIndex !== expectedNextLeafIndex) {
+      throw new Error(
+        `Archive tree next leaf index mismatch after building block ${globalVariables.blockNumber} (expected ${expectedNextLeafIndex} but got ${newArchive.nextAvailableLeafIndex})`,
+      );
+    }
+
+    const indexWithinCheckpoint = IndexWithinCheckpoint(this.blocks.length);
+    const block = new L2Block(newArchive, header, body, this.checkpointNumber, indexWithinCheckpoint);
+    this.blocks.push(block);
+
+    // Accumulate the streaming bundle now that the block is fully built, so a mid-build throw above leaves the
+    // checkpoint's message list (and thus its rolling hash) consistent with the blocks actually built.
+    this.l1ToL2Messages.push(...l1ToL2Messages);
+
+    const [msSpongeAbsorb] = await elapsed(() => this.spongeBlob.absorb(blockBlobFields));
+    timings.spongeAbsorb = msSpongeAbsorb;
+    this.blobFields.push(...blockBlobFields);
+
+    this.logger.debug(`Built block ${header.getBlockNumber()}`, {
+      globalVariables: globalVariables.toInspect(),
+      archiveRoot: newArchive.root.toString(),
+      stateReference: header.state.toInspect(),
+      blockHash: (await block.hash()).toString(),
+      txs: block.body.txEffects.map(tx => tx.txHash.toString()),
+    });
+
+    return { block, timings };
+  }
+
+  async completeCheckpoint(): Promise<Checkpoint> {
+    if (!this.blocks.length) {
+      throw new Error('Cannot complete a checkpoint with no blocks');
+    }
+
+    const numBlobFields = this.blobFields.length + 1; // +1 for the checkpoint end marker.
+    const checkpointEndMarker = encodeCheckpointEndMarker({ numBlobFields });
+    this.blobFields.push(checkpointEndMarker);
+
+    const blocks = this.blocks;
+    const blockHeadersHash = await computeBlockHeadersHash(blocks.map(block => block.header));
+
+    const newArchive = this.lastArchives[this.lastArchives.length - 1];
+
+    const blobs = await getBlobsPerL1Block(this.blobFields);
+    const blobsHash = computeBlobsHashFromBlobs(blobs);
+
+    const inboxRollingHash = accumulateInboxRollingHash(this.previousInboxRollingHash, this.l1ToL2Messages);
+
+    const { slotNumber, coinbase, feeRecipient, gasFees } = this.constants;
+    const checkpointOutHash = computeCheckpointOutHash(
+      blocks.map(block => block.body.txEffects.map(tx => tx.l2ToL1Msgs)),
+    );
+    const epochOutHash = accumulateCheckpointOutHashes([...this.previousCheckpointOutHashes, checkpointOutHash]);
+
+    // All blocks in the checkpoint have the same timestamp
+    const timestamp = blocks[blocks.length - 1].timestamp;
+
+    const totalManaUsed = blocks.reduce((acc, block) => acc.add(block.header.totalManaUsed), Fr.ZERO);
+    const accumulatedFees = blocks.reduce((acc, block) => acc.add(block.header.totalFees), Fr.ZERO);
+
+    const header = CheckpointHeader.from({
+      lastArchiveRoot: this.lastArchives[0].root,
+      blobsHash,
+      inboxRollingHash,
+      epochOutHash,
+      blockHeadersHash,
+      slotNumber,
+      timestamp,
+      coinbase,
+      feeRecipient,
+      gasFees,
+      totalManaUsed,
+      accumulatedFees,
+    });
+
+    this.logger.debug(`Completed checkpoint ${this.checkpointNumber}`, {
+      checkpointNumber: this.checkpointNumber,
+      headerHash: header.hash().toString(),
+      checkpointOutHash: checkpointOutHash.toString(),
+      numPreviousCheckpointOutHashes: this.previousCheckpointOutHashes.length,
+      ...header.toInspect(),
+    });
+
+    return new Checkpoint(newArchive, header, blocks, this.checkpointNumber, this.feeAssetPriceModifier);
+  }
+
+  clone() {
+    const clone = new LightweightCheckpointBuilder(
+      this.checkpointNumber,
+      this.constants,
+      this.feeAssetPriceModifier,
+      [...this.l1ToL2Messages],
+      [...this.previousCheckpointOutHashes],
+      this.previousInboxRollingHash,
+      this.db,
+      this.logger.getBindings(),
+    );
+    clone.lastArchives = [...this.lastArchives];
+    clone.spongeBlob = this.spongeBlob.clone();
+    clone.blocks = [...this.blocks];
+    clone.blobFields = [...this.blobFields];
+    return clone;
+  }
+}

package/src/mocks/fixtures.ts

@@ -0,0 +1,120 @@
+import { BlockNumber, SlotNumber } from '@aztec-labs/foundation/branded-types';
+import { randomBytes } from '@aztec-labs/foundation/crypto/random';
+import { Fr } from '@aztec-labs/foundation/curves/bn254';
+import { EthAddress } from '@aztec-labs/foundation/eth-address';
+import type { Logger } from '@aztec-labs/foundation/log';
+import type { FieldsOf } from '@aztec-labs/foundation/types';
+import { fileURLToPath } from '@aztec-labs/foundation/url';
+import { getVKTreeRoot } from '@aztec-labs/noir-protocol-circuits-types/vk-tree';
+import { protocolContractsHash } from '@aztec-labs/protocol-contracts';
+import { type CircuitSimulator, NativeACVMSimulator, WASMSimulatorWithBlobs } from '@aztec-labs/simulator/server';
+import { AztecAddress } from '@aztec-labs/stdlib/aztec-address';
+import { GasFees } from '@aztec-labs/stdlib/gas';
+import { CheckpointConstantData } from '@aztec-labs/stdlib/rollup';
+import { GlobalVariables } from '@aztec-labs/stdlib/tx';
+import { promises as fs } from 'fs';
+import path from 'path';
+
+const {
+  TEMP_DIR = '/tmp',
+  BB_BINARY_PATH = '',
+  BB_WORKING_DIRECTORY = '',
+  BB_SKIP_CLEANUP = '',
+  ACVM_BINARY_PATH = '',
+  ACVM_WORKING_DIRECTORY = '',
+} = process.env;
+
+// Determines if we have access to the bb binary and a tmp folder for temp files
+export const getEnvironmentConfig = async (logger: Logger) => {
+  try {
+    const expectedBBPath = BB_BINARY_PATH
+      ? BB_BINARY_PATH
+      : `${path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../labs-aztec-toolchain/bin')}/bb-avm`;
+    await fs.access(expectedBBPath, fs.constants.R_OK);
+    const tempWorkingDirectory = `${TEMP_DIR}/${randomBytes(4).toString('hex')}`;
+    const bbWorkingDirectory = BB_WORKING_DIRECTORY ? BB_WORKING_DIRECTORY : `${tempWorkingDirectory}/bb`;
+    await fs.mkdir(bbWorkingDirectory, { recursive: true });
+    logger.info(`Found native BB binary at ${expectedBBPath} with working directory ${bbWorkingDirectory}`);
+
+    const expectedAcvmPath = ACVM_BINARY_PATH
+      ? ACVM_BINARY_PATH
+      : `${path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../labs-aztec-toolchain/bin')}/acvm`;
+    await fs.access(expectedAcvmPath, fs.constants.R_OK);
+    const acvmWorkingDirectory = ACVM_WORKING_DIRECTORY ? ACVM_WORKING_DIRECTORY : `${tempWorkingDirectory}/acvm`;
+    await fs.mkdir(acvmWorkingDirectory, { recursive: true });
+    logger.info(`Found native ACVM binary at ${expectedAcvmPath} with working directory ${acvmWorkingDirectory}`);
+
+    const bbSkipCleanup = ['1', 'true'].includes(BB_SKIP_CLEANUP);
+    bbSkipCleanup && logger.verbose(`Not going to clean up BB working directory ${bbWorkingDirectory} after run`);
+
+    return {
+      acvmWorkingDirectory,
+      bbWorkingDirectory,
+      expectedAcvmPath,
+      expectedBBPath,
+      directoryToCleanup: ACVM_WORKING_DIRECTORY && BB_WORKING_DIRECTORY ? undefined : tempWorkingDirectory,
+      bbSkipCleanup,
+    };
+  } catch (err) {
+    logger.info(`Native BB not available: ${err}`);
+    return undefined;
+  }
+};
+
+export async function getSimulator(
+  config: { acvmWorkingDirectory: string | undefined; acvmBinaryPath: string | undefined },
+  logger?: Logger,
+): Promise<CircuitSimulator> {
+  if (config.acvmBinaryPath && config.acvmWorkingDirectory) {
+    try {
+      await fs.access(config.acvmBinaryPath, fs.constants.R_OK);
+      await fs.mkdir(config.acvmWorkingDirectory, { recursive: true });
+      logger?.info(
+        `Using native ACVM at ${config.acvmBinaryPath} and working directory ${config.acvmWorkingDirectory}`,
+      );
+      const acvmLogger = logger?.createChild('acvm-native');
+      return new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath, undefined, acvmLogger);
+    } catch {
+      logger?.warn(`Failed to access ACVM at ${config.acvmBinaryPath}, falling back to WASM`);
+    }
+  }
+  logger?.info('Using WASM ACVM simulation');
+  return new WASMSimulatorWithBlobs();
+}
+
+export const makeGlobals = (
+  blockNumber: number,
+  slotNumber = blockNumber,
+  overrides: Partial<FieldsOf<GlobalVariables> & FieldsOf<CheckpointConstantData>> = {},
+) => {
+  const checkpointConstants = makeCheckpointConstants(slotNumber, overrides);
+  return GlobalVariables.from({
+    chainId: checkpointConstants.chainId,
+    version: checkpointConstants.version,
+    blockNumber: BlockNumber(blockNumber) /** block number */,
+    slotNumber: SlotNumber(slotNumber) /** slot number */,
+    timestamp: BigInt(blockNumber * 123) /** block number * 123 as pseudo-timestamp for testing */,
+    coinbase: checkpointConstants.coinbase,
+    feeRecipient: checkpointConstants.feeRecipient,
+    gasFees: checkpointConstants.gasFees,
+    ...overrides,
+  });
+};
+
+export const makeCheckpointConstants = (
+  slotNumber: number,
+  overrides: Partial<FieldsOf<CheckpointConstantData>> = {},
+) => {
+  return CheckpointConstantData.from({
+    chainId: Fr.ZERO,
+    version: Fr.ZERO,
+    vkTreeRoot: getVKTreeRoot(),
+    protocolContractsHash,
+    proverId: Fr.ZERO,
+    slotNumber: SlotNumber(slotNumber),
+    coinbase: EthAddress.ZERO,
+    feeRecipient: AztecAddress.ZERO,
+    gasFees: GasFees.empty(),
+    ...overrides,
+  });
+};