@aztec-labs/foundation 6.0.0-nightly.20260829

package/src/trees/merkle_tree.ts

@@ -0,0 +1,75 @@
+/** A simple immutable Merkle tree container. Use a MerkleTreeCalculator to create a new instance from a set of leaves. */
+export class MerkleTree {
+  constructor(
+    public readonly height: number,
+    public readonly nodes: Buffer[],
+  ) {
+    const expectedNodeCount = 2 ** (height + 1) - 1;
+    if (nodes.length !== expectedNodeCount) {
+      throw new Error(`Invalid node count for Merkle tree: got ${nodes.length} but expected ${expectedNodeCount}`);
+    }
+  }
+
+  get root(): Buffer {
+    return this.nodes[this.nodes.length - 1];
+  }
+
+  get leaves(): Buffer[] {
+    return this.nodes.slice(0, 2 ** this.height);
+  }
+
+  /** Returns a sibling path to the given element or to the element in the given index. */
+  public getSiblingPath(leafIndex: number): Buffer[];
+  public getSiblingPath(leaf: Buffer): Buffer[];
+  public getSiblingPath(leafIndexOrLeaf: number | Buffer): Buffer[] {
+    if (Buffer.isBuffer(leafIndexOrLeaf)) {
+      return this.getSiblingPath(this.getIndex(leafIndexOrLeaf));
+    }
+    const leafIndex = leafIndexOrLeaf;
+    if (leafIndex < 0 || leafIndex >= 2 ** this.height) {
+      throw new Error(`Invalid leaf index: got ${leafIndex} but leaves count is ${2 ** this.height}`);
+    }
+    const tree = this.nodes;
+    let rowSize = Math.ceil(tree.length / 2);
+    let rowOffset = 0;
+    let index = leafIndex;
+    const siblingPath: Buffer[] = [];
+    while (rowSize > 1) {
+      const isRight = index & 1;
+      siblingPath.push(tree[rowOffset + index + (isRight ? -1 : 1)]);
+      rowOffset += rowSize;
+      rowSize >>= 1;
+      index >>= 1;
+    }
+    return siblingPath;
+  }
+
+  /** Returns the leaf index for a given element. */
+  public getIndex(element: Buffer) {
+    return this.leaves.findIndex(leaf => leaf.equals(element));
+  }
+
+  /** Returns a nice string representation of the tree, useful for debugging purposes. */
+  public drawTree(elemSize = 8) {
+    const levels: string[][] = [];
+    const tree = this.nodes;
+    const maxRowSize = Math.ceil(tree.length / 2);
+    let paddingSize = 1;
+    let rowSize = maxRowSize;
+    let rowOffset = 0;
+    while (rowSize > 0) {
+      levels.push(
+        tree
+          .slice(rowOffset, rowOffset + rowSize)
+          .map(n => n.toString('hex').slice(0, elemSize) + ' '.repeat((paddingSize - 1) * (elemSize + 1))),
+      );
+      rowOffset += rowSize;
+      paddingSize <<= 1;
+      rowSize >>= 1;
+    }
+    return levels
+      .reverse()
+      .map(row => row.join(' '))
+      .join('\n');
+  }
+}

package/src/trees/merkle_tree_calculator.ts

@@ -0,0 +1,66 @@
+import { MerkleTree } from './merkle_tree.js';
+
+type MerkleHashFn = (left: Buffer, right: Buffer) => Promise<Buffer<ArrayBuffer>>;
+
+/**
+ * Merkle tree calculator.
+ */
+export class MerkleTreeCalculator {
+  private constructor(
+    private height: number,
+    private zeroHashes: Buffer[],
+    private hasher: MerkleHashFn,
+  ) {
+    this.hasher = hasher;
+  }
+
+  static async create(height: number, zeroLeaf: Buffer = Buffer.alloc(32), hasher: MerkleHashFn) {
+    const zeroHashes = [zeroLeaf];
+    for (let i = 0; i < height; i++) {
+      zeroHashes.push((await hasher(zeroHashes[i], zeroHashes[i])) as Buffer<ArrayBuffer>);
+    }
+    return new MerkleTreeCalculator(height, zeroHashes, hasher);
+  }
+
+  async computeTree(leaves: Buffer[] = []): Promise<MerkleTree> {
+    if (leaves.length === 0) {
+      leaves = new Array(2 ** this.height).fill(this.zeroHashes[0]);
+    }
+
+    let result = leaves.slice();
+
+    for (let i = 0; i < this.height; ++i) {
+      const numLeaves = 2 ** (this.height - i);
+      const newLeaves: Buffer[] = [];
+      for (let j = 0; j < leaves.length / 2; ++j) {
+        const l = leaves[j * 2];
+        const r = leaves[j * 2 + 1] || this.zeroHashes[i];
+        newLeaves[j] = await this.hasher(l, r);
+      }
+      result = result.concat(new Array(numLeaves - leaves.length).fill(this.zeroHashes[i]), newLeaves);
+      leaves = newLeaves;
+    }
+
+    return new MerkleTree(this.height, result);
+  }
+
+  async computeTreeRoot(leaves: Buffer[] = []): Promise<Buffer> {
+    if (leaves.length === 0) {
+      return this.zeroHashes[this.zeroHashes.length - 1];
+    }
+
+    leaves = leaves.slice();
+
+    for (let i = 0; i < this.height; ++i) {
+      let j = 0;
+      for (; j < leaves.length / 2; ++j) {
+        const l = leaves[j * 2];
+        const r = leaves[j * 2 + 1] || this.zeroHashes[i];
+        leaves[j] = await this.hasher(l, r);
+      }
+      leaves = leaves.slice(0, j);
+    }
+
+    return leaves[0];
+  }
+}

package/src/trees/sibling_path.ts

@@ -0,0 +1,177 @@
+import { z } from 'zod';
+
+import { makeTuple } from '../array/array.js';
+import { Fr } from '../curves/bn254/index.js';
+import { schemas } from '../schemas/index.js';
+import { BufferReader, type Tuple, assertLength, serializeArrayOfBufferableToVector } from '../serialize/index.js';
+import { bufferToHex, hexToBuffer } from '../string/index.js';
+import type { Hasher } from './hasher.js';
+
+/**
+ * Upper bound on the elements a serialized sibling path may declare. The deepest protocol tree is 42 levels,
+ * and the stacked path proving L2-to-L1 message inclusion spans four unbalanced trees, so this leaves room to
+ * spare while keeping a malformed length prefix from driving a large allocation. `stdlib` asserts that it stays
+ * above every protocol tree height, which this package cannot check itself without depending on `@aztec-labs/constants`.
+ */
+export const MAX_SIBLING_PATH_LENGTH = 128;
+
+/**
+ * Contains functionality to compute and serialize/deserialize a sibling path.
+ * E.g. Sibling path for a leaf at index 3 in a tree of depth 3 consists of:
+ *      d0:                                            [ root ]
+ *      d1:                      [ ]                                               [*]
+ *      d2:         [*]                      [ ]                       [ ]                     [ ]
+ *      d3:   [ ]         [ ]          [*]         [ ]           [ ]         [ ]          [ ]        [ ].
+ *
+ *      And the elements would be ordered as: [ leaf_at_index_2, node_at_level_2_index_0, node_at_level_1_index_1 ].
+ */
+export class SiblingPath<N extends number> {
+  private data: Tuple<Buffer, N>;
+
+  /**
+   * Constructor.
+   * @param pathSize - The size of the sibling path.
+   * @param path - The sibling path data.
+   */
+  constructor(
+    /** Size of the sibling path (number of fields it contains). */
+    public pathSize: N,
+    /** The sibling path data. */
+    path: Buffer[],
+  ) {
+    this.data = assertLength(path, pathSize);
+  }
+
+  static get schema() {
+    return schemas.Buffer.transform(b => SiblingPath.fromBuffer(b));
+  }
+
+  static schemaFor<N extends number>(size: N) {
+    return schemas.Buffer.transform(b => SiblingPath.fromBuffer(b) as SiblingPath<N>).superRefine((path, ctx) => {
+      if (path.pathSize !== size) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Expected sibling path size ${size} but got ${path.pathSize}`,
+        });
+      }
+    });
+  }
+
+  toJSON() {
+    return this.toBuffer();
+  }
+
+  /**
+   * Returns sibling path hashed up from the a element.
+   * @param size - The number of elements in a given path.
+   * @param zeroElement - Value of the zero element.
+   * @param hasher - Implementation of a hasher interface.
+   * @returns A sibling path hashed up from a zero element.
+   */
+  public static ZERO<N extends number>(size: N, zeroElement: Buffer, hasher: Hasher): SiblingPath<N> {
+    const bufs: Buffer[] = [];
+    let current = zeroElement;
+    for (let i = 0; i < size; ++i) {
+      bufs.push(current);
+      current = hasher.hash(current, current);
+    }
+    return new SiblingPath(size, bufs);
+  }
+
+  static random<N extends number>(number: N) {
+    const data = Array.from({ length: number }, () => Fr.random().toBuffer());
+    return new SiblingPath(number, data);
+  }
+
+  /**
+   * Serializes this SiblingPath object to a buffer.
+   * @returns The buffer representation of this object.
+   */
+  public toBuffer(): Buffer {
+    return serializeArrayOfBufferableToVector(this.data);
+  }
+
+  /**
+   * Returns the path buffer underlying the sibling path.
+   * @returns The Buffer array representation of this object.
+   */
+  public toBufferArray(): Buffer[] {
+    return this.data;
+  }
+
+  /**
+   * Convert the Sibling Path object into an array of field elements.
+   * @returns The field array representation of this object.
+   */
+  public toFields(): Fr[] {
+    return this.data.map(buf => Fr.fromBuffer(buf));
+  }
+
+  /**
+   * Convert Sibling Path object into a tuple of field elements.
+   * @returns A tuple representation of the sibling path.
+   */
+  public toTuple(): Tuple<Fr, N> {
+    const array = this.toFields();
+    return makeTuple(array.length as N, i => array[i], 0);
+  }
+
+  /**
+   * Deserializes a SiblingPath from a buffer.
+   * @param buf - A buffer containing the buffer representation of SiblingPath.
+   * @param offset - An offset to start deserializing from.
+   * @returns A SiblingPath object.
+   * @throws If the length prefix exceeds MAX_SIBLING_PATH_LENGTH or the buffer holds fewer elements than it declares.
+   */
+  static fromBuffer<N extends number>(buf: Buffer, offset = 0): SiblingPath<N> {
+    const reader = new BufferReader(buf, offset);
+    const path = reader.readVector({ fromBuffer: r => r.readBytes(Fr.SIZE_IN_BYTES) }, MAX_SIBLING_PATH_LENGTH);
+    return new SiblingPath<N>(path.length as N, path);
+  }
+
+  /**
+   * Serializes this SiblingPath object to a hex string representation.
+   * @returns A hex string representation of the sibling path.
+   */
+  public toString(): string {
+    return bufferToHex(this.toBuffer());
+  }
+
+  /**
+   * Deserializes a SiblingPath object from a hex string representation.
+   * @param repr - A hex string representation of the sibling path.
+   * @returns A SiblingPath object.
+   */
+  public static fromString<N extends number>(repr: string): SiblingPath<N> {
+    return SiblingPath.fromBuffer<N>(hexToBuffer(repr));
+  }
+
+  /**
+   * Generate a subtree path from the current sibling path.
+   * @param subtreeHeight - The size of the subtree that we are getting the path for.
+   * @returns A new sibling path that is the for the requested subtree.
+   */
+  public getSubtreeSiblingPath<SubtreeHeight extends number, SubtreeSiblingPathHeight extends number>(
+    subtreeHeight: SubtreeHeight,
+  ): SiblingPath<SubtreeSiblingPathHeight> {
+    // Drop the size of the subtree from the path, and return the rest.
+    const subtreeData = this.data.slice(subtreeHeight);
+    const subtreePathSize = (this.pathSize - subtreeHeight) as SubtreeSiblingPathHeight;
+    return new SiblingPath(subtreePathSize, subtreeData);
+  }
+}
+
+/** Computes the expected root of a merkle tree given a leaf and its sibling path. */
+export async function computeRootFromSiblingPath(
+  leaf: Buffer,
+  siblingPath: Buffer[],
+  index: number,
+  hasher: (left: Buffer, right: Buffer) => Promise<Buffer>,
+) {
+  let result = leaf;
+  for (const sibling of siblingPath) {
+    result = index & 1 ? await hasher(sibling, result) : await hasher(result, sibling);
+    index >>= 1;
+  }
+  return result;
+}

package/src/trees/unbalanced_merkle_tree_calculator.ts

@@ -0,0 +1,176 @@
+import { type Hasher, shaMerkleHash } from './hasher.js';
+import { SiblingPath } from './sibling_path.js';
+import { type TreeNodeLocation, UnbalancedTreeStore } from './unbalanced_tree_store.js';
+
+interface TreeNode {
+  value: Buffer;
+  leafIndex?: number;
+}
+
+/**
+ * An ephemeral unbalanced Merkle tree implementation.
+ * Follows the rollup implementation which greedily hashes pairs of nodes up the tree.
+ * Remaining rightmost nodes are shifted up until they can be paired.
+ * The values that match the `valueToCompress` are skipped and the sibling of the compressed leaf are shifted up until
+ * they can be paired.
+ * If there is only one leaf, the root is the leaf.
+ */
+export class UnbalancedMerkleTreeCalculator {
+  private store: UnbalancedTreeStore<TreeNode>;
+  private leafLocations: TreeNodeLocation[] = [];
+
+  public constructor(
+    private readonly leaves: Buffer[],
+    private readonly valueToCompress: Buffer,
+    private readonly emptyRoot: Buffer,
+    private readonly hasher: Hasher['hash'],
+  ) {
+    this.store = new UnbalancedTreeStore(leaves.length);
+    this.buildTree();
+  }
+
+  static create(
+    leaves: Buffer[],
+    valueToCompress = Buffer.alloc(0),
+    emptyRoot = Buffer.alloc(32),
+    hasher = shaMerkleHash,
+  ) {
+    return new UnbalancedMerkleTreeCalculator(leaves, valueToCompress, emptyRoot, hasher);
+  }
+
+  /**
+   * Returns the root of the tree.
+   * @returns The root of the tree.
+   */
+  public getRoot(): Buffer {
+    return this.store.getRoot()!.value;
+  }
+
+  /**
+   * Returns a sibling path for the element.
+   * @param value - The value of the element.
+   * @returns A sibling path for the element.
+   * Note: The sibling path is an array of sibling hashes, with the lowest hash (leaf hash) first, and the highest hash last.
+   */
+  public getSiblingPath<N extends number>(value: Buffer): SiblingPath<N> {
+    const leafIndex = this.leaves.findIndex(leaf => leaf.equals(value));
+    if (leafIndex === -1) {
+      throw Error(`Leaf value ${value.toString('hex')} not found in tree.`);
+    }
+
+    return this.getSiblingPathByLeafIndex(leafIndex);
+  }
+
+  /**
+   * Returns a sibling path for the leaf at the given index.
+   * @param leafIndex - The index of the leaf.
+   * @returns A sibling path for the leaf.
+   */
+  public getSiblingPathByLeafIndex<N extends number>(leafIndex: number): SiblingPath<N> {
+    if (leafIndex >= this.leaves.length) {
+      throw Error(`Leaf index ${leafIndex} out of bounds. Tree has ${this.leaves.length} leaves.`);
+    }
+
+    const leaf = this.leaves[leafIndex];
+    if (leaf.equals(this.valueToCompress)) {
+      throw Error(`Leaf at index ${leafIndex} has been compressed.`);
+    }
+
+    const path: Buffer[] = [];
+    let location = this.leafLocations[leafIndex];
+    while (location.level > 0) {
+      const sibling = this.store.getSibling(location)!;
+      path.push(sibling.value);
+      location = this.store.getParentLocation(location);
+    }
+
+    return new SiblingPath<N>(path.length as N, path);
+  }
+
+  public getLeafLocation(leafIndex: number) {
+    return this.leafLocations[leafIndex];
+  }
+
+  /**
+   * Adds leaves and nodes to the store. Updates the leafLocations.
+   * @param leaves - The leaves of the tree.
+   */
+  private buildTree() {
+    this.leafLocations = this.leaves.map((value, i) => this.store.setLeaf(i, { value, leafIndex: i }));
+
+    // Start with the leaves that are not compressed.
+    let toProcess = this.leafLocations.filter((_, i) => !this.leaves[i].equals(this.valueToCompress));
+    if (!toProcess.length) {
+      // All leaves are compressed. Set empty root to the root.
+      this.store.setNode({ level: 0, index: 0 }, { value: this.emptyRoot });
+      return;
+    }
+
+    const level = toProcess[0].level;
+    for (let i = level; i > 0; i--) {
+      const toProcessNext = [];
+      for (const location of toProcess) {
+        if (location.level !== i) {
+          toProcessNext.push(location);
+          continue;
+        }
+
+        const parentLocation = this.store.getParentLocation(location);
+        if (this.store.getNode(parentLocation)) {
+          // Parent has been updated by its (left) sibling.
+          continue;
+        }
+
+        const sibling = this.store.getSibling(location);
+        // If sibling is undefined, all its children are compressed.
+        const shouldShiftUp = !sibling || sibling.value.equals(this.valueToCompress);
+        if (shouldShiftUp) {
+          // The node becomes the parent if the sibling is a compressed leaf.
+          this.shiftNodeUp(location, parentLocation);
+        } else {
+          // Hash the value with the (right) sibling and update the parent node.
+          const node = this.store.getNode(location)!;
+          const parentValue = this.hasher(node.value, sibling.value);
+          this.store.setNode(parentLocation, { value: parentValue });
+        }
+
+        // Add the parent location to be processed next.
+        toProcessNext.push(parentLocation);
+      }
+
+      toProcess = toProcessNext;
+    }
+  }
+
+  private shiftNodeUp(from: TreeNodeLocation, to: TreeNodeLocation) {
+    // Collect all nodes that need to shift.
+    const nodesToShift = this.collectNodesToShift(from, to);
+    // Move all nodes to their new locations.
+    for (const { node, newLocation } of nodesToShift) {
+      this.store.setNode(newLocation, node);
+      if (node.leafIndex !== undefined) {
+        this.leafLocations[node.leafIndex] = newLocation;
+      }
+    }
+  }
+
+  private collectNodesToShift(from: TreeNodeLocation, to: TreeNodeLocation) {
+    const node = this.store.getNode(from);
+    if (!node) {
+      return [];
+    }
+
+    let result = [{ node, newLocation: to }];
+
+    // If not a leaf, collect its children.
+    if (node.leafIndex === undefined) {
+      const [leftChild, rightChild] = this.store.getChildLocations(from);
+      const [leftChildNew, rightChildNew] = this.store.getChildLocations(to);
+      result = result
+        .concat(this.collectNodesToShift(leftChild, leftChildNew))
+        .concat(this.collectNodesToShift(rightChild, rightChildNew));
+    }
+
+    return result;
+  }
+}

package/src/trees/unbalanced_merkle_tree_root.ts

@@ -0,0 +1,112 @@
+import { computeBalancedMerkleTreeRoot, computeBalancedMerkleTreeRootAsync } from './balanced_merkle_tree_root.js';
+import { shaMerkleHash } from './hasher.js';
+import { UnbalancedMerkleTreeCalculator } from './unbalanced_merkle_tree_calculator.js';
+
+export const computeUnbalancedShaRoot = (leaves: Buffer[]) => computeUnbalancedMerkleTreeRoot(leaves, shaMerkleHash);
+
+export const computeWonkyShaRoot = (leaves: Buffer[]) => computeWonkyMerkleTreeRoot(leaves);
+
+/**
+ * Computes the Merkle root of an unbalanced tree.
+ *
+ * Unlike a balanced Merkle tree, which requires the number of leaves to be a power of two, an unbalanced tree can have
+ * any number of leaves.
+ *
+ * The tree is constructed by iteratively extracting the smallest power-of-two-sized subtrees from **right to left**.
+ * For each such subtree, it computes the subtree root and then combines all subtree roots (again from right to left)
+ * into a single root using the provided hash function.
+ *
+ * Note: We need the final tree to be as shallow as possible, to minimize the size of the sibling path required to prove
+ * membership of a leaf. Therefor, the computation proceeds from right to left - smaller subtrees must always be
+ * combined before being merged with a larger sibling on their left.
+ *
+ * For example, consider an unbalanced tree made of three subtrees of sizes 2, 4, and 8. If we combine the size-2 and
+ * size-4 subtrees first (producing a subtree of depth 3), and then merge it with the size-8 subtree (also depth 3), the
+ * resulting tree has a maximum depth of 4.
+ *
+ * But if we instead combine the size-4 and size-8 subtrees first (depth 4), and then merge with the size-2 subtree
+ * (depth 1), the final tree has a depth of 5.
+ */
+export function computeUnbalancedMerkleTreeRoot(
+  leaves: Buffer[],
+  hasher = shaMerkleHash,
+  emptyRoot = Buffer.alloc(32),
+): Buffer {
+  if (!leaves.length) {
+    return emptyRoot;
+  }
+
+  if (leaves.length === 1) {
+    return leaves[0];
+  }
+
+  let numRemainingLeaves = leaves.length;
+  let subtreeSize = 1;
+  let root: Buffer | undefined;
+  while (numRemainingLeaves > 1) {
+    if ((numRemainingLeaves & subtreeSize) !== 0) {
+      const subtreeLeaves = leaves.slice(numRemainingLeaves - subtreeSize, numRemainingLeaves);
+      const subtreeRoot = computeBalancedMerkleTreeRoot(subtreeLeaves, hasher);
+      if (!root) {
+        root = subtreeRoot;
+      } else {
+        root = hasher(subtreeRoot, root);
+      }
+
+      numRemainingLeaves -= subtreeSize;
+    }
+
+    subtreeSize *= 2;
+  }
+
+  return root!;
+}
+
+export async function computeUnbalancedMerkleTreeRootAsync(
+  leaves: Buffer[],
+  hasher: (left: Uint8Array, right: Uint8Array) => Promise<Buffer<ArrayBuffer>>,
+  emptyRoot = Buffer.alloc(32),
+): Promise<Buffer> {
+  if (!leaves.length) {
+    return emptyRoot;
+  }
+
+  if (leaves.length === 1) {
+    return leaves[0];
+  }
+
+  let numRemainingLeaves = leaves.length;
+  let subtreeSize = 1;
+  let root: Buffer | undefined;
+  while (numRemainingLeaves > 1) {
+    if ((numRemainingLeaves & subtreeSize) !== 0) {
+      const subtreeLeaves = leaves.slice(numRemainingLeaves - subtreeSize, numRemainingLeaves);
+      const subtreeRoot = await computeBalancedMerkleTreeRootAsync(subtreeLeaves, hasher);
+      if (!root) {
+        root = subtreeRoot;
+      } else {
+        root = await hasher(subtreeRoot, root);
+      }
+
+      numRemainingLeaves -= subtreeSize;
+    }
+
+    subtreeSize *= 2;
+  }
+
+  return root!;
+}
+
+// A **wonky** tree is a "compressed" unbalanced Merkle tree.
+// It is constructed in the same way as an unbalanced tree: by first creating the largest possible left subtree, with
+// the remaining leaves forming a right subtree that follows the same process recursively.
+// During construction, leaves equal to `valueToCompress` are skipped (compressed) and do not contribute to the tree.
+export function computeWonkyMerkleTreeRoot(
+  leaves: Buffer[],
+  valueToCompress = Buffer.alloc(32),
+  emptyRoot = Buffer.alloc(32),
+  hasher = shaMerkleHash,
+): Buffer {
+  const calculator = UnbalancedMerkleTreeCalculator.create(leaves, valueToCompress, emptyRoot, hasher);
+  return calculator.getRoot();
+}