@aztec/merkle-tree 0.23.0 → 0.24.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/merkle-tree",
-  "version": "0.23.0",
+  "version": "0.24.0",
   "type": "module",
   "exports": "./dest/index.js",
   "typedocOptions": {
@@ -32,10 +32,10 @@
     "testTimeout": 15000
   },
   "dependencies": {
-    "@aztec/circuit-types": "0.23.0",
-    "@aztec/foundation": "0.23.0",
-    "@aztec/kv-store": "0.23.0",
-    "@aztec/types": "0.23.0",
+    "@aztec/circuit-types": "0.24.0",
+    "@aztec/foundation": "0.24.0",
+    "@aztec/kv-store": "0.24.0",
+    "@aztec/types": "0.24.0",
     "sha256": "^0.2.0",
     "tslib": "^2.4.0"
   },

package/src/hasher_with_stats.ts

@@ -0,0 +1,51 @@
+import { Hasher } from '@aztec/types/interfaces';
+
+import { createHistogram, performance } from 'perf_hooks';
+
+/**
+ * A helper class to track stats for a Hasher
+ */
+export class HasherWithStats implements Hasher {
+  hashCount = 0;
+  hashInputsCount = 0;
+  hashHistogram = createHistogram();
+  hashInputsHistogram = createHistogram();
+
+  hash: Hasher['hash'];
+  hashInputs: Hasher['hashInputs'];
+
+  constructor(hasher: Hasher) {
+    this.hash = performance.timerify(
+      (lhs, rhs) => {
+        this.hashCount++;
+        return hasher.hash(lhs, rhs);
+      },
+      { histogram: this.hashHistogram },
+    );
+    this.hashInputs = performance.timerify(
+      (inputs: Buffer[]) => {
+        this.hashInputsCount++;
+        return hasher.hashInputs(inputs);
+      },
+      { histogram: this.hashInputsHistogram },
+    );
+  }
+
+  stats() {
+    return {
+      hashCount: this.hashCount,
+      // timerify records in ns, convert to ms
+      hashDuration: this.hashHistogram.mean / 1e6,
+      hashInputsCount: this.hashInputsCount,
+      hashInputsDuration: this.hashInputsHistogram.mean / 1e6,
+    };
+  }
+
+  reset() {
+    this.hashCount = 0;
+    this.hashHistogram.reset();
+
+    this.hashInputsCount = 0;
+    this.hashInputsHistogram.reset();
+  }
+}

package/src/index.ts

@@ -0,0 +1,15 @@
+export * from './interfaces/append_only_tree.js';
+export * from './interfaces/indexed_tree.js';
+export * from './interfaces/merkle_tree.js';
+export * from './interfaces/update_only_tree.js';
+export * from './pedersen.js';
+export * from './sparse_tree/sparse_tree.js';
+export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tree.js';
+export * from './standard_tree/standard_tree.js';
+export { INITIAL_LEAF, getTreeMeta } from './tree_base.js';
+export { newTree } from './new_tree.js';
+export { loadTree } from './load_tree.js';
+export * from './snapshots/snapshot_builder.js';
+export * from './snapshots/full_snapshot.js';
+export * from './snapshots/append_only_snapshot.js';
+export * from './snapshots/indexed_tree_snapshot.js';

package/src/interfaces/append_only_tree.ts

@@ -0,0 +1,13 @@
+import { TreeSnapshotBuilder } from '../snapshots/snapshot_builder.js';
+import { MerkleTree } from './merkle_tree.js';
+
+/**
+ * A Merkle tree that supports only appending leaves and not updating existing leaves.
+ */
+export interface AppendOnlyTree extends MerkleTree, TreeSnapshotBuilder {
+  /**
+   * Appends a set of leaf values to the tree.
+   * @param leaves - The set of leaves to be appended.
+   */
+  appendLeaves(leaves: Buffer[]): Promise<void>;
+}

package/src/interfaces/indexed_tree.ts

@@ -0,0 +1,118 @@
+import { SiblingPath } from '@aztec/circuit-types';
+import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees';
+
+import { AppendOnlyTree } from './append_only_tree.js';
+
+/**
+ * Factory for creating leaf preimages.
+ */
+export interface PreimageFactory {
+  /**
+   * Creates a new preimage from a leaf.
+   * @param leaf - Leaf to create a preimage from.
+   * @param nextKey - Next key of the leaf.
+   * @param nextIndex - Next index of the leaf.
+   */
+  fromLeaf(leaf: IndexedTreeLeaf, nextKey: bigint, nextIndex: bigint): IndexedTreeLeafPreimage;
+  /**
+   * Creates a new preimage from a buffer.
+   * @param buffer - Buffer to create a preimage from.
+   */
+  fromBuffer(buffer: Buffer): IndexedTreeLeafPreimage;
+  /**
+   * Creates an empty preimage.
+   */
+  empty(): IndexedTreeLeafPreimage;
+  /**
+   * Creates a copy of a preimage.
+   * @param preimage - Preimage to be cloned.
+   */
+  clone(preimage: IndexedTreeLeafPreimage): IndexedTreeLeafPreimage;
+}
+
+/**
+ * All of the data to be return during batch insertion.
+ */
+export interface LowLeafWitnessData<N extends number> {
+  /**
+   * Preimage of the low nullifier that proves non membership.
+   */
+  leafPreimage: IndexedTreeLeafPreimage;
+  /**
+   * Sibling path to prove membership of low nullifier.
+   */
+  siblingPath: SiblingPath<N>;
+  /**
+   * The index of low nullifier.
+   */
+  index: bigint;
+}
+
+/**
+ * The result of a batch insertion in an indexed merkle tree.
+ */
+export interface BatchInsertionResult<TreeHeight extends number, SubtreeSiblingPathHeight extends number> {
+  /**
+   * Data for the leaves to be updated when inserting the new ones.
+   */
+  lowLeavesWitnessData?: LowLeafWitnessData<TreeHeight>[];
+  /**
+   * Sibling path "pointing to" where the new subtree should be inserted into the tree.
+   */
+  newSubtreeSiblingPath: SiblingPath<SubtreeSiblingPathHeight>;
+  /**
+   * The new leaves being inserted in high to low order. This order corresponds with the order of the low leaves witness.
+   */
+  sortedNewLeaves: Buffer[];
+  /**
+   * The indexes of the sorted new leaves to the original ones.
+   */
+  sortedNewLeavesIndexes: number[];
+}
+
+/**
+ * Indexed merkle tree.
+ */
+export interface IndexedTree extends AppendOnlyTree {
+  /**
+   * Finds the index of the largest leaf whose value is less than or equal to the provided value.
+   * @param newValue - The new value to be inserted into the tree.
+   * @param includeUncommitted - If true, the uncommitted changes are included in the search.
+   * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`.
+   */
+  findIndexOfPreviousKey(
+    newValue: bigint,
+    includeUncommitted: boolean,
+  ):
+    | {
+        /**
+         * The index of the found leaf.
+         */
+        index: bigint;
+        /**
+         * A flag indicating if the corresponding leaf's value is equal to `newValue`.
+         */
+        alreadyPresent: boolean;
+      }
+    | undefined;
+
+  /**
+   * Gets the latest LeafPreimage copy.
+   * @param index - Index of the leaf of which to obtain the LeafPreimage copy.
+   * @param includeUncommitted - If true, the uncommitted changes are included in the search.
+   * @returns A copy of the leaf preimage at the given index or undefined if the leaf was not found.
+   */
+  getLatestLeafPreimageCopy(index: bigint, includeUncommitted: boolean): IndexedTreeLeafPreimage | undefined;
+
+  /**
+   * Batch insert multiple leaves into the tree.
+   * @param leaves - Leaves to insert into the tree.
+   * @param subtreeHeight - Height of the subtree.
+   * @param includeUncommitted - If true, the uncommitted changes are included in the search.
+   */
+  batchInsert<TreeHeight extends number, SubtreeHeight extends number, SubtreeSiblingPathHeight extends number>(
+    leaves: Buffer[],
+    subtreeHeight: SubtreeHeight,
+    includeUncommitted: boolean,
+  ): Promise<BatchInsertionResult<TreeHeight, SubtreeSiblingPathHeight>>;
+}

package/src/interfaces/merkle_tree.ts

@@ -0,0 +1,60 @@
+import { SiblingPath } from '@aztec/circuit-types';
+
+/**
+ * Defines the interface for a source of sibling paths.
+ */
+export interface SiblingPathSource {
+  /**
+   * Returns the sibling path for a requested leaf index.
+   * @param index - The index of the leaf for which a sibling path is required.
+   * @param includeUncommitted - Set to true to include uncommitted updates in the sibling path.
+   */
+  getSiblingPath<N extends number>(index: bigint, includeUncommitted: boolean): Promise<SiblingPath<N>>;
+}
+
+/**
+ * Defines the interface for a Merkle tree.
+ */
+export interface MerkleTree extends SiblingPathSource {
+  /**
+   * Returns the current root of the tree.
+   * @param includeUncommitted - Set to true to include uncommitted updates in the calculated root.
+   */
+  getRoot(includeUncommitted: boolean): Buffer;
+
+  /**
+   * Returns the number of leaves in the tree.
+   * @param includeUncommitted - Set to true to include uncommitted updates in the returned value.
+   */
+  getNumLeaves(includeUncommitted: boolean): bigint;
+
+  /**
+   * Commit pending updates to the tree.
+   */
+  commit(): Promise<void>;
+
+  /**
+   * Returns the depth of the tree.
+   */
+  getDepth(): number;
+
+  /**
+   * Rollback pending update to the tree.
+   */
+  rollback(): Promise<void>;
+
+  /**
+   * Returns the value of a leaf at the specified index.
+   * @param index - The index of the leaf value to be returned.
+   * @param includeUncommitted - Set to true to include uncommitted updates in the data set.
+   */
+  getLeafValue(index: bigint, includeUncommitted: boolean): Buffer | undefined;
+
+  /**
+   * Returns the index of a leaf given its value, or undefined if no leaf with that value is found.
+   * @param leaf - The leaf value to look for.
+   * @param includeUncommitted - Indicates whether to include uncommitted data.
+   * @returns The index of the first leaf found with a given value (undefined if not found).
+   */
+  findLeafIndex(leaf: Buffer, includeUncommitted: boolean): bigint | undefined;
+}

package/src/interfaces/update_only_tree.ts

@@ -0,0 +1,14 @@
+import { TreeSnapshotBuilder } from '../snapshots/snapshot_builder.js';
+import { MerkleTree } from './merkle_tree.js';
+
+/**
+ * A Merkle tree that supports updates at arbitrary indices but not appending.
+ */
+export interface UpdateOnlyTree extends MerkleTree, TreeSnapshotBuilder {
+  /**
+   * Updates a leaf at a given index in the tree.
+   * @param leaf - The leaf value to be updated.
+   * @param index - The leaf to be updated.
+   */
+  updateLeaf(leaf: Buffer, index: bigint): Promise<void>;
+}

package/src/load_tree.ts

@@ -0,0 +1,23 @@
+import { AztecKVStore } from '@aztec/kv-store';
+import { Hasher } from '@aztec/types/interfaces';
+
+import { TreeBase, getTreeMeta } from './tree_base.js';
+
+/**
+ * Creates a new tree and sets its root, depth and size based on the meta data which are associated with the name.
+ * @param c - The class of the tree to be instantiated.
+ * @param db - A database used to store the Merkle tree data.
+ * @param hasher - A hasher used to compute hash paths.
+ * @param name - Name of the tree.
+ * @returns The newly created tree.
+ */
+export function loadTree<T extends TreeBase>(
+  c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, root: Buffer) => T,
+  store: AztecKVStore,
+  hasher: Hasher,
+  name: string,
+): Promise<T> {
+  const { root, depth, size } = getTreeMeta(store, name);
+  const tree = new c(store, hasher, name, depth, size, root);
+  return Promise.resolve(tree);
+}

package/src/new_tree.ts

@@ -0,0 +1,27 @@
+import { AztecKVStore } from '@aztec/kv-store';
+import { Hasher } from '@aztec/types/interfaces';
+
+import { TreeBase } from './tree_base.js';
+
+/**
+ * Creates a new tree.
+ * @param c - The class of the tree to be instantiated.
+ * @param db - A database used to store the Merkle tree data.
+ * @param hasher - A hasher used to compute hash paths.
+ * @param name - Name of the tree.
+ * @param depth - Depth of the tree.
+ * @param prefilledSize - A number of leaves that are prefilled with values.
+ * @returns The newly created tree.
+ */
+export async function newTree<T extends TreeBase>(
+  c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint) => T,
+  store: AztecKVStore,
+  hasher: Hasher,
+  name: string,
+  depth: number,
+  prefilledSize = 1,
+): Promise<T> {
+  const tree = new c(store, hasher, name, depth, 0n);
+  await tree.init(prefilledSize);
+  return tree;
+}

package/src/pedersen.ts

@@ -0,0 +1,25 @@
+import { pedersenHash } from '@aztec/foundation/crypto';
+import { Hasher } from '@aztec/types/interfaces';
+
+/**
+ * A helper class encapsulating Pedersen hash functionality.
+ * @deprecated Don't call pedersen directly in production code. Instead, create suitably-named functions for specific
+ * purposes.
+ */
+export class Pedersen implements Hasher {
+  /*
+   * @deprecated Don't call pedersen directly in production code. Instead, create suitably-named functions for specific
+   * purposes.
+   */
+  public hash(lhs: Uint8Array, rhs: Uint8Array): Buffer {
+    return pedersenHash([Buffer.from(lhs), Buffer.from(rhs)]);
+  }
+
+  /*
+   * @deprecated Don't call pedersen directly in production code. Instead, create suitably-named functions for specific
+   * purposes.
+   */
+  public hashInputs(inputs: Buffer[]): Buffer {
+    return pedersenHash(inputs);
+  }
+}

package/src/snapshots/append_only_snapshot.ts

@@ -0,0 +1,262 @@
+import { SiblingPath } from '@aztec/circuit-types';
+import { AztecKVStore, AztecMap } from '@aztec/kv-store';
+import { Hasher } from '@aztec/types/interfaces';
+
+import { AppendOnlyTree } from '../interfaces/append_only_tree.js';
+import { TreeBase } from '../tree_base.js';
+import { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js';
+
+// stores the last block that modified this node
+const nodeModifiedAtBlockKey = (level: number, index: bigint) => `node:${level}:${index}:modifiedAtBlock`;
+
+// stores the value of the node at the above block
+const historicalNodeKey = (level: number, index: bigint) => `node:${level}:${index}:value`;
+
+/**
+ * Metadata for a snapshot, per block
+ */
+type SnapshotMetadata = {
+  /** The tree root at the time */
+  root: Buffer;
+  /** The number of filled leaves */
+  numLeaves: bigint;
+};
+
+/**
+ * A more space-efficient way of storing snapshots of AppendOnlyTrees that trades space need for slower
+ * sibling path reads.
+ *
+ * Complexity:
+ *
+ * N - count of non-zero nodes in tree
+ * M - count of snapshots
+ * H - tree height
+ *
+ * Space complexity: O(N + M) (N nodes - stores the last snapshot for each node and M - ints, for each snapshot stores up to which leaf its written to)
+ * Sibling path access:
+ *  Best case: O(H) database reads + O(1) hashes
+ *  Worst case: O(H) database reads + O(H) hashes
+ */
+export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder {
+  #nodeValue: AztecMap<ReturnType<typeof historicalNodeKey>, Buffer>;
+  #nodeLastModifiedByBlock: AztecMap<ReturnType<typeof nodeModifiedAtBlockKey>, number>;
+  #snapshotMetadata: AztecMap<number, SnapshotMetadata>;
+
+  constructor(private db: AztecKVStore, private tree: TreeBase & AppendOnlyTree, private hasher: Hasher) {
+    const treeName = tree.getName();
+    this.#nodeValue = db.openMap(`append_only_snapshot:${treeName}:node`);
+    this.#nodeLastModifiedByBlock = db.openMap(`append_ony_snapshot:${treeName}:block`);
+    this.#snapshotMetadata = db.openMap(`append_only_snapshot:${treeName}:snapshot_metadata`);
+  }
+
+  getSnapshot(block: number): Promise<TreeSnapshot> {
+    const meta = this.#getSnapshotMeta(block);
+
+    if (typeof meta === 'undefined') {
+      return Promise.reject(new Error(`Snapshot for tree ${this.tree.getName()} at block ${block} does not exist`));
+    }
+
+    return Promise.resolve(
+      new AppendOnlySnapshot(
+        this.#nodeValue,
+        this.#nodeLastModifiedByBlock,
+        block,
+        meta.numLeaves,
+        meta.root,
+        this.tree,
+        this.hasher,
+      ),
+    );
+  }
+
+  snapshot(block: number): Promise<TreeSnapshot> {
+    return this.db.transaction(() => {
+      const meta = this.#getSnapshotMeta(block);
+      if (typeof meta !== 'undefined') {
+        // no-op, we already have a snapshot
+        return new AppendOnlySnapshot(
+          this.#nodeValue,
+          this.#nodeLastModifiedByBlock,
+          block,
+          meta.numLeaves,
+          meta.root,
+          this.tree,
+          this.hasher,
+        );
+      }
+
+      const root = this.tree.getRoot(false);
+      const depth = this.tree.getDepth();
+      const queue: [Buffer, number, bigint][] = [[root, 0, 0n]];
+
+      // walk the tree in BF and store latest nodes
+      while (queue.length > 0) {
+        const [node, level, index] = queue.shift()!;
+
+        const historicalValue = this.#nodeValue.get(historicalNodeKey(level, index));
+        if (!historicalValue || !node.equals(historicalValue)) {
+          // we've never seen this node before or it's different than before
+          // update the historical tree and tag it with the block that modified it
+          void this.#nodeLastModifiedByBlock.set(nodeModifiedAtBlockKey(level, index), block);
+          void this.#nodeValue.set(historicalNodeKey(level, index), node);
+        } else {
+          // if this node hasn't changed, that means, nothing below it has changed either
+          continue;
+        }
+
+        if (level + 1 > depth) {
+          // short circuit if we've reached the leaf level
+          // otherwise getNode might throw if we ask for the children of a leaf
+          continue;
+        }
+
+        // these could be undefined because zero hashes aren't stored in the tree
+        const [lhs, rhs] = [this.tree.getNode(level + 1, 2n * index), this.tree.getNode(level + 1, 2n * index + 1n)];
+
+        if (lhs) {
+          queue.push([lhs, level + 1, 2n * index]);
+        }
+
+        if (rhs) {
+          queue.push([rhs, level + 1, 2n * index + 1n]);
+        }
+      }
+
+      const numLeaves = this.tree.getNumLeaves(false);
+      void this.#snapshotMetadata.set(block, {
+        numLeaves,
+        root,
+      });
+
+      return new AppendOnlySnapshot(
+        this.#nodeValue,
+        this.#nodeLastModifiedByBlock,
+        block,
+        numLeaves,
+        root,
+        this.tree,
+        this.hasher,
+      );
+    });
+  }
+
+  #getSnapshotMeta(block: number): SnapshotMetadata | undefined {
+    return this.#snapshotMetadata.get(block);
+  }
+}
+
+/**
+ * a
+ */
+class AppendOnlySnapshot implements TreeSnapshot {
+  constructor(
+    private nodes: AztecMap<string, Buffer>,
+    private nodeHistory: AztecMap<string, number>,
+    private block: number,
+    private leafCount: bigint,
+    private historicalRoot: Buffer,
+    private tree: TreeBase & AppendOnlyTree,
+    private hasher: Hasher,
+  ) {}
+
+  public getSiblingPath<N extends number>(index: bigint): SiblingPath<N> {
+    const path: Buffer[] = [];
+    const depth = this.tree.getDepth();
+    let level = depth;
+
+    while (level > 0) {
+      const isRight = index & 0x01n;
+      const siblingIndex = isRight ? index - 1n : index + 1n;
+
+      const sibling = this.#getHistoricalNodeValue(level, siblingIndex);
+      path.push(sibling);
+
+      level -= 1;
+      index >>= 1n;
+    }
+
+    return new SiblingPath<N>(depth as N, path);
+  }
+
+  getDepth(): number {
+    return this.tree.getDepth();
+  }
+
+  getNumLeaves(): bigint {
+    return this.leafCount;
+  }
+
+  getRoot(): Buffer {
+    // we could recompute it, but it's way cheaper to just store the root
+    return this.historicalRoot;
+  }
+
+  getLeafValue(index: bigint): Buffer | undefined {
+    const leafLevel = this.getDepth();
+    const blockNumber = this.#getBlockNumberThatModifiedNode(leafLevel, index);
+
+    // leaf hasn't been set yet
+    if (typeof blockNumber === 'undefined') {
+      return undefined;
+    }
+
+    // leaf was set some time in the past
+    if (blockNumber <= this.block) {
+      return this.nodes.get(historicalNodeKey(leafLevel, index));
+    }
+
+    // leaf has been set but in a block in the future
+    return undefined;
+  }
+
+  #getHistoricalNodeValue(level: number, index: bigint): Buffer {
+    const blockNumber = this.#getBlockNumberThatModifiedNode(level, index);
+
+    // node has never been set
+    if (typeof blockNumber === 'undefined') {
+      return this.tree.getZeroHash(level);
+    }
+
+    // node was set some time in the past
+    if (blockNumber <= this.block) {
+      return this.nodes.get(historicalNodeKey(level, index))!;
+    }
+
+    // the node has been modified since this snapshot was taken
+    // because we're working with an AppendOnly tree, historical leaves never change
+    // so what we do instead is rebuild this Merkle path up using zero hashes as needed
+    // worst case this will do O(H) hashes
+    //
+    // we first check if this subtree was touched by the block
+    // compare how many leaves this block added to the leaf interval of this subtree
+    // if they don't intersect then the whole subtree was a hash of zero
+    // if they do then we need to rebuild the merkle tree
+    const depth = this.tree.getDepth();
+    const leafStart = index * 2n ** BigInt(depth - level);
+    if (leafStart >= this.leafCount) {
+      return this.tree.getZeroHash(level);
+    }
+
+    const [lhs, rhs] = [
+      this.#getHistoricalNodeValue(level + 1, 2n * index),
+      this.#getHistoricalNodeValue(level + 1, 2n * index + 1n),
+    ];
+
+    return this.hasher.hash(lhs, rhs);
+  }
+
+  #getBlockNumberThatModifiedNode(level: number, index: bigint): number | undefined {
+    return this.nodeHistory.get(nodeModifiedAtBlockKey(level, index));
+  }
+
+  findLeafIndex(value: Buffer): bigint | undefined {
+    const numLeaves = this.getNumLeaves();
+    for (let i = 0n; i < numLeaves; i++) {
+      const currentValue = this.getLeafValue(i);
+      if (currentValue && currentValue.equals(value)) {
+        return i;
+      }
+    }
+    return undefined;
+  }
+}