@feelyourprotocol/binarytree 8141.0.0

package/src/constructors.ts

@@ -0,0 +1,50 @@
+import {
+  KeyEncoding,
+  MapDB,
+  ValueEncoding,
+  bytesToHex,
+  unprefixedHexToBytes,
+} from '@feelyourprotocol/util'
+import { blake3 } from '@noble/hashes/blake3.js'
+
+import { BinaryTree } from './binaryTree.ts'
+import { ROOT_DB_KEY } from './types.ts'
+
+import type { BinaryTreeOpts } from './types.ts'
+
+export async function createBinaryTree(opts?: Partial<BinaryTreeOpts>) {
+  const key = bytesToHex(ROOT_DB_KEY)
+
+  // Provide sensible default options
+  const parsedOptions = {
+    ...opts,
+    db: opts?.db ?? new MapDB<string, Uint8Array>(),
+    useRootPersistence: opts?.useRootPersistence ?? false,
+    cacheSize: opts?.cacheSize ?? 0,
+    hashFunction: opts?.hashFunction ?? blake3,
+  }
+
+  if (parsedOptions.useRootPersistence === true) {
+    if (parsedOptions.root === undefined) {
+      const root = await parsedOptions.db.get(key, {
+        keyEncoding: KeyEncoding.Bytes,
+        valueEncoding: ValueEncoding.Bytes,
+      })
+      if (typeof root === 'string') {
+        parsedOptions.root = unprefixedHexToBytes(root)
+      } else {
+        parsedOptions.root = root
+      }
+    } else {
+      await parsedOptions.db.put(key, parsedOptions.root, {
+        keyEncoding: KeyEncoding.Bytes,
+        valueEncoding: ValueEncoding.Bytes,
+      })
+    }
+  }
+
+  const tree = new BinaryTree(parsedOptions)
+  // If the root node does not exist, initialize the empty root node
+  if (parsedOptions.root === undefined) await tree.createRootNode()
+  return tree
+}

package/src/db/checkpoint.ts

@@ -0,0 +1,297 @@
+import {
+  KeyEncoding,
+  ValueEncoding,
+  bytesToUnprefixedHex,
+  unprefixedHexToBytes,
+} from '@feelyourprotocol/util'
+import { LRUCache } from 'lru-cache'
+
+import type { BatchDBOp, DB, EncodingOpts } from '@feelyourprotocol/util'
+import type { Checkpoint, CheckpointDBOpts } from '../types.ts'
+
+/**
+ * DB is a thin wrapper around the underlying levelup db,
+ * which validates inputs and sets encoding type.
+ */
+export class CheckpointDB implements DB {
+  public checkpoints: Checkpoint[]
+  public db: DB<string, string | Uint8Array>
+  public readonly cacheSize: number
+  private readonly valueEncoding: ValueEncoding
+
+  // Starting with lru-cache v8 undefined and null are not allowed any more
+  // as cache values. At the same time our design works well, since undefined
+  // indicates for us that we know that the value is not present in the
+  // underlying trie database as well (so it carries real value).
+  //
+  // Solution here seems therefore adequate, other solutions would rather
+  // be some not so clean workaround.
+  //
+  // (note that @ts-ignore doesn't work since stripped on declaration (.d.ts) files)
+  protected _cache?: LRUCache<string, Uint8Array>
+  // protected _cache?: LRUCache<string, Uint8Array | undefined>
+
+  _stats = {
+    cache: {
+      reads: 0,
+      hits: 0,
+      writes: 0,
+    },
+    db: {
+      reads: 0,
+      hits: 0,
+      writes: 0,
+    },
+  }
+
+  /**
+   * Initialize a DB instance.
+   */
+  constructor(opts: CheckpointDBOpts) {
+    this.db = opts.db
+    this.cacheSize = opts.cacheSize ?? 0
+    this.valueEncoding = opts.valueEncoding ?? ValueEncoding.String
+    // Roots of trie at the moment of checkpoint
+    this.checkpoints = []
+
+    if (this.cacheSize > 0) {
+      this._cache = new LRUCache({
+        max: this.cacheSize,
+        updateAgeOnGet: true,
+      })
+    }
+  }
+
+  /**
+   * Flush the checkpoints and use the given checkpoints instead.
+   * @param {Checkpoint[]} checkpoints
+   */
+  setCheckpoints(checkpoints: Checkpoint[]) {
+    this.checkpoints = []
+
+    for (let i = 0; i < checkpoints.length; i++) {
+      this.checkpoints.push({
+        root: checkpoints[i].root,
+        keyValueMap: new Map(checkpoints[i].keyValueMap),
+      })
+    }
+  }
+
+  /**
+   * Is the DB during a checkpoint phase?
+   */
+  hasCheckpoints() {
+    return this.checkpoints.length > 0
+  }
+
+  /**
+   * Adds a new checkpoint to the stack
+   * @param root
+   */
+  checkpoint(root: Uint8Array) {
+    this.checkpoints.push({ keyValueMap: new Map<string, Uint8Array>(), root })
+  }
+
+  /**
+   * Commits the latest checkpoint
+   */
+  async commit() {
+    const { keyValueMap } = this.checkpoints.pop()!
+    if (!this.hasCheckpoints()) {
+      // This was the final checkpoint, we should now commit and flush everything to disk
+      const batchOp: BatchDBOp[] = []
+      for (const [key, value] of keyValueMap.entries()) {
+        if (value === undefined) {
+          batchOp.push({
+            type: 'del',
+            key: unprefixedHexToBytes(key),
+          })
+        } else {
+          batchOp.push({
+            type: 'put',
+            key: unprefixedHexToBytes(key),
+            value,
+          })
+        }
+      }
+      await this.batch(batchOp)
+    } else {
+      // dump everything into the current (higher level) diff cache
+      const currentKeyValueMap = this.checkpoints[this.checkpoints.length - 1].keyValueMap
+      for (const [key, value] of keyValueMap.entries()) {
+        currentKeyValueMap.set(key, value)
+      }
+    }
+  }
+
+  /**
+   * Reverts the latest checkpoint
+   */
+  async revert() {
+    const { root } = this.checkpoints.pop()!
+    return root
+  }
+
+  /**
+   * @inheritDoc
+   */
+  async get(key: Uint8Array): Promise<Uint8Array | undefined> {
+    // Using deprecated bytesToUnprefixedHex for performance: used as cache/database keys (string encoding).
+    const keyHex = bytesToUnprefixedHex(key)
+    if (this._cache !== undefined) {
+      const value = this._cache.get(keyHex)
+      this._stats.cache.reads += 1
+      if (value !== undefined) {
+        this._stats.cache.hits += 1
+        return value
+      }
+    }
+
+    // Lookup the value in our diff cache. We return the latest checkpointed value (which should be the value on disk)
+    for (let index = this.checkpoints.length - 1; index >= 0; index--) {
+      if (this.checkpoints[index].keyValueMap.has(keyHex)) {
+        return this.checkpoints[index].keyValueMap.get(keyHex)
+      }
+    }
+    // Nothing has been found in diff cache, look up from disk
+    const value = await this.db.get(keyHex, {
+      keyEncoding: KeyEncoding.String,
+      valueEncoding: this.valueEncoding,
+    })
+    this._stats.db.reads += 1
+    if (value !== undefined) {
+      this._stats.db.hits += 1
+    }
+    const returnValue =
+      value !== undefined
+        ? value instanceof Uint8Array
+          ? value
+          : unprefixedHexToBytes(value as string)
+        : undefined
+    this._cache?.set(keyHex, returnValue)
+    if (this.hasCheckpoints()) {
+      // Since we are a checkpoint, put this value in diff cache,
+      // so future `get` calls will not look the key up again from disk.
+      this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, returnValue)
+    }
+
+    return returnValue
+  }
+
+  /**
+   * @inheritDoc
+   */
+  async put(key: Uint8Array, value: Uint8Array): Promise<void> {
+    // Using deprecated bytesToUnprefixedHex for performance: used as cache/database keys and values (string encoding).
+    const keyHex = bytesToUnprefixedHex(key)
+    if (this.hasCheckpoints()) {
+      // put value in diff cache
+      this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, value)
+    } else {
+      const valuePut =
+        this.valueEncoding === ValueEncoding.Bytes ? value : bytesToUnprefixedHex(value)
+      await this.db.put(keyHex, valuePut, {
+        keyEncoding: KeyEncoding.String,
+        valueEncoding: this.valueEncoding,
+      })
+      this._stats.db.writes += 1
+
+      if (this._cache !== undefined) {
+        this._cache.set(keyHex, value)
+        this._stats.cache.writes += 1
+      }
+    }
+  }
+
+  /**
+   * @inheritDoc
+   */
+  async del(key: Uint8Array): Promise<void> {
+    // Using deprecated bytesToUnprefixedHex for performance: used as cache/database keys (string encoding).
+    const keyHex = bytesToUnprefixedHex(key)
+    if (this.hasCheckpoints()) {
+      // delete the value in the current diff cache
+      this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, undefined)
+    } else {
+      // delete the value on disk
+      await this.db.del(keyHex, {
+        keyEncoding: KeyEncoding.String,
+      })
+      this._stats.db.writes += 1
+
+      if (this._cache !== undefined) {
+        this._cache.set(keyHex, undefined)
+        this._stats.cache.writes += 1
+      }
+    }
+  }
+
+  /**
+   * @inheritDoc
+   */
+  async batch(opStack: BatchDBOp[]): Promise<void> {
+    if (this.hasCheckpoints()) {
+      for (const op of opStack) {
+        if (op.type === 'put') {
+          await this.put(op.key, op.value)
+        } else if (op.type === 'del') {
+          await this.del(op.key)
+        }
+      }
+    } else {
+      const convertedOps = opStack.map((op) => {
+        const convertedOp: {
+          key: string
+          value: Uint8Array | string | undefined
+          type: 'put' | 'del'
+          opts?: EncodingOpts
+        } = {
+          key: bytesToUnprefixedHex(op.key),
+          value: op.type === 'put' ? op.value : undefined,
+          type: op.type,
+          opts: { ...op.opts, ...{ valueEncoding: this.valueEncoding } },
+        }
+        this._stats.db.writes += 1
+        if (op.type === 'put' && this.valueEncoding === ValueEncoding.String) {
+          convertedOp.value = bytesToUnprefixedHex(convertedOp.value as Uint8Array)
+        }
+        return convertedOp
+      })
+      await this.db.batch(convertedOps as any)
+    }
+  }
+
+  stats(reset = true) {
+    const stats = { ...this._stats, size: this._cache?.size ?? 0 }
+    if (reset) {
+      this._stats = {
+        cache: {
+          reads: 0,
+          hits: 0,
+          writes: 0,
+        },
+        db: {
+          reads: 0,
+          hits: 0,
+          writes: 0,
+        },
+      }
+    }
+    return stats
+  }
+
+  /**
+   * @inheritDoc
+   */
+  shallowCopy(): CheckpointDB {
+    return new CheckpointDB({
+      db: this.db,
+      cacheSize: this.cacheSize,
+      valueEncoding: this.valueEncoding,
+    })
+  }
+
+  open() {
+    return Promise.resolve()
+  }
+}

package/src/db/index.ts

@@ -0,0 +1 @@
+export * from './checkpoint.ts'

package/src/index.ts

@@ -0,0 +1,6 @@
+export * from './binaryTree.ts'
+export * from './constructors.ts'
+export * from './db/index.ts'
+export * from './node/index.ts'
+export * from './proof.ts'
+export * from './types.ts'

package/src/node/index.ts

@@ -0,0 +1,3 @@
+export * from './internalNode.ts'
+export * from './types.ts'
+export * from './util.ts'

package/src/node/internalNode.ts

@@ -0,0 +1,112 @@
+import { RLP } from '@feelyourprotocol/rlp'
+import { EthereumJSErrorWithoutCode, bitsToBytes, bytesToBits } from '@feelyourprotocol/util'
+
+import { BinaryNodeType } from './types.ts'
+
+import type { BinaryNodeOptions, ChildBinaryNode } from './types.ts'
+
+export class InternalBinaryNode {
+  public children: Array<ChildBinaryNode | null>
+
+  public type = BinaryNodeType.Internal
+
+  constructor(options: BinaryNodeOptions[typeof BinaryNodeType.Internal]) {
+    this.children = options.children ?? Array(2).fill(null)
+  }
+
+  static fromRawNode(rawNode: Uint8Array[]): InternalBinaryNode {
+    const nodeType = rawNode[0][0]
+    if (nodeType !== BinaryNodeType.Internal) {
+      throw EthereumJSErrorWithoutCode('Invalid node type')
+    }
+
+    // The length of the rawNode should be the # of children * 2 (for hash and path) + 1 for the node type
+
+    if (rawNode.length !== 2 * 2 + 1) {
+      throw EthereumJSErrorWithoutCode('Invalid node length')
+    }
+    const [, leftChildHash, rightChildHash, leftChildRawPath, rightChildRawPath] = rawNode
+
+    const decodeChild = (hash: Uint8Array, rawPath: Uint8Array): ChildBinaryNode | null => {
+      if (hash.length === 0) return null
+      const decoded = RLP.decode(rawPath)
+
+      if (!Array.isArray(decoded) || decoded.length !== 2) {
+        throw EthereumJSErrorWithoutCode('Invalid RLP encoding for child path')
+      }
+
+      const [encodedLength, encodedPath] = decoded as Uint8Array[]
+
+      if (encodedLength.length !== 1) {
+        throw EthereumJSErrorWithoutCode('Invalid path length encoding')
+      }
+
+      const pathLength = encodedLength[0]
+      const path = bytesToBits(encodedPath, pathLength)
+
+      return { hash, path }
+    }
+
+    const children = [
+      decodeChild(leftChildHash, leftChildRawPath),
+      decodeChild(rightChildHash, rightChildRawPath),
+    ]
+
+    return new InternalBinaryNode({ children })
+  }
+
+  /**
+   * Generates a new Internal node
+   * @param children the children nodes
+   * @returns a new Internal node
+   */
+  static create(children?: (ChildBinaryNode | null)[]): InternalBinaryNode {
+    if (children !== undefined && children.length !== 2) {
+      throw EthereumJSErrorWithoutCode('Internal node must have 2 children')
+    }
+    return new InternalBinaryNode({ children })
+  }
+
+  getChild(index: number): ChildBinaryNode | null {
+    return this.children[index]
+  }
+
+  setChild(index: number, child: ChildBinaryNode | null): void {
+    this.children[index] = child
+  }
+
+  /**
+   * @returns the RLP serialized node
+   */
+  serialize(): Uint8Array {
+    return RLP.encode(this.raw())
+  }
+
+  /**
+   * Returns the raw serialized representation of this internal node as an array of Uint8Arrays.
+   *
+   * The returned array contains:
+   * 1. A single-byte Uint8Array indicating the node type (BinaryNodeType.Internal).
+   * 2. For each child (left then right):
+   *    - The child’s hash, or an empty Uint8Array if the child is null.
+   * 3. For each child (left then right):
+   *    - An RLP-encoded tuple [pathLength, packedPathBytes] where:
+   *         - `pathLength` is a one-byte Uint8Array representing the number of meaningful bits in the child’s path.
+   *         - `packedPathBytes` is the packed byte representation of the child's bit path (as produced by `bitsToBytes`).
+   *
+   * @returns {Uint8Array[]} An array of Uint8Arrays representing the node's serialized internal data.
+   * @dev When decoding, the stored child path (an RLP-encoded tuple) must be converted back into the original bit array.
+   */
+
+  raw(): Uint8Array[] {
+    return [
+      new Uint8Array([BinaryNodeType.Internal]),
+      ...this.children.map((child) => (child !== null ? child.hash : new Uint8Array())),
+      ...this.children.map((child) =>
+        child !== null
+          ? RLP.encode([new Uint8Array([child.path.length]), bitsToBytes(child.path)])
+          : new Uint8Array(),
+      ),
+    ]
+  }
+}

package/src/node/stemNode.ts

@@ -0,0 +1,87 @@
+import { RLP } from '@feelyourprotocol/rlp'
+import { EthereumJSErrorWithoutCode } from '@feelyourprotocol/util'
+
+import { BinaryNodeType, NODE_WIDTH } from './types.ts'
+
+import type { BinaryNodeOptions } from './types.ts'
+
+export class StemBinaryNode {
+  public stem: Uint8Array
+  public values: (Uint8Array | null)[] // Array of 256 possible values represented as 32 byte Uint8Arrays
+
+  public type = BinaryNodeType.Stem
+
+  constructor(options: BinaryNodeOptions[typeof BinaryNodeType.Stem]) {
+    this.stem = options.stem
+    this.values = options.values ?? new Array(256).fill(null)
+  }
+
+  static fromRawNode(rawNode: Uint8Array[]): StemBinaryNode {
+    const nodeType = rawNode[0][0]
+    if (nodeType !== BinaryNodeType.Stem) {
+      throw EthereumJSErrorWithoutCode('Invalid node type')
+    }
+
+    // The length of the rawNode should be the # of values (node width) + 2 for the node type and the stem
+    if (rawNode.length !== NODE_WIDTH + 2) {
+      throw EthereumJSErrorWithoutCode('Invalid node length')
+    }
+
+    const stem = rawNode[1]
+    const rawValues = rawNode.slice(2, rawNode.length)
+    const values = rawValues.map((el) => (el.length === 0 ? null : el))
+
+    return new StemBinaryNode({ stem, values })
+  }
+
+  /**
+   * Generates a new Stem node
+   * @param stem the 31 byte stem corresponding to the where the stem node is located in the tree
+   * @returns a new Stem node
+   */
+  static create(stem: Uint8Array): StemBinaryNode {
+    return new StemBinaryNode({ stem })
+  }
+
+  // Retrieve the value at the provided index from the values array
+  getValue(index: number): Uint8Array | null {
+    return this.values[index]
+  }
+
+  setValue(index: number, value: Uint8Array | null): void {
+    this.values[index] = value
+  }
+
+  /**
+   * @returns the RLP serialized node
+   */
+  serialize(): Uint8Array {
+    return RLP.encode(this.raw())
+  }
+
+  /**
+   * Returns the raw serialized representation of the node as an array of Uint8Arrays.
+   * The returned array is constructed as follows:
+   * - The first element is a Uint8Array containing a single byte that represents the node type,
+   * - The second element is the node's `stem` property.
+   * - The remaining elements are derived from the node's `values` array:
+   *   - For each value, if it is `null`, it is converted to an empty Uint8Array.
+   *   - Otherwise, the value is included as-is.
+   *
+   * @returns {Uint8Array[]} An array of Uint8Arrays representing the node's raw data.
+   */
+  raw(): Uint8Array[] {
+    return [
+      new Uint8Array([BinaryNodeType.Stem]),
+      this.stem,
+      ...this.values.map((val) => {
+        switch (val) {
+          case null:
+            return new Uint8Array()
+          default:
+            return val
+        }
+      }),
+    ]
+  }
+}

package/src/node/types.ts

@@ -0,0 +1,41 @@
+import type { InternalBinaryNode } from './internalNode.ts'
+import type { StemBinaryNode } from './stemNode.ts'
+
+export type BinaryNodeType = (typeof BinaryNodeType)[keyof typeof BinaryNodeType]
+export const BinaryNodeType = {
+  Internal: 0,
+  Stem: 1,
+} as const
+
+export interface TypedBinaryNode {
+  [BinaryNodeType.Internal]: InternalBinaryNode
+  [BinaryNodeType.Stem]: StemBinaryNode
+}
+
+export type BinaryNode = TypedBinaryNode[BinaryNodeType]
+
+/**
+ * @dev A child node in a binary tree internal node.
+ * @param hash The hash of the child node.
+ * @param path The path to the child node, in bits.
+ * */
+export type ChildBinaryNode = {
+  hash: Uint8Array
+  path: number[]
+}
+
+interface InternalBinaryNodeOptions {
+  children?: (ChildBinaryNode | null)[]
+}
+
+interface StemBinaryNodeOptions {
+  stem: Uint8Array
+  values?: (Uint8Array | null)[]
+}
+
+export interface BinaryNodeOptions {
+  [BinaryNodeType.Internal]: InternalBinaryNodeOptions
+  [BinaryNodeType.Stem]: StemBinaryNodeOptions
+}
+
+export const NODE_WIDTH = 256

package/src/node/util.ts

@@ -0,0 +1,38 @@
+import { RLP } from '@feelyourprotocol/rlp'
+import { EthereumJSErrorWithoutCode } from '@feelyourprotocol/util'
+
+import { InternalBinaryNode } from './internalNode.ts'
+import { StemBinaryNode } from './stemNode.ts'
+import { type BinaryNode, BinaryNodeType } from './types.ts'
+
+export function decodeRawBinaryNode(raw: Uint8Array[]): BinaryNode {
+  const nodeType = raw[0][0]
+  switch (nodeType) {
+    case BinaryNodeType.Internal:
+      return InternalBinaryNode.fromRawNode(raw)
+    case BinaryNodeType.Stem:
+      return StemBinaryNode.fromRawNode(raw)
+    default:
+      throw EthereumJSErrorWithoutCode('Invalid node type')
+  }
+}
+
+export function decodeBinaryNode(raw: Uint8Array) {
+  const decoded = RLP.decode(Uint8Array.from(raw)) as Uint8Array[]
+  if (!Array.isArray(decoded)) {
+    throw EthereumJSErrorWithoutCode('Invalid node')
+  }
+  return decodeRawBinaryNode(decoded)
+}
+
+export function isRawBinaryNode(node: Uint8Array | Uint8Array[]): node is Uint8Array[] {
+  return Array.isArray(node) && !(node instanceof Uint8Array)
+}
+
+export function isInternalBinaryNode(node: BinaryNode): node is InternalBinaryNode {
+  return node.type === BinaryNodeType.Internal
+}
+
+export function isStemBinaryNode(node: BinaryNode): node is StemBinaryNode {
+  return node.type === BinaryNodeType.Stem
+}

package/src/proof.ts

@@ -0,0 +1,54 @@
+import { EthereumJSErrorWithoutCode, equalsBytes } from '@feelyourprotocol/util'
+
+import { createBinaryTree } from './constructors.ts'
+import { decodeBinaryNode } from './node/index.ts'
+
+import type { BinaryTree } from './binaryTree.ts'
+import type { BinaryNode } from './node/index.ts'
+import type { StemBinaryNode } from './node/stemNode.ts'
+
+/**
+ * Saves the nodes from a proof into the tree.
+ * @param proof
+ */
+export async function binaryTreeFromProof(proof: Uint8Array[]): Promise<BinaryTree> {
+  const proofTrie = await createBinaryTree()
+  const putStack: [Uint8Array, BinaryNode][] = proof.map((bytes) => {
+    const node = decodeBinaryNode(bytes)
+    return [proofTrie['merkelize'](node), node]
+  })
+  await proofTrie.saveStack(putStack)
+  const root = putStack[0][0]
+  proofTrie.root(root)
+  return proofTrie
+}
+
+/**
+ * Verifies a proof.
+ * @param rootHash
+ * @param key
+ * @param proof
+ * @throws If proof is found to be invalid.
+ * @returns The value from the key, or null if valid proof of non-existence.
+ */
+export async function verifyBinaryProof(
+  rootHash: Uint8Array,
+  key: Uint8Array,
+  proof: Uint8Array[],
+): Promise<Uint8Array | null> {
+  const proofTrie = await binaryTreeFromProof(proof)
+  if (!equalsBytes(proofTrie.root(), rootHash)) {
+    throw EthereumJSErrorWithoutCode('rootHash does not match proof root')
+  }
+  const [value] = await proofTrie.get(key.slice(0, 31), [key[31]])
+  const valueNode = decodeBinaryNode(proof[proof.length - 1]) as StemBinaryNode
+  const expectedValue = valueNode.values[key[31]]
+  if (!expectedValue) {
+    if (value) {
+      throw EthereumJSErrorWithoutCode('Proof is invalid')
+    }
+  } else if (value && !equalsBytes(value, expectedValue)) {
+    throw EthereumJSErrorWithoutCode('Proof is invalid')
+  }
+  return value
+}