@feelyourprotocol/mpt 8141.0.0

package/src/util/asyncWalk.ts

@@ -0,0 +1,60 @@
+import { RLP } from '@feelyourprotocol/rlp'
+import { bytesToHex, equalsBytes } from '@feelyourprotocol/util'
+
+import { BranchMPTNode } from '../node/branch.ts'
+import { ExtensionMPTNode } from '../node/extension.ts'
+
+import type { MerklePatriciaTrie } from '../mpt.ts'
+import type { MPTNode } from '../types.ts'
+
+export type NodeFilter = (node: MPTNode, key: number[]) => Promise<boolean>
+export type OnFound = (node: MPTNode, key: number[]) => Promise<any>
+
+/**
+ * Walk MerklePatriciaTrie via async generator
+ * @param nodeHash - The root key to walk on.
+ * @param currentKey - The current (partial) key.
+ * @param onFound - Called on every node found (before filter)
+ * @param filter - Filter nodes yielded by the generator.
+ * @param visited - Set of visited nodes
+ * @returns AsyncIterable<{ node: MPTNode; currentKey: number[] }>
+ * Iterate through nodes with
+ * `for await (const { node, currentKey } of trie._walkTrie(root)) { ... }`
+ */
+export async function* _walkTrie(
+  this: MerklePatriciaTrie,
+  nodeHash: Uint8Array,
+  currentKey: number[] = [],
+  onFound: OnFound = async (_trieNode: MPTNode, _key: number[]) => {},
+  filter: NodeFilter = async (_trieNode: MPTNode, _key: number[]) => true,
+  visited: Set<string> = new Set<string>(),
+): AsyncIterable<{ node: MPTNode; currentKey: number[] }> {
+  if (equalsBytes(nodeHash, this.EMPTY_TRIE_ROOT)) {
+    return
+  }
+  try {
+    const node = await this.lookupNode(nodeHash)
+    if (node === undefined || visited.has(bytesToHex(this.hash(node!.serialize())))) {
+      return
+    }
+    visited.add(bytesToHex(this.hash(node!.serialize())))
+    await onFound(node!, currentKey)
+    if (await filter(node!, currentKey)) {
+      yield { node: node!, currentKey }
+    }
+    if (node instanceof BranchMPTNode) {
+      for (const [nibble, childNode] of node._branches.entries()) {
+        const nextKey = [...currentKey, nibble]
+        const _childNode: Uint8Array =
+          childNode instanceof Uint8Array ? childNode : this.hash(RLP.encode(childNode))
+        yield* _walkTrie.bind(this)(_childNode, nextKey, onFound, filter, visited)
+      }
+    } else if (node instanceof ExtensionMPTNode) {
+      const childNode = node.value()
+      const nextKey = [...currentKey, ...node._nibbles]
+      yield* _walkTrie.bind(this)(childNode, nextKey, onFound, filter, visited)
+    }
+  } catch {
+    return
+  }
+}

package/src/util/encoding.ts

@@ -0,0 +1,209 @@
+import { concatBytes, hexToBytes, unprefixedHexToBytes } from '@feelyourprotocol/util'
+
+import { nibblesTypeToPackedBytes } from './nibbles.ts'
+
+import type { Nibbles } from '../types.ts'
+
+// Reference: https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/
+//
+// MerklePatriciaTrie keys are dealt with in three distinct encodings:
+//
+// KEYBYTES encoding contains the actual key and nothing else. This encoding is the
+// input to most API functions.
+//
+// HEX encoding contains one byte for each nibble of the key and an optional trailing
+// 'terminator' byte of value 0x10 which indicates whether or not the node at the key
+// contains a value. Hex key encoding is used for nodes loaded in memory because it's
+// convenient to access.
+//
+// COMPACT encoding is defined by the Ethereum Yellow Paper (it's called "hex prefix
+// encoding" there) and contains the bytes of the key and a flag. The high nibble of the
+// first byte contains the flag; the lowest bit encoding the oddness of the length and
+// the second-lowest encoding whether the node at the key is a value node. The low nibble
+// of the first byte is zero in the case of an even number of nibbles and the first nibble
+// in the case of an odd number. All remaining nibbles (now an even number) fit properly
+// into the remaining bytes. Compact encoding is used for nodes stored on disk.
+
+/**
+ *
+ * @param nibbles byte sequence
+ * @returns boolean indicating if input hex nibble sequence has terminator indicating leaf-node
+ *          terminator is represented with 16 because a nibble ranges from 0 - 15(f)
+ */
+const hasTerminator = (nibbles: Uint8Array) => {
+  return nibbles.length > 0 && nibbles[nibbles.length - 1] === 16
+}
+
+export const nibblesToBytes = (nibbles: Uint8Array) => {
+  const bytes = new Uint8Array(nibbles.length / 2)
+  for (let bi = 0, ni = 0; ni < nibbles.length; bi += 1, ni += 2) {
+    bytes[bi] = (nibbles[ni] << 4) | nibbles[ni + 1]
+  }
+
+  return bytes
+}
+
+export const hexToKeybytes = (hex: Uint8Array) => {
+  if (hasTerminator(hex)) {
+    hex = hex.subarray(0, hex.length - 1)
+  }
+  if (hex.length % 2 === 1) {
+    throw Error("Can't convert hex key of odd length")
+  }
+
+  return nibblesToBytes(hex)
+}
+
+// hex to compact
+export const nibblesToCompactBytes = (nibbles: Uint8Array) => {
+  let terminator = 0
+  if (hasTerminator(nibbles)) {
+    terminator = 1
+    // Remove the terminator from the sequence
+    nibbles = nibbles.subarray(0, nibbles.length - 1)
+  }
+  const buf = new Uint8Array(nibbles.length / 2 + 1)
+  // Shift the terminator info into the first nibble of buf[0]
+  buf[0] = terminator << 5
+  // If odd length, then add that flag into the first nibble and put the odd nibble to
+  // second part of buf[0] which otherwise will be left padded with a 0
+  if ((nibbles.length & 1) === 1) {
+    buf[0] |= 1 << 4
+    buf[0] |= nibbles[0]
+    nibbles = nibbles.subarray(1)
+  }
+
+  // create bytes out of the rest even nibbles
+  return concatBytes(buf.subarray(0, 1), nibblesToBytes(nibbles))
+}
+
+export const bytesToNibbles = (str: Uint8Array) => {
+  const l = str.length * 2 + 1
+  const nibbles = new Uint8Array(l)
+  for (let i = 0; i < str.length; i++) {
+    const b = str[i]
+    nibbles[i * 2] = b / 16
+    nibbles[i * 2 + 1] = b % 16
+  }
+  // This will get removed from calling function if the first nibble
+  // indicates that terminator is not present
+  nibbles[l - 1] = 16
+  return nibbles
+}
+
+export const compactBytesToNibbles = (compact: Uint8Array) => {
+  if (compact.length === 0) {
+    return compact
+  }
+  let base = bytesToNibbles(compact)
+  // delete terminator flag if terminator flag was not in first nibble
+  if (base[0] < 2) {
+    base = base.subarray(0, base.length - 1)
+  }
+  // chop the terminator nibble and the even padding (if there is one)
+  // i.e.  chop 2 left nibbles when even else 1 when odd
+  const chop = 2 - (base[0] & 1)
+  return base.subarray(chop)
+}
+
+/**
+ * Converts each nibble into a single byte
+ *
+ * @param arr Nibble typed nibble array
+ * @returns Uint8Array typed byte array
+ */
+export const nibbleTypeToByteType = (arr: Nibbles): Uint8Array => {
+  const l = arr.length
+  const buf = new Uint8Array(l)
+  for (let i = 0; i < buf.length; i++) {
+    buf[i] = arr[i]
+  }
+
+  return buf
+}
+
+/**
+ * Turns each byte into a single nibble, only extracting the lower nibble of each byte
+ *
+ * @param key Uint8Array typed byte array
+ * @returns Nibble typed nibble array
+ */
+export const byteTypeToNibbleType = (key: Uint8Array): Nibbles => {
+  const nibbles = [] as Nibbles
+
+  for (let i = 0; i < key.length; i++) {
+    const q = i
+    nibbles[q] = key[i] % 16
+  }
+
+  return nibbles
+}
+
+/**
+ * Takes a string path and extends it by the given extension nibbles
+ *
+ * @param path String node path
+ * @param extension nibbles to extend by
+ * @param retType string indicating whether to return the key in "keybyte" or "hex" encoding
+ * @returns hex-encoded key
+ */
+export const pathToHexKey = (path: string, extension: Nibbles, retType: string): Uint8Array => {
+  const b = hexToBytes(`0x${path}`)
+  const n = byteTypeToNibbleType(b)
+  if (retType === 'hex') {
+    return nibbleTypeToByteType(n.concat(extension))
+  } else if (retType === 'keybyte') {
+    return nibblesTypeToPackedBytes(n.concat(extension))
+  }
+  throw Error('retType must be either "keybyte" or "hex"')
+}
+
+export const mergeAndFormatKeyPaths = (pathStrings: string[]) => {
+  const ret: string[][] = []
+  let paths: string[] = []
+  let i = 0
+  while (i < pathStrings.length) {
+    const outerPathString = pathStrings[i]!.split('/')
+    const outerAccountPath = outerPathString[0]
+    const outerStoragePath = outerPathString[1]
+
+    paths.push(outerAccountPath)
+    if (outerStoragePath !== undefined) {
+      paths.push(outerStoragePath)
+    }
+
+    let j = ++i
+    while (j < pathStrings.length) {
+      const innerPathString = pathStrings[j]!.split('/')
+      const innerAccountPath = innerPathString[0]
+      const innerStoragePath = innerPathString[1]
+
+      if (innerAccountPath === outerAccountPath) {
+        paths.push(innerStoragePath)
+      } else {
+        ret.push(paths)
+        paths = []
+        i = j
+        break
+      }
+      j++
+    }
+    if (paths.length > 0) {
+      ret.push(paths)
+      paths = []
+    }
+  }
+  if (paths.length > 0) ret.push(paths)
+
+  return ret.map((pathStrings) =>
+    pathStrings.map((s) => {
+      if (s.length < 64) {
+        // partial path is compact encoded
+        return nibblesToCompactBytes(unprefixedHexToBytes(s))
+      } else {
+        // full path is keybyte encoded
+        return hexToKeybytes(unprefixedHexToBytes(s))
+      }
+    }),
+  )
+}

package/src/util/genesisState.ts

@@ -0,0 +1,52 @@
+import { RLP } from '@feelyourprotocol/rlp'
+import {
+  Account,
+  hexToBytes,
+  isHexString,
+  unpadBytes,
+  unprefixedHexToBytes,
+} from '@feelyourprotocol/util'
+import { keccak_256 } from '@noble/hashes/sha3.js'
+
+import { MerklePatriciaTrie } from '../mpt.ts'
+
+import type { AccountState, GenesisState } from '@feelyourprotocol/common'
+
+/**
+ * Derives the stateRoot of the genesis block based on genesis allocations
+ */
+export async function genesisMPTStateRoot(genesisState: GenesisState) {
+  const trie = new MerklePatriciaTrie({ useKeyHashing: true })
+  for (const [key, value] of Object.entries(genesisState)) {
+    const address = isHexString(key) ? hexToBytes(key) : unprefixedHexToBytes(key)
+    const account = new Account()
+    if (typeof value === 'string') {
+      account.balance = BigInt(value)
+    } else {
+      const [balance, code, storage, nonce] = value as Partial<AccountState>
+      if (balance !== undefined) {
+        account.balance = BigInt(balance)
+      }
+      if (code !== undefined) {
+        const codeBytes = isHexString(code) ? hexToBytes(code) : unprefixedHexToBytes(code)
+        account.codeHash = keccak_256(codeBytes)
+      }
+      if (storage !== undefined) {
+        const storageTrie = new MerklePatriciaTrie({ useKeyHashing: true })
+        for (const [k, val] of storage) {
+          const storageKey = isHexString(k) ? hexToBytes(k) : unprefixedHexToBytes(k)
+          const storageVal = RLP.encode(
+            unpadBytes(isHexString(val) ? hexToBytes(val) : unprefixedHexToBytes(val)),
+          )
+          await storageTrie.put(storageKey, storageVal)
+        }
+        account.storageRoot = storageTrie.root()
+      }
+      if (nonce !== undefined) {
+        account.nonce = BigInt(nonce)
+      }
+    }
+    await trie.put(address, account.serialize())
+  }
+  return trie.root()
+}

package/src/util/hex.ts

@@ -0,0 +1,47 @@
+import type { Nibbles } from '../types.ts'
+
+/**
+ * Prepends hex prefix to an array of nibbles.
+ * @param key - Array of nibbles
+ * @returns returns buffer of encoded data
+ **/
+export function addHexPrefix(key: Nibbles, terminator: boolean): Nibbles {
+  // odd
+  if (key.length % 2) {
+    key.unshift(1)
+  } else {
+    // even
+    key.unshift(0)
+    key.unshift(0)
+  }
+
+  if (terminator) {
+    key[0] += 2
+  }
+
+  return key
+}
+
+/**
+ * Removes hex prefix of an array of nibbles.
+ * @param val - Array of nibbles
+ * @private
+ */
+export function removeHexPrefix(val: Nibbles): Nibbles {
+  if (val[0] % 2) {
+    val = val.slice(1)
+  } else {
+    val = val.slice(2)
+  }
+
+  return val
+}
+
+/**
+ * Returns true if hex-prefixed path is for a terminating (leaf) node.
+ * @param key - a hex-prefixed array of nibbles
+ * @private
+ */
+export function isTerminator(key: Nibbles): boolean {
+  return key[0] > 1
+}

package/src/util/index.ts

@@ -0,0 +1,3 @@
+export * from './encoding.ts'
+export * from './genesisState.ts'
+export * from './walkController.ts'

package/src/util/nibbles.ts

@@ -0,0 +1,80 @@
+import type { Nibbles } from '../types.ts'
+
+/**
+ * Converts a bytes to a nibble array.
+ * @private
+ * @param key
+ */
+export function bytesToNibbles(key: Uint8Array): Nibbles {
+  const nibbles = [] as Nibbles
+
+  for (let i = 0; i < key.length; i++) {
+    let q = i * 2
+    nibbles[q] = key[i] >> 4
+    ++q
+    nibbles[q] = key[i] % 16
+  }
+
+  return nibbles
+}
+
+/**
+ * Converts a nibble array into bytes.
+ * @private
+ * @param arr - Nibble array
+ */
+export function nibblesTypeToPackedBytes(arr: Nibbles): Uint8Array {
+  const buf = new Uint8Array(arr.length / 2)
+  for (let i = 0; i < buf.length; i++) {
+    let q = i * 2
+    buf[i] = (arr[q] << 4) + arr[++q]
+  }
+  return buf
+}
+
+/**
+ * Compare two nibble array.
+ * * `0` is returned if `n2` === `n1`.
+ * * `1` is returned if `n2` > `n1`.
+ * * `-1` is returned if `n2` < `n1`.
+ * @param n1 - Nibble array
+ * @param n2 - Nibble array
+ */
+export function nibblesCompare(n1: Nibbles, n2: Nibbles) {
+  const cmpLength = Math.min(n1.length, n2.length)
+
+  let res = 0
+  for (let i = 0; i < cmpLength; i++) {
+    if (n1[i] < n2[i]) {
+      res = -1
+      break
+    } else if (n1[i] > n2[i]) {
+      res = 1
+      break
+    }
+  }
+
+  if (res === 0) {
+    if (n1.length < n2.length) {
+      res = -1
+    } else if (n1.length > n2.length) {
+      res = 1
+    }
+  }
+
+  return res
+}
+
+/**
+ * Returns the number of in order matching nibbles of two give nibble arrays.
+ * @private
+ * @param nib1
+ * @param nib2
+ */
+export function matchingNibbleLength(nib1: Nibbles, nib2: Nibbles): number {
+  let i = 0
+  while (nib1[i] === nib2[i] && nib1.length > i) {
+    i++
+  }
+  return i
+}

package/src/util/walkController.ts

@@ -0,0 +1,172 @@
+import { EthereumJSErrorWithoutCode, PrioritizedTaskExecutor } from '@feelyourprotocol/util'
+
+import { BranchMPTNode, ExtensionMPTNode, LeafMPTNode } from '../node/index.ts'
+
+import type { MerklePatriciaTrie } from '../mpt.ts'
+import type { FoundNodeFunction, MPTNode, Nibbles, NodeReferenceOrRawMPTNode } from '../types.ts'
+
+/**
+ * Interface to control how the trie is being traversed. Schedules node visits via a
+ * prioritized task queue and invokes the provided callback for each node.
+ *
+ * Used by {@link MerklePatriciaTrie.findPath}, {@link MerklePatriciaTrie.walkTrie}, etc.
+ */
+export class WalkController {
+  /** The {@link FoundNodeFunction} to call when a node is found. */
+  readonly onNode: FoundNodeFunction
+
+  /** Task executor that prioritizes node visits (shorter paths first). */
+  readonly taskExecutor: PrioritizedTaskExecutor
+
+  /** The trie being walked. */
+  readonly trie: MerklePatriciaTrie
+
+  private _resolvePromise: () => void
+  private _rejectPromise: (error: unknown) => void
+
+  /**
+   * @param onNode - The {@link FoundNodeFunction} to call when a node is found.
+   * @param trie - The trie to walk on.
+   * @param poolSize - The size of the task queue.
+   */
+  private constructor(onNode: FoundNodeFunction, trie: MerklePatriciaTrie, poolSize: number) {
+    this.onNode = onNode
+    this.taskExecutor = new PrioritizedTaskExecutor(poolSize)
+    this.trie = trie
+    this._resolvePromise = () => {}
+    this._rejectPromise = () => {}
+  }
+
+  /**
+   * Creates and starts an async walk over a trie from the given root.
+   * Resolves when all reachable nodes have been visited and no new tasks were scheduled.
+   *
+   * @param onNode - The {@link FoundNodeFunction} to call when a node is found.
+   * @param trie - The trie to walk on.
+   * @param rootHash - The root hash (32-byte keccak) to start walking from.
+   * @param poolSize - Task execution pool size to prevent OOM errors. Defaults to 500.
+   * @returns A Promise that resolves when the walk is finished.
+   */
+  static async newWalk(
+    onNode: FoundNodeFunction,
+    trie: MerklePatriciaTrie,
+    rootHash: Uint8Array,
+    poolSize?: number,
+  ): Promise<void> {
+    const controller = new WalkController(onNode, trie, poolSize ?? 500)
+    await controller._startWalk(rootHash)
+  }
+
+  private async _startWalk(rootHash: Uint8Array): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this._resolvePromise = resolve
+      this._rejectPromise = reject
+      this.trie
+        .lookupNode(rootHash)
+        .then((rootNode) => {
+          this._processNode(rootHash, rootNode, [])
+        })
+        .catch((error: unknown) => {
+          this._rejectPromise(error)
+        })
+    })
+  }
+
+  /**
+   * Runs all children of a node. Priority of these nodes is the key length of the children.
+   * Used when walking an Extension or when exploring all branches of a Branch node.
+   *
+   * @param node - Node to get all children of and call onNode on.
+   * @param currentKeyNibbles - The current key (nibbles) which would yield the `node` when
+   *        trying to get this node with a `get` operation. Defaults to `[]`.
+   * @returns `void`
+   */
+  allChildren(node: MPTNode, currentKeyNibbles: Nibbles = []): void {
+    if (node instanceof LeafMPTNode) {
+      return
+    }
+    let childEntries: [Nibbles, NodeReferenceOrRawMPTNode][]
+    if (node instanceof ExtensionMPTNode) {
+      childEntries = [[node.key(), node.value()]]
+    } else if (node instanceof BranchMPTNode) {
+      childEntries = node
+        .getChildren()
+        .map(([branchIndex, branchRef]) => [[branchIndex], branchRef])
+    } else {
+      return
+    }
+    for (const [keyExtension, childRef] of childEntries) {
+      const childKeyNibbles = currentKeyNibbles.concat(keyExtension)
+      const taskPriority = childKeyNibbles.length
+      this.pushNodeToQueue(childRef, childKeyNibbles, taskPriority)
+    }
+  }
+
+  /**
+   * Pushes a node to the queue. If the queue has capacity, the node is executed immediately,
+   * otherwise it is queued for later execution.
+   *
+   * @param nodeRef - A node reference (32-byte keccak hash or raw encoding) to enqueue.
+   * @param currentKeyNibbles - The current key (nibbles) corresponding to this node. Defaults to `[]`.
+   * @param priority - Optional priority. Defaults to key length.
+   * @returns `void`
+   */
+  pushNodeToQueue(
+    nodeRef: NodeReferenceOrRawMPTNode,
+    currentKeyNibbles: Nibbles = [],
+    priority?: number,
+  ): void {
+    const taskPriority = priority ?? currentKeyNibbles.length
+    this.taskExecutor.executeOrQueue(taskPriority, (taskFinishedCallback: () => void) => {
+      this.trie
+        .lookupNode(nodeRef)
+        .then((decodedNode) => {
+          taskFinishedCallback()
+          this._processNode(nodeRef, decodedNode, currentKeyNibbles)
+        })
+        .catch((error: unknown) => {
+          this._rejectPromise(error)
+        })
+    })
+  }
+
+  /**
+   * Pushes a branch of a certain BranchMPTNode to the event queue.
+   * Used by findPath when following a specific key (only one child index is traversed).
+   *
+   * @param node - The BranchMPTNode to select a branch on.
+   * @param currentKeyNibbles - The current key which leads to the corresponding node. Defaults to `[]`.
+   * @param childIndex - The child index (0–15) to add to the event queue.
+   * @param priority - Optional priority of the event. Defaults to the total key length.
+   * @returns `void`
+   * @throws If `node` is not a BranchMPTNode or if the branch at `childIndex` is empty.
+   */
+  onlyBranchIndex(
+    node: BranchMPTNode,
+    currentKeyNibbles: Nibbles = [],
+    childIndex: number,
+    priority?: number,
+  ): void {
+    if (!(node instanceof BranchMPTNode)) {
+      throw EthereumJSErrorWithoutCode('Expected branch node')
+    }
+    const childRef = node.getBranch(childIndex)
+    if (!childRef) {
+      throw EthereumJSErrorWithoutCode('Could not get branch of childIndex')
+    }
+    const childKeyNibbles = currentKeyNibbles.concat(childIndex)
+    const taskPriority = priority ?? childKeyNibbles.length
+    this.pushNodeToQueue(childRef, childKeyNibbles, taskPriority)
+  }
+
+  private _processNode(
+    nodeRef: NodeReferenceOrRawMPTNode,
+    node: MPTNode | null,
+    currentKeyNibbles: Nibbles,
+  ): void {
+    this.onNode(nodeRef, node, currentKeyNibbles, this)
+    if (this.taskExecutor.finished()) {
+      this._resolvePromise()
+    }
+  }
+}