hypercore 10.0.0-alpha.5 → 10.0.0-alpha.52

package/lib/merkle-tree.js

@@ -1,9 +1,13 @@
 const flat = require('flat-tree')
 const crypto = require('hypercore-crypto')
-const uint64le = require('uint64le')
+const c = require('compact-encoding')
+const Xache = require('xache')
+const b4a = require('b4a')
+const caps = require('./caps')
 
-const BLANK_HASH = Buffer.alloc(32)
-const OLD_TREE = Buffer.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
+const BLANK_HASH = b4a.alloc(32)
+const OLD_TREE = b4a.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
+const TREE_CACHE = 128 // speeds up linear scans by A LOT
 
 class NodeQueue {
   constructor (nodes, extra = null) {
@@ -56,11 +60,11 @@ class MerkleTreeBatch {
   }
 
   signable (hash = this.hash()) {
-    return signable(hash, this.length, this.fork)
+    return caps.treeSignable(hash, this.length, this.fork)
   }
 
-  signedBy (key) {
-    return this.signature !== null && this.tree.crypto.verify(this.signable(), this.signature, key)
+  signableLegacy (hash = this.hash()) {
+    return caps.treeSignableLegacy(hash, this.length, this.fork)
   }
 
   append (buf) {
@@ -138,6 +142,7 @@ class MerkleTreeBatch {
         : this.ancestors
 
       this.tree.truncated = true
+      this.tree.cache = new Xache({ maxSize: this.tree.cache.maxSize })
       truncateMap(this.tree.unflushed, this.ancestors)
       if (this.tree.flushing !== null) truncateMap(this.tree.flushing, this.ancestors)
     }
@@ -212,9 +217,9 @@ class ReorgBatch extends MerkleTreeBatch {
     if (this.want === null) return true
 
     const nodes = []
-    const root = verifyBlock(proof, this.tree.crypto, nodes)
+    const root = verifyTree(proof, this.tree.crypto, nodes)
 
-    if (root === null || !root.hash.equals(this.diff.hash)) return false
+    if (root === null || !b4a.equals(root.hash, this.diff.hash)) return false
 
     this.nodes.push(...nodes)
     return this._update(nodes)
@@ -226,13 +231,14 @@ class ReorgBatch extends MerkleTreeBatch {
 
     let diff = null
     const ite = flat.iterator(this.diff.index)
+    const startingDiff = this.diff
 
     while ((ite.index & 1) !== 0) {
       const left = n.get(ite.leftChild())
       if (!left) break
 
       const existing = await this.tree.get(left.index, false)
-      if (!existing || !existing.hash.equals(left.hash)) {
+      if (!existing || !b4a.equals(existing.hash, left.hash)) {
         diff = left
       } else {
         diff = n.get(ite.sibling())
@@ -241,20 +247,19 @@ class ReorgBatch extends MerkleTreeBatch {
 
     if ((this.diff.index & 1) === 0) return true
     if (diff === null) return false
+    if (startingDiff !== this.diff) return false
 
     return this._updateDiffRoot(diff)
   }
 
   _updateDiffRoot (diff) {
+    if (this.want === null) return true
+
     const spans = flat.spans(diff.index)
     const start = spans[0] / 2
     const end = Math.min(this.treeLength, spans[1] / 2 + 1)
     const len = end - start
 
-    if (this.diff !== null && len >= this.want.end - this.want.start) {
-      return false
-    }
-
     this.ancestors = start
     this.diff = diff
 
@@ -272,11 +277,15 @@ class ReorgBatch extends MerkleTreeBatch {
 }
 
 class ByteSeeker {
-  constructor (tree, bytes) {
+  constructor (tree, bytes, padding = 0) {
     this.tree = tree
     this.bytes = bytes
-    this.start = bytes >= tree.byteLength ? tree.length : 0
-    this.end = bytes < tree.byteLength ? tree.length : 0
+    this.padding = padding
+
+    const size = tree.byteLength - (tree.length * padding)
+
+    this.start = bytes >= size ? tree.length : 0
+    this.end = bytes < size ? tree.length : 0
   }
 
   nodes () {
@@ -287,12 +296,12 @@ class ByteSeeker {
     if (!bytes) return [0, 0]
 
     for (const node of this.tree.roots) { // all async ticks happen once we find the root so safe
-      if (bytes === node.size) {
-        return [flat.rightSpan(node.index) + 2, 0]
-      }
+      let size = node.size
+      if (this.padding > 0) size -= this.padding * flat.countLeaves(node.index)
 
-      if (bytes > node.size) {
-        bytes -= node.size
+      if (bytes === size) return [flat.rightSpan(node.index) + 2, 0]
+      if (bytes > size) {
+        bytes -= size
         continue
       }
 
@@ -301,9 +310,12 @@ class ByteSeeker {
       while ((ite.index & 1) !== 0) {
         const l = await this.tree.get(ite.leftChild(), false)
         if (l) {
-          if (l.size === bytes) return [ite.rightSpan() + 2, 0]
-          if (l.size > bytes) continue
-          bytes -= l.size
+          let size = l.size
+          if (this.padding > 0) size -= this.padding * ite.countLeaves()
+
+          if (size === bytes) return [ite.rightSpan() + 2, 0]
+          if (size > bytes) continue
+          bytes -= size
           ite.sibling()
         } else {
           ite.parent()
@@ -341,13 +353,14 @@ module.exports = class MerkleTree {
 
     this.storage = storage
     this.unflushed = new Map()
+    this.cache = new Xache({ maxSize: TREE_CACHE })
     this.flushing = null
     this.truncated = false
     this.truncateTo = 0
   }
 
   addNode (node) {
-    if (node.size === 0 && node.hash.equals(BLANK_HASH)) node = blankNode(node.index)
+    if (node.size === 0 && b4a.equals(node.hash, BLANK_HASH)) node = blankNode(node.index)
     this.unflushed.set(node.index, node)
   }
 
@@ -355,8 +368,8 @@ module.exports = class MerkleTree {
     return new MerkleTreeBatch(this)
   }
 
-  seek (bytes) {
-    return new ByteSeeker(this, bytes)
+  seek (bytes, padding) {
+    return new ByteSeeker(this, bytes, padding)
   }
 
   hash () {
@@ -364,14 +377,39 @@ module.exports = class MerkleTree {
   }
 
   signable (hash = this.hash()) {
-    return signable(hash, this.length, this.fork)
+    return caps.treeSignable(hash, this.length, this.fork)
   }
 
-  signedBy (key) {
-    return this.signature !== null && this.crypto.verify(this.signable(), this.signature, key)
+  getRoots (length) {
+    const indexes = flat.fullRoots(2 * length)
+    const roots = new Array(indexes.length)
+
+    for (let i = 0; i < indexes.length; i++) {
+      roots[i] = this.get(indexes[i], true)
+    }
+
+    return Promise.all(roots)
+  }
+
+  async upgradeable (length) {
+    const indexes = flat.fullRoots(2 * length)
+    const roots = new Array(indexes.length)
+
+    for (let i = 0; i < indexes.length; i++) {
+      roots[i] = this.get(indexes[i], false)
+    }
+
+    for (const node of await Promise.all(roots)) {
+      if (node === null) return false
+    }
+
+    return true
   }
 
   get (index, error = true) {
+    const c = this.cache.get(index)
+    if (c) return c
+
     let node = this.unflushed.get(index)
 
     if (this.flushing !== null && node === undefined) {
@@ -391,7 +429,7 @@ module.exports = class MerkleTree {
       return Promise.resolve(node)
     }
 
-    return getStoredNode(this.storage, index, error)
+    return getStoredNode(this.storage, index, this.cache, error)
   }
 
   async flush () {
@@ -433,17 +471,23 @@ module.exports = class MerkleTree {
     // TODO: write neighbors together etc etc
     // TODO: bench loading a full disk page and copy to that instead
     return new Promise((resolve, reject) => {
-      const slab = Buffer.allocUnsafe(40 * this.flushing.size)
+      const slab = b4a.allocUnsafe(40 * this.flushing.size)
 
       let error = null
       let missing = this.flushing.size + 1
       let offset = 0
 
       for (const node of this.flushing.values()) {
-        const b = slab.slice(offset, offset += 40)
-        uint64le.encode(node.size, b, 0)
-        node.hash.copy(b, 8)
-        this.storage.write(node.index * 40, b, done)
+        const state = {
+          start: 0,
+          end: 40,
+          buffer: slab.subarray(offset, offset += 40)
+        }
+
+        c.uint64.encode(state, node.size)
+        c.raw.encode(state, node.hash)
+
+        this.storage.write(node.index * 40, state.buffer, done)
       }
 
       done(null)
@@ -458,6 +502,7 @@ module.exports = class MerkleTree {
   }
 
   clear () {
+    this.cache = new Xache({ maxSize: this.cache.maxSize })
     this.truncated = true
     this.truncateTo = 0
     this.roots = []
@@ -510,8 +555,8 @@ module.exports = class MerkleTree {
 
     let unverified = null
 
-    if (proof.block || proof.seek) {
-      unverified = verifyBlock(proof, this.crypto, batch.nodes)
+    if (proof.block || proof.hash || proof.seek) {
+      unverified = verifyTree(proof, this.crypto, batch.nodes)
     }
 
     if (!verifyUpgrade(proof, unverified, batch)) {
@@ -520,7 +565,7 @@ module.exports = class MerkleTree {
 
     for (const root of batch.roots) {
       const existing = await this.get(root.index, false)
-      if (existing && existing.hash.equals(root.hash)) continue
+      if (existing && b4a.equals(existing.hash, root.hash)) continue
       batch._updateDiffRoot(root)
       break
     }
@@ -538,7 +583,7 @@ module.exports = class MerkleTree {
   async verify (proof) {
     const batch = new MerkleTreeBatch(this)
 
-    let unverified = verifyBlock(proof, this.crypto, batch.nodes)
+    let unverified = verifyTree(proof, this.crypto, batch.nodes)
 
     if (proof.upgrade) {
       if (verifyUpgrade(proof, unverified, batch)) {
@@ -548,7 +593,7 @@ module.exports = class MerkleTree {
 
     if (unverified) {
       const verified = await this.get(unverified.index)
-      if (!verified.hash.equals(unverified.hash)) {
+      if (!b4a.equals(verified.hash, unverified.hash)) {
         throw new Error('Invalid checksum at node ' + unverified.index)
       }
     }
@@ -556,89 +601,102 @@ module.exports = class MerkleTree {
     return batch
   }
 
-  async proof ({ block, seek, upgrade }) {
+  async proof ({ block, hash, seek, upgrade }) {
     // Important that this does not throw inbetween making the promise arrays
     // and finalise being called, otherwise there will be lingering promises in the background
 
-    const signature = this.signature
     const fork = this.fork
+    const signature = this.signature
     const head = 2 * this.length
     const from = upgrade ? upgrade.start * 2 : 0
     const to = upgrade ? from + upgrade.length * 2 : head
+    const node = normalizeIndexed(block, hash)
 
     if (from >= to || to > head) {
       throw new Error('Invalid upgrade')
     }
-    if (seek && block && upgrade && block.index * 2 >= from) {
-      throw new Error('Cannot both do a seek and block request when upgrading')
+    if (seek && upgrade && node !== null && node.index >= from) {
+      throw new Error('Cannot both do a seek and block/hash request when upgrading')
     }
 
     let subTree = head
 
     const p = {
-      block: null,
+      node: null,
       seek: null,
       upgrade: null,
       additionalUpgrade: null
     }
 
-    if (block && (!upgrade || block.index < upgrade.start)) {
-      subTree = nodesToRoot(2 * block.index, block.nodes, to)
+    if (node !== null && (!upgrade || node.lastIndex < upgrade.start)) {
+      subTree = nodesToRoot(node.index, node.nodes, to)
       const seekRoot = seek ? await seekUntrustedTree(this, subTree, seek.bytes) : head
-      blockAndSeekProof(this, block, seek, seekRoot, subTree, p)
-    } else if ((block || seek) && upgrade) {
-      subTree = seek ? await seekFromHead(this, to, seek.bytes) : 2 * block.index
+      blockAndSeekProof(this, node, seek, seekRoot, subTree, p)
+    } else if ((node || seek) && upgrade) {
+      subTree = seek ? await seekFromHead(this, to, seek.bytes) : node.index
     }
 
     if (upgrade) {
-      upgradeProof(this, block, seek, from, to, subTree, p)
+      upgradeProof(this, node, seek, from, to, subTree, p)
       if (head > to) additionalUpgradeProof(this, to, head, p)
     }
 
-    try {
-      const result = { fork, block: null, seek: null, upgrade: null }
-
-      if (block) {
-        const nodes = await Promise.all(p.block)
+    const [pNode, pSeek, pUpgrade, pAdditional] = await settleProof(p)
+    const result = { fork, block: null, hash: null, seek: null, upgrade: null }
 
-        result.block = {
-          index: block.index,
-          value: null,
-          nodes
-        }
+    if (block) {
+      result.block = {
+        index: block.index,
+        value: null, // populated upstream, alloc it here for simplicity
+        nodes: pNode
       }
-      if (seek && p.seek !== null) {
-        const nodes = await Promise.all(p.seek)
+    } else if (hash) {
+      result.hash = {
+        index: hash.index,
+        nodes: pNode
+      }
+    }
 
-        result.seek = {
-          bytes: seek.bytes,
-          nodes
-        }
+    if (seek && pSeek !== null) {
+      result.seek = {
+        bytes: seek.bytes,
+        nodes: pSeek
       }
-      if (upgrade) {
-        const nodes = await Promise.all(p.upgrade)
-        const additionalNodes = await Promise.all(p.additionalUpgrade || [])
-
-        result.upgrade = {
-          start: upgrade.start,
-          length: upgrade.length,
-          nodes,
-          additionalNodes,
-          signature
-        }
+    }
+
+    if (upgrade) {
+      result.upgrade = {
+        start: upgrade.start,
+        length: upgrade.length,
+        nodes: pUpgrade,
+        additionalNodes: pAdditional || [],
+        signature
       }
+    }
 
-      return result
-    } catch (err) {
-      // Make sure we await all pending p so don't have background async state...
-      if (p.seek !== null) await Promise.allSettled(p.seek)
-      if (p.block !== null) await Promise.allSettled(p.block)
-      if (p.upgrade !== null) await Promise.allSettled(p.upgrade)
-      if (p.additionalUpgrade !== null) await Promise.allSettled(p.additionalUpgrade)
-      throw err
+    return result
+  }
+
+  // Successor to .nodes()
+  async missingNodes (index) {
+    const head = 2 * this.length
+    const ite = flat.iterator(index)
+
+    // See iterator.rightSpan()
+    const iteRightSpan = ite.index + ite.factor / 2 - 1
+    // If the index is not in the current tree, we do not know how many missing nodes there are...
+    if (iteRightSpan >= head) return 0
+
+    let cnt = 0
+    while (!ite.contains(head) && (await this.get(ite.index, false)) === null) {
+      cnt++
+      ite.parent()
     }
+
+    return cnt
   }
 
+  // Deprecated
   async nodes (index) {
     const head = 2 * this.length
     const ite = flat.iterator(index)
@@ -693,7 +751,7 @@ module.exports = class MerkleTree {
     await new Promise((resolve, reject) => {
       storage.read(0, OLD_TREE.length, (err, buf) => {
         if (err) return resolve()
-        if (buf.equals(OLD_TREE)) return reject(new Error('Storage contains an incompatible merkle tree'))
+        if (b4a.equals(buf, OLD_TREE)) return reject(new Error('Storage contains an incompatible merkle tree'))
         resolve()
       })
     })
@@ -704,7 +762,7 @@ module.exports = class MerkleTree {
 
     const roots = []
     for (const index of flat.fullRoots(2 * length)) {
-      roots.push(await getStoredNode(storage, index, true))
+      roots.push(await getStoredNode(storage, index, null, true))
     }
 
     return new MerkleTree(storage, roots, opts.fork || 0, opts.signature || null)
@@ -713,8 +771,14 @@ module.exports = class MerkleTree {
 
 // All the methods needed for proof verification
 
-function verifyBlock ({ block, seek }, crypto, nodes) {
-  if (!block && (!seek || !seek.nodes.length)) return null
+function verifyTree ({ block, hash, seek }, crypto, nodes) {
+  const untrustedNode = block
+    ? { index: 2 * block.index, value: block.value, nodes: block.nodes }
+    : hash
+      ? { index: hash.index, value: null, nodes: hash.nodes }
+      : null
+
+  if (untrustedNode === null && (!seek || !seek.nodes.length)) return null
 
   let root = null
 
@@ -734,12 +798,12 @@ function verifyBlock ({ block, seek }, crypto, nodes) {
     }
   }
 
-  if (!block) return root
+  if (untrustedNode === null) return root
 
-  const ite = flat.iterator(2 * block.index)
-  const blockHash = block.value && blockNode(crypto, ite.index, block.value)
+  const ite = flat.iterator(untrustedNode.index)
+  const blockHash = untrustedNode.value && blockNode(crypto, ite.index, untrustedNode.value)
 
-  const q = new NodeQueue(block.nodes, root)
+  const q = new NodeQueue(untrustedNode.nodes, root)
 
   root = blockHash || q.shift(ite.index)
   nodes.push(root)
@@ -888,13 +952,13 @@ function seekProof (tree, seekRoot, root, p) {
   }
 }
 
-function blockAndSeekProof (tree, block, seek, seekRoot, root, p) {
-  if (!block) return seekProof(tree, seekRoot, root, p)
+function blockAndSeekProof (tree, node, seek, seekRoot, root, p) {
+  if (!node) return seekProof(tree, seekRoot, root, p)
 
-  const ite = flat.iterator(2 * block.index)
+  const ite = flat.iterator(node.index)
 
-  p.block = []
-  if (!block.value) p.block.push(tree.get(ite.index))
+  p.node = []
+  if (!node.value) p.node.push(tree.get(ite.index))
 
   while (ite.index !== root) {
     ite.sibling()
@@ -902,14 +966,14 @@ function blockAndSeekProof (tree, block, seek, seekRoot, root, p) {
     if (seek && ite.contains(seekRoot) && ite.index !== seekRoot) {
       seekProof(tree, seekRoot, ite.index, p)
     } else {
-      p.block.push(tree.get(ite.index))
+      p.node.push(tree.get(ite.index))
     }
 
     ite.parent()
   }
 }
 
-function upgradeProof (tree, block, seek, from, to, subTree, p) {
+function upgradeProof (tree, node, seek, from, to, subTree, p) {
   if (from === 0) p.upgrade = []
 
   for (const ite = flat.iterator(0); ite.fullRoot(to); ite.nextTree()) {
@@ -928,8 +992,8 @@ function upgradeProof (tree, block, seek, from, to, subTree, p) {
       while (ite.index !== root) {
         ite.sibling()
         if (ite.index > target) {
-          if (p.block === null && p.seek === null && ite.contains(subTree)) {
-            blockAndSeekProof(tree, block, seek, subTree, ite.index, p)
+          if (p.node === null && p.seek === null && ite.contains(subTree)) {
+            blockAndSeekProof(tree, node, seek, subTree, ite.index, p)
           } else {
             p.upgrade.push(tree.get(ite.index))
           }
@@ -946,8 +1010,8 @@ function upgradeProof (tree, block, seek, from, to, subTree, p) {
 
     // if the subtree included is a child of this tree, include that one
     // instead of a dup node
-    if (p.block === null && p.seek === null && ite.contains(subTree)) {
-      blockAndSeekProof(tree, block, seek, subTree, ite.index, p)
+    if (p.node === null && p.seek === null && ite.contains(subTree)) {
+      blockAndSeekProof(tree, node, seek, subTree, ite.index, p)
       continue
     }
 
@@ -1029,7 +1093,7 @@ function blankNode (index) {
 
 // Storage methods
 
-function getStoredNode (storage, index, error) {
+function getStoredNode (storage, index, cache, error) {
   return new Promise((resolve, reject) => {
     storage.read(40 * index, 40, (err, data) => {
       if (err) {
@@ -1038,16 +1102,18 @@ function getStoredNode (storage, index, error) {
         return
       }
 
-      const hash = data.slice(8)
-      const size = uint64le.decode(data, 0)
+      const hash = data.subarray(8)
+      const size = c.decode(c.uint64, data)
 
-      if (size === 0 && Buffer.compare(hash, BLANK_HASH) === 0) {
+      if (size === 0 && b4a.compare(hash, BLANK_HASH) === 0) {
         if (error) reject(new Error('Could not load node: ' + index))
         else resolve(null)
         return
       }
 
-      resolve({ index, size, hash })
+      const node = { index, size, hash }
+      if (cache !== null) cache.set(index, node)
+      resolve(node)
     })
   })
 }
@@ -1066,7 +1132,7 @@ async function autoLength (storage) {
   if (!nodes) return 0
   const ite = flat.iterator(nodes - 1)
   let index = nodes - 1
-  while (await getStoredNode(storage, ite.parent(), false)) index = ite.index
+  while (await getStoredNode(storage, ite.parent(), null, false)) index = ite.index
   return flat.rightSpan(index) / 2 + 1
 }
 
@@ -1087,10 +1153,27 @@ function log2 (n) {
   return res
 }
 
-function signable (hash, length, fork) {
-  const buf = Buffer.alloc(48)
-  hash.copy(buf)
-  uint64le.encode(length, buf, 32)
-  uint64le.encode(fork, buf, 40)
-  return buf
+function normalizeIndexed (block, hash) {
+  if (block) return { value: true, index: block.index * 2, nodes: block.nodes, lastIndex: block.index }
+  if (hash) return { value: false, index: hash.index, nodes: hash.nodes, lastIndex: flat.rightSpan(hash.index) / 2 }
+  return null
+}
+
+async function settleProof (p) {
+  const result = [
+    p.node && Promise.all(p.node),
+    p.seek && Promise.all(p.seek),
+    p.upgrade && Promise.all(p.upgrade),
+    p.additionalUpgrade && Promise.all(p.additionalUpgrade)
+  ]
+
+  try {
+    return await Promise.all(result)
+  } catch (err) {
+    if (p.node) await Promise.allSettled(p.node)
+    if (p.seek) await Promise.allSettled(p.seek)
+    if (p.upgrade) await Promise.allSettled(p.upgrade)
+    if (p.additionalUpgrade) await Promise.allSettled(p.additionalUpgrade)
+    throw err
+  }
 }