hypercore 11.0.29 → 11.0.30

package/index.js

@@ -14,8 +14,10 @@ const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
+const caps = require('./lib/caps')
 const { manifestHash, createManifest } = require('./lib/verifier')
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
+const { MerkleTree } = require('./lib/merkle-tree')
 const {
   ASSERTION,
   BAD_ARGUMENT,
@@ -617,24 +619,6 @@ class Hypercore extends EventEmitter {
     this.emit('migrate', this.key)
   }
 
-  createTreeBatch () {
-    return this.state.createTreeBatch()
-  }
-
-  async restoreBatch (length, additionalBlocks = []) {
-    if (this.opened === false) await this.opening
-    const batch = this.state.createTreeBatch()
-
-    if (length > batch.length + additionalBlocks.length) {
-      throw BAD_ARGUMENT('Insufficient additional blocks were passed')
-    }
-
-    let i = 0
-    while (batch.length < length) batch.append(additionalBlocks[i++])
-
-    return length < batch.length ? batch.restore(length) : batch
-  }
-
   findingPeers () {
     this._findingPeers++
     if (this.core !== null && !this.closing) this.core.replicator.findingPeers++
@@ -692,8 +676,7 @@ class Hypercore extends EventEmitter {
     if (this.opened === false) await this.opening
     if (!isValidIndex(bytes)) throw ASSERTION('seek is invalid')
 
-    const tree = (opts && opts.tree) || this.state.core.tree
-    const s = tree.seek(this.state, bytes, this.padding)
+    const s = MerkleTree.seek(this.state, bytes, this.padding)
 
     const offset = await s.update()
     if (offset) return offset
@@ -800,8 +783,6 @@ class Hypercore extends EventEmitter {
   }
 
   async _get (index, opts) {
-    if (this.core.isFlushing) await this.core.flushed()
-
     const block = await readBlock(this.state.storage.read(), index)
 
     if (block !== null) return block
@@ -935,16 +916,30 @@ class Hypercore extends EventEmitter {
     return this.state.append(buffers, { keyPair, signature, preappend })
   }
 
-  async treeHash (length) {
-    if (length === undefined) {
-      await this.ready()
-      length = this.state.length
-    }
+  async signable (length = -1, fork = -1) {
+    if (this.opened === false) await this.opening
+    if (length === -1) length = this.length
+    if (fork === -1) fork = this.fork
 
-    const roots = await this.state.tree.getRoots(length)
+    return caps.treeSignable(this.key, await this.treeHash(length), length, fork)
+  }
+
+  async treeHash (length = -1) {
+    if (this.opened === false) await this.opening
+    if (length === -1) length = this.length
+
+    const roots = await MerkleTree.getRoots(this.state, length)
     return crypto.tree(roots)
   }
 
+  async proof (opts) {
+    if (this.opened === false) await this.opening
+    const rx = this.state.storage.read()
+    const promise = MerkleTree.proof(this.state, rx)
+    rx.tryFlush()
+    return promise
+  }
+
   registerExtension (name, handlers = {}) {
     if (this.extensions.has(name)) {
       const ext = this.extensions.get(name)

package/lib/copy-prologue.js

@@ -14,7 +14,7 @@ module.exports = copyPrologue
 async function copyPrologue (src, dst) {
   const prologue = dst.header.manifest.prologue
 
-  if (src.tree.length < prologue.length || prologue.length === 0) return
+  if (src.length < prologue.length || prologue.length === 0) return
 
   const stack = []
   const roots = flat.fullRoots(prologue.length * 2)

package/lib/core.js

@@ -4,7 +4,6 @@ const unslab = require('unslab')
 const z32 = require('z32')
 const Mutex = require('./mutex')
 const { MerkleTree, ReorgBatch } = require('./merkle-tree')
-const BlockStore = require('./block-store')
 const BitInterlude = require('./bit-interlude')
 const Bitfield = require('./bitfield')
 const RemoteBitfield = require('./remote-bitfield')
@@ -35,8 +34,6 @@ module.exports = class Core {
     this.preupdate = null
     this.header = null
     this.compat = false
-    this.tree = null
-    this.blocks = null
     this.bitfield = null
     this.verifier = null
     this.truncating = 0
@@ -54,9 +51,6 @@ module.exports = class Core {
     this.closed = false
 
     this._manifestFlushed = false
-    this._onflush = null
-    this._flushing = null
-    this._activeBatch = null
     this._bitfield = null
     this._verifies = null
     this._verifiesFlushed = null
@@ -239,22 +233,20 @@ module.exports = class Core {
 
     const prologue = header.manifest ? header.manifest.prologue : null
 
-    const tree = await MerkleTree.open(storage)
     const bitfield = await Bitfield.open(storage, header.tree.length)
-    const blocks = new BlockStore(storage)
 
     const treeInfo = {
       fork: header.tree.fork,
       length: header.tree.length,
       signature: header.tree.signature,
-      roots: header.tree.length ? await tree.getRoots(header.tree.length) : [],
+      roots: header.tree.length ? await MerkleTree.getRootsFromStorage(storage, header.tree.length) : [],
       prologue
     }
 
     if (overwrite) {
       const tx = storage.write()
-      tree.clear(tx)
-      blocks.clear(tx)
+      tx.deleteTreeNodeRange(0, -1)
+      tx.deleteBlockRange(0, -1)
       bitfield.clear(tx)
       await tx.flush()
     }
@@ -282,11 +274,9 @@ module.exports = class Core {
     this.storage = storage
     this.header = header
     this.compat = compat
-    this.tree = tree
-    this.blocks = blocks
     this.bitfield = bitfield
     this.verifier = verifier
-    this.state = new SessionState(this, null, storage, this.blocks, tree, treeInfo, null)
+    this.state = new SessionState(this, null, storage, treeInfo, null)
 
     if (this.key === null) this.key = this.header.key
     if (this.discoveryKey === null) this.discoveryKey = crypto.discoveryKey(this.key)
@@ -323,8 +313,7 @@ module.exports = class Core {
 
         const tx = this.state.createWriteBatch()
         this._setManifest(tx, Verifier.createManifest(manifest), null)
-
-        if (await this.state.flush(tx)) this.replicator.onupgrade()
+        await this.state.flush()
       }
     } finally {
       this.state._unlock()
@@ -377,10 +366,6 @@ module.exports = class Core {
     }
   }
 
-  get isFlushing () {
-    return !!(this._flushing || this.state._activeBatch)
-  }
-
   flushed () {
     return this.state.flushed()
   }
@@ -392,7 +377,7 @@ module.exports = class Core {
 
     if (this.state.length > treeLength) {
       for (const root of this.state.roots) {
-        const batchRoot = await state.tree.get(root.index)
+        const batchRoot = await MerkleTree.get(state, root.index)
         if (batchRoot.size !== root.size || !b4a.equals(batchRoot.hash, root.hash)) {
           return false
         }
@@ -487,7 +472,7 @@ module.exports = class Core {
 
       const ranges = bits.flush(tx, this.bitfield)
 
-      await this.state.flush(tx)
+      await this.state.flush()
 
       for (const { start, end, value } of ranges) {
         this._setBitfieldRanges(start, end, value)
@@ -514,7 +499,7 @@ module.exports = class Core {
       return false
     }
 
-    const batch = this.tree.verifyFullyRemote(proof, this.state)
+    const batch = MerkleTree.verifyFullyRemote(this.state, proof)
 
     try {
       this._verifyBatchUpgrade(batch, proof.manifest)
@@ -530,13 +515,13 @@ module.exports = class Core {
         this._setManifest(tx, proof.manifest, null)
       }
 
-      await this.state.flush(tx)
+      await this.state.flush()
     } finally {
       this.state.mutex.unlock()
     }
 
     const remoteTreeHash = crypto.tree(proof.upgrade.nodes)
-    const localTreeHash = crypto.tree(await this.tree.getRoots(proof.upgrade.length))
+    const localTreeHash = crypto.tree(await MerkleTree.getRootsFromStorage(this.storage, proof.upgrade.length))
 
     if (b4a.equals(localTreeHash, remoteTreeHash)) return false
 
@@ -545,8 +530,8 @@ module.exports = class Core {
   }
 
   async verifyReorg (proof) {
-    const batch = new ReorgBatch(this.tree, this.state)
-    await this.tree.reorg(proof, batch)
+    const batch = new ReorgBatch(this.state)
+    await MerkleTree.reorg(this.state, proof, batch)
     this._verifyBatchUpgrade(batch, proof.manifest)
     return batch
   }
@@ -557,7 +542,7 @@ module.exports = class Core {
     // but this is easy atm (and the above layer will just retry)
     if (proof.fork !== this.state.fork) return false
 
-    const batch = await this.tree.verify(proof, this.state)
+    const batch = await MerkleTree.verify(this.state, proof)
     if (!batch.commitable()) return false
 
     const value = (proof.block && proof.block.value) || null

package/lib/merkle-tree.js

@@ -36,7 +36,7 @@ class NodeQueue {
 }
 
 class MerkleTreeBatch {
-  constructor (tree, session) {
+  constructor (session) {
     this.fork = session.fork
     this.roots = [...session.roots]
     this.length = session.length
@@ -50,8 +50,7 @@ class MerkleTreeBatch {
     this.truncated = false
     this.treeLength = session.length
     this.treeFork = session.fork
-    this.tree = tree
-    this.storage = tree.storage
+    this.storage = session.storage
     this.session = session
     this.nodes = []
     this.upgraded = false
@@ -120,7 +119,7 @@ class MerkleTreeBatch {
   }
 
   clone () {
-    const b = new MerkleTreeBatch(this.tree, this.session)
+    const b = new MerkleTreeBatch(this.session)
 
     b.fork = this.fork
     b.roots = [...this.roots]
@@ -158,11 +157,12 @@ class MerkleTreeBatch {
       if (n.index === index) return n
     }
 
-    return this.tree.get(index, error)
+    return MerkleTree.get(this.session, index, error)
   }
 
+  // deprecated, use sssion proof instead
   proof (batch, { block, hash, seek, upgrade }) {
-    return generateProof(batch, this, block, hash, seek, upgrade)
+    return generateProof(this.session, batch, block, hash, seek, upgrade)
   }
 
   verifyUpgrade (proof) {
@@ -287,7 +287,7 @@ class MerkleTreeBatch {
   async restore (length) {
     if (length === this.length) return this
 
-    const roots = unslabNodes(await this.tree.getRoots(length))
+    const roots = unslabNodes(await MerkleTree.getRootsFromStorage(this.storage, length))
 
     this.roots = roots
     this.length = length
@@ -301,8 +301,8 @@ class MerkleTreeBatch {
 }
 
 class ReorgBatch extends MerkleTreeBatch {
-  constructor (tree, session) {
-    super(tree, session)
+  constructor (session) {
+    super(session)
 
     this.roots = []
     this.length = 0
@@ -347,7 +347,7 @@ class ReorgBatch extends MerkleTreeBatch {
       const left = n.get(ite.leftChild())
       if (!left) break
 
-      const existing = await this.tree.get(left.index, false)
+      const existing = await MerkleTree.get(this.session, left.index, false)
       if (!existing || !b4a.equals(existing.hash, left.hash)) {
         diff = left
       } else {
@@ -387,16 +387,15 @@ class ReorgBatch extends MerkleTreeBatch {
 }
 
 class ByteSeeker {
-  constructor (tree, session, bytes, padding = 0) {
-    this.tree = tree
+  constructor (session, bytes, padding = 0) {
     this.session = session
     this.bytes = bytes
     this.padding = padding
 
-    const size = tree.byteLength - (tree.length * padding)
+    const size = session.byteLength - (session.length * padding)
 
-    this.start = bytes >= size ? tree.length : 0
-    this.end = bytes < size ? tree.length : 0
+    this.start = bytes >= size ? session.length : 0
+    this.end = bytes < size ? session.length : 0
   }
 
   async _seek (bytes) {
@@ -414,7 +413,7 @@ class ByteSeeker {
       const ite = flat.iterator(node.index)
 
       while ((ite.index & 1) !== 0) {
-        const l = await this.tree.get(ite.leftChild(), false)
+        const l = await MerkleTree.get(this.session, ite.leftChild(), false)
 
         if (l) {
           const size = getUnpaddedSize(l, this.padding, ite)
@@ -449,9 +448,9 @@ class ByteSeeker {
 }
 
 class TreeProof {
-  constructor (tree, block, hash, seek, upgrade) {
-    this.fork = tree.fork
-    this.signature = tree.signature
+  constructor (session, block, hash, seek, upgrade) {
+    this.fork = session.fork
+    this.signature = session.signature
 
     this.block = block
     this.hash = hash
@@ -508,10 +507,6 @@ class TreeProof {
 }
 
 class MerkleTree {
-  constructor (storage) {
-    this.storage = storage
-  }
-
   static hash (s) {
     return unslab(crypto.tree(s.roots))
   }
@@ -528,14 +523,14 @@ class MerkleTree {
     return totalSpan(roots)
   }
 
-  clone (storage) {
-    return new MerkleTree(storage)
+  static getRoots (session, length) {
+    return MerkleTree.getRootsFromStorage(session.storage, length)
   }
 
-  getRoots (length) {
+  static getRootsFromStorage (storage, length) {
     const indexes = flat.fullRoots(2 * length)
     const roots = new Array(indexes.length)
-    const readBatch = this.storage.read()
+    const readBatch = storage.read()
 
     for (let i = 0; i < indexes.length; i++) {
       roots[i] = readBatch.getTreeNode(indexes[i], true)
@@ -546,7 +541,7 @@ class MerkleTree {
     return Promise.all(roots)
   }
 
-  getNeededNodes (length, start, end) {
+  static getNeededNodes (session, length, start, end) {
     const nodes = new Map()
     const head = length * 2
 
@@ -555,7 +550,7 @@ class MerkleTree {
 
       while (true) {
         if (nodes.has(ite.index)) break
-        nodes.set(ite.index, this.get(ite.index, true))
+        nodes.set(ite.index, this.get(session, ite.index, true))
 
         const sibling = ite.sibling()
 
@@ -563,17 +558,17 @@ class MerkleTree {
         if (ite.contains(head)) break
 
         if (nodes.has(sibling)) break
-        nodes.set(sibling, this.get(sibling, true))
+        nodes.set(sibling, this.get(session, sibling, true))
       }
     }
 
     return Promise.all([...nodes.values()])
   }
 
-  async upgradeable (length) {
+  static async upgradeable (session, length) {
     const indexes = flat.fullRoots(2 * length)
     const roots = new Array(indexes.length)
-    const readBatch = this.storage.read()
+    const readBatch = session.storage.read()
 
     for (let i = 0; i < indexes.length; i++) {
       roots[i] = readBatch.getTreeNode(indexes[i], false)
@@ -588,24 +583,17 @@ class MerkleTree {
     return true
   }
 
-  seek (session, bytes, padding) {
-    return new ByteSeeker(this, session, bytes, padding)
+  static seek (session, bytes, padding) {
+    return new ByteSeeker(session, bytes, padding)
   }
 
-  get (index, error = true, readBatch = null) {
+  static get (session, index, error = true, readBatch = null) {
     if (readBatch) return readBatch.getTreeNode(index, error)
 
-    return getTreeNode(this.storage, index, error)
+    return getTreeNode(session.storage, index, error)
   }
 
-  clear (tx) {
-    this.truncated = true
-    this.truncateTo = 0
-
-    return tx.deleteTreeNodeRange(0, -1)
-  }
-
-  async truncate (length, batch, fork = batch.fork) {
+  static async truncate (session, length, batch, fork = batch.fork) {
     const head = length * 2
     const fullRoots = flat.fullRoots(head)
 
@@ -614,7 +602,7 @@ class MerkleTree {
       if (i < batch.roots.length && batch.roots[i].index === root) continue
 
       while (batch.roots.length > i) batch.roots.pop()
-      batch.roots.push(unslabNode(await this.get(root)))
+      batch.roots.push(unslabNode(await this.get(session, root)))
     }
 
     while (batch.roots.length > fullRoots.length) {
@@ -630,7 +618,7 @@ class MerkleTree {
     return batch
   }
 
-  async reorg (proof, batch) {
+  static async reorg (session, proof, batch) {
     let unverified = null
 
     if (proof.block || proof.hash || proof.seek) {
@@ -642,7 +630,7 @@ class MerkleTree {
     }
 
     for (const root of batch.roots) {
-      const existing = await this.get(root.index, false)
+      const existing = await MerkleTree.get(session, root.index, false)
       if (existing && b4a.equals(existing.hash, root.hash)) continue
       batch._updateDiffRoot(root)
       break
@@ -658,9 +646,9 @@ class MerkleTree {
     return batch
   }
 
-  verifyFullyRemote (proof, session) {
+  static verifyFullyRemote (session, proof) {
     // TODO: impl this less hackishly
-    const batch = new MerkleTreeBatch(this, session)
+    const batch = new MerkleTreeBatch(session)
 
     batch.fork = proof.fork
     batch.roots = []
@@ -679,8 +667,8 @@ class MerkleTree {
     return batch
   }
 
-  async verify (proof, session) {
-    const batch = new MerkleTreeBatch(this, session)
+  static async verify (session, proof) {
+    const batch = new MerkleTreeBatch(session)
 
     let unverified = verifyTree(proof, batch.nodes)
 
@@ -691,7 +679,7 @@ class MerkleTree {
     }
 
     if (unverified) {
-      const verified = await this.get(unverified.index)
+      const verified = await MerkleTree.get(session, unverified.index)
       if (!b4a.equals(verified.hash, unverified.hash)) {
         throw INVALID_CHECKSUM('Invalid checksum at node ' + unverified.index)
       }
@@ -700,12 +688,11 @@ class MerkleTree {
     return batch
   }
 
-  proof (rx, batch, { block, hash, seek, upgrade }) {
-    return generateProof(rx, batch, block, hash, seek, upgrade)
+  static proof (session, rx, { block, hash, seek, upgrade }) {
+    return generateProof(session, rx, block, hash, seek, upgrade)
   }
 
-  // Successor to .nodes()
-  async missingNodes (index, length) {
+  static async missingNodes (session, index, length) {
     const head = 2 * length
     const ite = flat.iterator(index)
 
@@ -716,7 +703,7 @@ class MerkleTree {
 
     let cnt = 0
     // TODO: we could prop use a read batch here and do this in blocks of X for perf
-    while (!ite.contains(head) && !(await hasTreeNode(this.storage, ite.index))) {
+    while (!ite.contains(head) && !(await hasTreeNode(session.storage, ite.index))) {
       cnt++
       ite.parent()
     }
@@ -724,27 +711,16 @@ class MerkleTree {
     return cnt
   }
 
-  // Deprecated
-  async nodes (index) {
-    const head = 2 * this.length
-    const ite = flat.iterator(index)
-
-    let cnt = 0
-    while (!ite.contains(head) && (await this.get(ite.index, false)) === null) {
-      cnt++
-      ite.parent()
-    }
-
-    return cnt
+  static byteOffset (session, index) {
+    return getByteOffsetSession(session, index, null)
   }
 
-  static async open (storage, length, opts = {}) {
-    const roots = []
-    for (const index of flat.fullRoots(2 * length)) {
-      roots.push(unslabNode(await getTreeNode(storage, index, true)))
-    }
-
-    return new MerkleTree(storage, roots, opts.fork || 0, opts.signature || null, opts.prologue || null)
+  static byteRange (session, index) {
+    const rx = session.storage.read()
+    const offset = getByteOffsetSession(session, index, rx)
+    const size = getNodeSize(index, rx)
+    rx.tryFlush()
+    return Promise.all([offset, size])
   }
 }
 
@@ -758,14 +734,44 @@ async function getNodeSize (index, readBatch) {
   return (await readBatch.getTreeNode(index, true)).size
 }
 
-function getByteOffsetTree (tree, index, readBatch) {
+async function getByteOffsetSession (session, index, readBatch) {
+  if (index === 2 * session.length) return session.byteLength
+
+  const treeNodes = readBatch === null
+    ? await getByteOffsetBatchFlush(session.roots, index, session.storage.read())
+    : await getByteOffsetBatch(session.roots, index, readBatch)
+
+  let offset = 0
+  for (const node of treeNodes) offset += node.size
+
+  return offset
+}
+
+async function getByteOffset (tree, index, readBatch) {
+  if (index === 2 * tree.length) return tree.byteLength
+
+  const treeNodes = await getByteOffsetBatch(tree.roots, index, readBatch)
+
+  let offset = 0
+  for (const node of treeNodes) offset += node.size
+
+  return offset
+}
+
+function getByteOffsetBatchFlush (roots, index, readBatch) {
+  const treeNodes = getByteOffsetBatch(roots, index, readBatch)
+  readBatch.tryFlush()
+  return treeNodes
+}
+
+function getByteOffsetBatch (roots, index, readBatch) {
   if ((index & 1) === 1) index = flat.leftSpan(index)
 
   let head = 0
 
   const promises = []
 
-  for (const node of tree.roots) { // all async ticks happen once we find the root so safe
+  for (const node of roots) { // all async ticks happen once we find the root so safe
     head += 2 * ((node.index - head) + 1)
 
     if (index >= head) {
@@ -779,7 +785,7 @@ function getByteOffsetTree (tree, index, readBatch) {
       if (index < ite.index) {
         ite.leftChild()
       } else {
-        promises.push(tree.get(ite.leftChild(), true, readBatch))
+        promises.push(readBatch.getTreeNode(ite.leftChild(), true))
         ite.sibling()
       }
     }
@@ -790,17 +796,6 @@ function getByteOffsetTree (tree, index, readBatch) {
   throw ASSERTION('Failed to find offset')
 }
 
-async function getByteOffset (tree, index, readBatch) {
-  if (index === 2 * tree.length) return tree.byteLength
-
-  const treeNodes = await getByteOffsetTree(tree, index, readBatch)
-
-  let offset = 0
-  for (const node of treeNodes) offset += node.size
-
-  return offset
-}
-
 function getByteRange (tree, index, readBatch) {
   const head = 2 * tree.length
   if (((index & 1) === 0 ? index : flat.rightSpan(index)) >= head) {
@@ -935,12 +930,13 @@ function verifyUpgrade ({ fork, upgrade }, blockRoot, batch) {
   return q.extra === null
 }
 
-async function seekFromHead (tree, head, bytes, padding) {
+async function seekFromHead (session, head, bytes, padding) {
   const roots = flat.fullRoots(head)
 
   for (let i = 0; i < roots.length; i++) {
     const root = roots[i]
-    const node = await tree.get(root, true)
+    const node = await MerkleTree.get(session, root, false)
+    if (node === null) throw new Error('Tree node missing')
     const size = getUnpaddedSize(node, padding, null)
 
     if (bytes === size) return root
@@ -949,7 +945,7 @@ async function seekFromHead (tree, head, bytes, padding) {
       continue
     }
 
-    return seekTrustedTree(tree, root, bytes, padding)
+    return seekTrustedTree(session, root, bytes, padding)
   }
 
   return head
@@ -957,13 +953,13 @@ async function seekFromHead (tree, head, bytes, padding) {
 
 // trust that bytes are within the root tree and find the block at bytes
 
-async function seekTrustedTree (tree, root, bytes, padding) {
+async function seekTrustedTree (session, root, bytes, padding) {
   if (!bytes) return root
 
   const ite = flat.iterator(root)
 
   while ((ite.index & 1) !== 0) {
-    const l = await tree.get(ite.leftChild(), false)
+    const l = await MerkleTree.get(session, ite.leftChild(), false)
 
     if (l) {
       const size = getUnpaddedSize(l, padding, ite)
@@ -982,60 +978,61 @@ async function seekTrustedTree (tree, root, bytes, padding) {
 
 // try to find the block at bytes without trusting that is *is* within the root passed
 
-async function seekUntrustedTree (tree, root, bytes, padding) {
-  const offset = await tree.byteOffset(root) - (padding ? padding * flat.leftSpan(root) / 2 : 0)
+async function seekUntrustedTree (session, root, bytes, padding) {
+  const offset = await getByteOffsetSession(session, root, null) - (padding ? padding * flat.leftSpan(root) / 2 : 0)
 
   if (offset > bytes) throw INVALID_OPERATION('Invalid seek')
   if (offset === bytes) return root
 
   bytes -= offset
 
-  const node = await tree.get(root, true)
+  const node = await MerkleTree.get(session, root, false)
+  if (node === null) throw new Error('Tree node missing')
 
   if (getUnpaddedSize(node, padding, null) <= bytes) throw INVALID_OPERATION('Invalid seek')
 
-  return seekTrustedTree(tree, root, bytes, padding)
+  return seekTrustedTree(session, root, bytes, padding)
 }
 
 // Below is proof production, ie, construct proofs to verify a request
 // Note, that all these methods are sync as we can statically infer which nodes
 // are needed for the remote to verify given they arguments they passed us
 
-function seekProof (tree, batch, seekRoot, root, p) {
+function seekProof (session, batch, seekRoot, root, p) {
   const ite = flat.iterator(seekRoot)
 
   p.seek = []
-  p.seek.push(tree.get(ite.index, true, batch))
+  p.seek.push(MerkleTree.get(session, ite.index, true, batch))
 
   while (ite.index !== root) {
     ite.sibling()
-    p.seek.push(tree.get(ite.index, true, batch))
+    p.seek.push(MerkleTree.get(session, ite.index, true, batch))
     ite.parent()
   }
 }
 
-function blockAndSeekProof (tree, batch, node, seek, seekRoot, root, p) {
-  if (!node) return seekProof(tree, batch, seekRoot, root, p)
+function blockAndSeekProof (session, batch, node, seek, seekRoot, root, p) {
+  if (!node) return seekProof(session, batch, seekRoot, root, p)
 
   const ite = flat.iterator(node.index)
 
   p.node = []
-  if (!node.value) p.node.push(tree.get(ite.index, true, batch))
+  if (!node.value) p.node.push(MerkleTree.get(session, ite.index, true, batch))
 
   while (ite.index !== root) {
     ite.sibling()
 
     if (seek && ite.contains(seekRoot) && ite.index !== seekRoot) {
-      seekProof(tree, batch, seekRoot, ite.index, p)
+      seekProof(session, batch, seekRoot, ite.index, p)
     } else {
-      p.node.push(tree.get(ite.index, true, batch))
+      p.node.push(MerkleTree.get(session, ite.index, true, batch))
     }
 
     ite.parent()
   }
 }
 
-function upgradeProof (tree, batch, node, seek, from, to, subTree, p) {
+function upgradeProof (session, batch, node, seek, from, to, subTree, p) {
   if (from === 0) p.upgrade = []
 
   for (const ite = flat.iterator(0); ite.fullRoot(to); ite.nextTree()) {
@@ -1055,7 +1052,7 @@ function upgradeProof (tree, batch, node, seek, from, to, subTree, p) {
         ite.sibling()
         if (ite.index > target) {
           if (p.node === null && p.seek === null && ite.contains(subTree)) {
-            blockAndSeekProof(tree, batch, node, seek, subTree, ite.index, p)
+            blockAndSeekProof(session, batch, node, seek, subTree, ite.index, p)
           } else {
             p.upgrade.push(batch.getTreeNode(ite.index, true))
           }
@@ -1073,16 +1070,16 @@ function upgradeProof (tree, batch, node, seek, from, to, subTree, p) {
     // if the subtree included is a child of this tree, include that one
     // instead of a dup node
     if (p.node === null && p.seek === null && ite.contains(subTree)) {
-      blockAndSeekProof(tree, batch, node, seek, subTree, ite.index, p)
+      blockAndSeekProof(session, batch, node, seek, subTree, ite.index, p)
       continue
     }
 
     // add root (can be optimised since the root might be in tree.roots)
-    p.upgrade.push(tree.get(ite.index, true, batch))
+    p.upgrade.push(MerkleTree.get(session, ite.index, true, batch))
   }
 }
 
-function additionalUpgradeProof (tree, batch, from, to, p) {
+function additionalUpgradeProof (session, batch, from, to, p) {
   if (from === 0) p.additionalUpgrade = []
 
   for (const ite = flat.iterator(0); ite.fullRoot(to); ite.nextTree()) {
@@ -1101,7 +1098,7 @@ function additionalUpgradeProof (tree, batch, from, to, p) {
       while (ite.index !== root) {
         ite.sibling()
         if (ite.index > target) {
-          p.additionalUpgrade.push(tree.get(ite.index, true, batch))
+          p.additionalUpgrade.push(MerkleTree.get(session, ite.index, true, batch))
         }
         ite.parent()
       }
@@ -1114,7 +1111,7 @@ function additionalUpgradeProof (tree, batch, from, to, p) {
     }
 
     // add root (can be optimised since the root is in tree.roots)
-    p.additionalUpgrade.push(tree.get(ite.index, true, batch))
+    p.additionalUpgrade.push(MerkleTree.get(session, ite.index, true, batch))
   }
 }
 
@@ -1200,22 +1197,22 @@ async function settleProof (p) {
 }
 
 // tree can be either the merkle tree or a merkle tree batch
-async function generateProof (readBatch, tree, block, hash, seek, upgrade) {
+async function generateProof (session, readBatch, block, hash, seek, upgrade) {
   // Important that this does not throw inbetween making the promise arrays
   // and finalise being called, otherwise there will be lingering promises in the background
 
-  if (tree.prologue && upgrade) {
-    upgrade.start = upgrade.start < tree.prologue.length ? 0 : upgrade.start
-    upgrade.length = upgrade.start < tree.prologue.length ? tree.prologue.length : upgrade.length
+  if (session.prologue && upgrade) {
+    upgrade.start = upgrade.start < session.prologue.length ? 0 : upgrade.start
+    upgrade.length = upgrade.start < session.prologue.length ? session.prologue.length : upgrade.length
   }
 
-  const head = 2 * tree.length
+  const head = 2 * session.length
   const from = upgrade ? upgrade.start * 2 : 0
   const to = upgrade ? from + upgrade.length * 2 : head
   const node = normalizeIndexed(block, hash)
 
   // can't do anything as we have no data...
-  if (head === 0) return new TreeProof(tree, null, null, null, null)
+  if (head === 0) return new TreeProof(session, null, null, null, null)
 
   if (from >= to || to > head) {
     throw INVALID_OPERATION('Invalid upgrade')
@@ -1226,19 +1223,19 @@ async function generateProof (readBatch, tree, block, hash, seek, upgrade) {
 
   let subTree = head
 
-  const p = new TreeProof(tree, block, hash, seek, upgrade)
+  const p = new TreeProof(session, block, hash, seek, upgrade)
 
   if (node !== null && (!upgrade || node.lastIndex < upgrade.start)) {
     subTree = nodesToRoot(node.index, node.nodes, to)
-    const seekRoot = seek ? await seekUntrustedTree(tree, subTree, seek.bytes, seek.padding) : head
-    blockAndSeekProof(tree, readBatch, node, seek, seekRoot, subTree, p.pending)
+    const seekRoot = seek ? await seekUntrustedTree(session, subTree, seek.bytes, seek.padding) : head
+    blockAndSeekProof(session, readBatch, node, seek, seekRoot, subTree, p.pending)
   } else if ((node || seek) && upgrade) {
-    subTree = seek ? await seekFromHead(tree, to, seek.bytes, seek.padding) : node.index
+    subTree = seek ? await seekFromHead(session, to, seek.bytes, seek.padding) : node.index
   }
 
   if (upgrade) {
-    upgradeProof(tree, readBatch, node, seek, from, to, subTree, p.pending)
-    if (head > to) additionalUpgradeProof(tree, readBatch, to, head, p.pending)
+    upgradeProof(session, readBatch, node, seek, from, to, subTree, p.pending)
+    if (head > to) additionalUpgradeProof(session, readBatch, to, head, p.pending)
   }
 
   return p

package/lib/multisig.js

@@ -1,6 +1,7 @@
 const c = require('compact-encoding')
 const b4a = require('b4a')
 const flat = require('flat-tree')
+const { MerkleTree } = require('./merkle-tree')
 const { multiSignature, multiSignaturev0 } = require('./messages')
 
 module.exports = {
@@ -36,8 +37,7 @@ async function partialSignature (core, signer, from, to = core.state.length, sig
 
 async function upgradeNodes (core, from, to) {
   const rx = core.state.storage.read()
-  const treeBatch = core.state.createTreeBatch()
-  const p = await core.state.tree.proof(rx, treeBatch, { upgrade: { start: from, length: to - from } })
+  const p = await MerkleTree.proof(core.state, rx, { upgrade: { start: from, length: to - from } })
   rx.tryFlush()
   return (await p.settle()).upgrade.nodes
 }

package/lib/replicator.js

@@ -19,7 +19,7 @@
     - Which blocks does the Peer have available (tracked in remoteBitfield)
     - Which blocks are you actively looking for from this peer (tracked in missingBlocks)
     - How many blocks are currently inflight (tracked in inflight)
-  The Peer uses this information to decide which blocks to request form the peer in response to _requestRange requests and the like.
+  The Peer uses this information to decide which blocks to request from the peer in response to _requestRange requests and the like.
 */
 
 const b4a = require('b4a')
@@ -29,6 +29,7 @@ const flatTree = require('flat-tree')
 const ReceiverQueue = require('./receiver-queue')
 const HotswapQueue = require('./hotswap-queue')
 const RemoteBitfield = require('./remote-bitfield')
+const { MerkleTree } = require('./merkle-tree')
 const { REQUEST_CANCELLED, REQUEST_TIMEOUT, INVALID_CAPABILITY, SNAPSHOT_NOT_AVAILABLE } = require('hypercore-errors')
 const m = require('./messages')
 const caps = require('./caps')
@@ -676,7 +677,7 @@ class Peer {
 
     try {
       // Rely on caching to make sure this is cheap...
-      const canUpgrade = await this.core.tree.upgradeable(remoteLength)
+      const canUpgrade = await MerkleTree.upgradeable(this.core.state, remoteLength)
 
       if (remoteFork !== this.core.state.fork) return false
 
@@ -696,12 +697,11 @@ class Peer {
         return new ProofRequest(msg, null, null, null)
       }
 
-      block = this.core.blocks.get(batch, index)
+      block = batch.getBlock(index)
     }
 
     const manifest = (msg.manifest && !this.core.compat) ? this.core.header.manifest : null
-    const treeBatch = this.core.state.createTreeBatch()
-    const proof = await this.core.tree.proof(batch, treeBatch, msg)
+    const proof = await MerkleTree.proof(this.core.state, batch, msg)
 
     return new ProofRequest(msg, proof, block, manifest)
   }
@@ -1404,11 +1404,11 @@ class Peer {
 
     try {
       if (req.block !== null && req.fork === fork) {
-        req.block.nodes = await this.core.tree.missingNodes(2 * req.block.index, this.core.state.length)
+        req.block.nodes = await MerkleTree.missingNodes(this.core.state, 2 * req.block.index, this.core.state.length)
         if (req.priority === PRIORITY.CANCELLED) return
       }
       if (req.hash !== null && req.fork === fork && req.hash.nodes === 0) {
-        req.hash.nodes = await this.core.tree.missingNodes(req.hash.index, this.core.state.length)
+        req.hash.nodes = await MerkleTree.missingNodes(this.core.state, req.hash.index, this.core.state.length)
         if (req.priority === PRIORITY.CANCELLED) return
 
         // nodes === 0, we already have it, bail
@@ -1442,7 +1442,7 @@ module.exports = class Replicator {
     this.allowFork = allowFork
     this.ondownloading = null // optional external hook for monitoring downloading status
     this.peers = []
-    this.findingPeers = 0 // updateable from the outside
+    this.findingPeers = 0 // updatable from the outside
     this.destroyed = false
     this.downloading = false
     this.activeSessions = 0
@@ -1856,7 +1856,7 @@ module.exports = class Replicator {
 
   async _resolveLocalBlock (b, reader, resolved) {
     try {
-      b.resolve(await this.core.blocks.get(reader, b.index))
+      b.resolve(await reader.getBlock(b.index))
     } catch (err) {
       b.reject(err)
       return
@@ -2538,7 +2538,7 @@ function clampRange (core, r) {
   if (r.blocks === null) {
     const start = core.bitfield.firstUnset(r.start)
 
-    if (r.end === -1) r.start = start === -1 ? core.tree.length : start
+    if (r.end === -1) r.start = start === -1 ? core.state.length : start
     else if (start === -1 || start >= r.end) r.start = r.end
     else {
       r.start = start

package/lib/session-state.js

@@ -11,7 +11,7 @@ const Bitfield = require('./bitfield')
 const { MerkleTree, MerkleTreeBatch } = require('./merkle-tree')
 
 module.exports = class SessionState {
-  constructor (core, parent, storage, blocks, tree, treeInfo, name) {
+  constructor (core, parent, storage, treeInfo, name) {
     this.core = core
     this.index = this.core.sessionStates.push(this) - 1
 
@@ -27,26 +27,20 @@ module.exports = class SessionState {
     this.atomized = null
     this.mutex = new Mutex()
 
-    this.blocks = blocks
-    this.tree = tree
-
     this.treeFork = -1 // only updated if truncated below dependency
 
     // merkle state
-    this.roots = []
-    this.length = 0
+    this.roots = treeInfo.roots.length ? treeInfo.roots : []
     this.fork = treeInfo.fork || 0
+    this.length = MerkleTree.span(this.roots) / 2
+    this.byteLength = MerkleTree.size(this.roots)
     this.prologue = treeInfo.prologue || null
     this.signature = treeInfo.signature || null
 
-    if (treeInfo.roots.length) this.setRoots(treeInfo.roots)
-
     this.snapshotCompatLength = this.isSnapshot() ? Math.min(this.length, this.core.state.length) : -1
 
     this.active = 0
 
-    this._onflush = null
-    this._flushing = null
     this._activeTx = null
     this._pendingBitfield = null
 
@@ -66,7 +60,7 @@ module.exports = class SessionState {
   }
 
   createTreeBatch () {
-    return new MerkleTreeBatch(this.tree, this)
+    return new MerkleTreeBatch(this)
   }
 
   addSession (s) {
@@ -113,16 +107,13 @@ module.exports = class SessionState {
   setRoots (roots) {
     this.roots = roots
     this.length = MerkleTree.span(roots) / 2
+    this.byteLength = MerkleTree.size(roots)
   }
 
   get encryptionFork () {
     return this.treeFork === -1 ? this.core.header.tree.fork : this.treeFork
   }
 
-  get byteLength () {
-    return MerkleTree.size(this.roots)
-  }
-
   async getFlushedTreeInfo (storage) {
     if (!this.atomized || !this.atomized.flushing) return this.treeInfo()
     await this.atomized.flushed()
@@ -203,8 +194,6 @@ module.exports = class SessionState {
       this.core,
       null,
       storage,
-      this.blocks,
-      this.tree.clone(storage),
       treeInfo,
       this.name
     )
@@ -215,19 +204,10 @@ module.exports = class SessionState {
   updateDependency (tx, length) {
     const dependency = updateDependency(this, length, false)
     if (dependency) tx.setDependency(dependency)
-
     return dependency
   }
 
-  _clearActiveBatch (err) {
-    if (!this._activeTx) return
-    this._activeTx = null
-
-    if (this._onflush) this._onflush(err)
-
-    this._onflush = null
-    this._flushing = null
-
+  _clearActiveBatch () {
     this._activeTx = null
   }
 
@@ -238,7 +218,7 @@ module.exports = class SessionState {
     return this._activeTx
   }
 
-  _unlock (lock) {
+  _unlock () {
     this._clearActiveBatch()
     this.mutex.unlock()
     this.core.checkIfIdle()
@@ -248,12 +228,8 @@ module.exports = class SessionState {
     const tx = this._activeTx
     this._activeTx = null
 
-    const flushing = tx.flush()
-
     try {
-      if (!this._flushing) this._flushing = flushing
-
-      return flushing
+      return await tx.flush()
     } finally {
       this._clearActiveBatch()
     }
@@ -284,6 +260,7 @@ module.exports = class SessionState {
 
       this.fork = src.fork
       this.length = src.length
+      this.byteLength = src.byteLength
       this.roots = src.roots.slice()
       this.signature = src.signature
 
@@ -314,21 +291,10 @@ module.exports = class SessionState {
       this.onappend(tree, bitfield, true)
     } finally {
       this.mutex.unlock()
+      this.core.checkIfIdle()
     }
   }
 
-  flushed () {
-    if (!this._activeTx) return
-
-    if (this._flushing) return this._flushing
-
-    this._flushing = new Promise(resolve => {
-      this._onflush = resolve
-    })
-
-    return this._flushing
-  }
-
   async setUserData (key, value) {
     await this.mutex.lock()
 
@@ -350,7 +316,9 @@ module.exports = class SessionState {
       const tx = this.createWriteBatch()
       this.updating = true
 
-      if (bitfield) this.blocks.put(tx, bitfield.start, value)
+      if (bitfield) {
+        tx.putBlock(bitfield.start, value)
+      }
 
       if (bitfield && this.isDefault()) {
         await storeBitfieldRange(this.storage, tx, bitfield.start, bitfield.start + 1, true)
@@ -375,6 +343,7 @@ module.exports = class SessionState {
       if (batch.upgraded) {
         this.roots = batch.roots
         this.length = batch.length
+        this.byteLength = batch.byteLength
         this.fork = batch.fork
         this.signature = batch.signature
 
@@ -400,7 +369,7 @@ module.exports = class SessionState {
 
     try {
       const batch = this.createTreeBatch()
-      await this.tree.truncate(length, batch, fork)
+      await MerkleTree.truncate(this, length, batch, fork)
 
       if (!signature && keyPair && length > 0) signature = this.core.verifier.sign(batch, keyPair)
       if (signature) batch.signature = signature
@@ -422,6 +391,7 @@ module.exports = class SessionState {
 
       this.fork = tree.fork
       this.length = tree.length
+      this.byteLength = MerkleTree.size(roots)
       this.roots = roots
       this.signature = tree.signature
 
@@ -447,6 +417,7 @@ module.exports = class SessionState {
 
       this.fork = batch.fork
       this.length = batch.length
+      this.byteLength = batch.byteLength
       this.roots = batch.roots
       this.signature = batch.signature
 
@@ -501,7 +472,7 @@ module.exports = class SessionState {
         }
       }
 
-      this.blocks.clear(tx, start, end)
+      tx.deleteBlockRange(start, end)
 
       const dependency = start < this.flushedLength() ? updateDependency(this, start, true) : null
 
@@ -563,7 +534,9 @@ module.exports = class SessionState {
 
       if (this.isDefault()) await storeBitfieldRange(this.storage, tx, batch.ancestors, batch.length, true)
 
-      this.blocks.putBatch(tx, this.length, values)
+      for (let i = 0; i < values.length; i++) {
+        tx.putBlock(this.length + i, values[i])
+      }
 
       const bitfield = {
         drop: false,
@@ -576,6 +549,7 @@ module.exports = class SessionState {
       this.fork = batch.fork
       this.roots = batch.roots
       this.length = batch.length
+      this.byteLength = batch.byteLength
       this.signature = batch.signature
 
       this.onappend(tree, bitfield, flushed)
@@ -717,6 +691,7 @@ module.exports = class SessionState {
       this.fork = tree.fork
       this.roots = roots
       this.length = tree.length
+      this.byteLength = MerkleTree.size(roots)
 
       if (truncating) this.ontruncate(tree, sharedLength, origLength, flushed)
       if (sharedLength < length) this.onappend(tree, null, flushed)
@@ -819,7 +794,7 @@ module.exports = class SessionState {
       signature
     }
 
-    const upgraded = treeLength < this.length || this.length < length || tree.fork !== this.tree.fork
+    const upgraded = treeLength < this.length || this.length < length || tree.fork !== this.fork
 
     if (upgraded) tx.setHead(tree)
 
@@ -828,6 +803,7 @@ module.exports = class SessionState {
     this.fork = tree.fork
     this.roots = roots
     this.length = length
+    this.byteLength = MerkleTree.size(roots)
     this.signature = signature
 
     return { tree, flushed }
@@ -861,7 +837,7 @@ module.exports = class SessionState {
       if (treeLength < length) {
         const tx = state.createWriteBatch()
 
-        state.blocks.clear(tx, treeLength, length)
+        tx.deleteBlockRange(treeLength, length)
         const dependency = state.updateDependency(tx, length)
 
         await state.flush(tx)
@@ -896,7 +872,7 @@ module.exports = class SessionState {
 
     head.length = length
 
-    const roots = await this.tree.getRoots(length)
+    const roots = await MerkleTree.getRootsFromStorage(this.storage, length)
     const rootHash = crypto.tree(roots)
 
     head.fork = this.fork
@@ -952,19 +928,23 @@ module.exports = class SessionState {
 
         // todo: validate treeInfo
 
+        let storage = null
+
         if (resumed) {
-          this.storage = resumed
+          storage = resumed
         } else {
           treeInfo.fork = fork
-          this.storage = await state.storage.createSession(this.name, treeInfo)
+          storage = await state.storage.createSession(this.name, treeInfo)
         }
 
-        this.tree = new MerkleTree(this.storage)
+        const roots = await MerkleTree.getRootsFromStorage(storage, length)
 
+        this.storage = storage
         this.prologue = state.prologue
         this.fork = fork
         this.length = length
-        this.roots = await this.tree.getRoots(length)
+        this.byteLength = MerkleTree.size(roots)
+        this.roots = roots
 
         if (truncation) {
           const { dependency } = truncation
@@ -1016,11 +996,9 @@ module.exports = class SessionState {
       throw new Error('Session already atomized')
     }
 
-    const tree = new MerkleTree(storage)
-
     const head = {
       fork: this.fork,
-      roots: length === this.length ? this.roots.slice() : await tree.getRoots(length),
+      roots: length === this.length ? this.roots.slice() : await MerkleTree.getRootsFromStorage(storage, length),
       length,
       prologue: this.prologue,
       signature: length === this.length ? this.signature : null
@@ -1030,8 +1008,6 @@ module.exports = class SessionState {
       this.core,
       atom ? this : null,
       storage,
-      this.core.blocks,
-      tree,
       head,
       atom ? this.name : name
     )
@@ -1094,7 +1070,7 @@ async function storeBitfieldRange (storage, tx, from, to, value) {
 
 async function truncateAndFlush (s, length) {
   const batch = s.createTreeBatch()
-  await s.tree.truncate(length, batch, s.tree.fork)
+  await MerkleTree.truncate(s, length, batch, s.fork)
   const tx = s.createWriteBatch()
 
   const info = await s._truncate(tx, batch)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.0.29",
+  "version": "11.0.30",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {

package/lib/block-store.js

@@ -1,25 +0,0 @@
-module.exports = class BlockStore {
-  constructor (storage) {
-    this.storage = storage
-  }
-
-  async get (rx, i) {
-    return rx.getBlock(i)
-  }
-
-  put (tx, i, data) {
-    tx.putBlock(i, data)
-  }
-
-  putBatch (tx, i, blocks) {
-    if (blocks.length === 0) return Promise.resolve()
-
-    for (let j = 0; j < blocks.length; j++) {
-      tx.putBlock(i + j, blocks[j])
-    }
-  }
-
-  clear (tx, start = 0, end = -1) {
-    tx.deleteBlockRange(start, end)
-  }
-}