hypercore 11.0.43 → 11.0.45

package/index.js

@@ -370,6 +370,9 @@ class Hypercore extends EventEmitter {
     if (this.state && checkout !== -1) {
       if (!opts.name && !opts.atom) throw ASSERTION('Checkouts must be named or atomized')
       if (checkout > this.state.length) throw ASSERTION('Invalid checkout ' + checkout + ' for ' + opts.name + ', length is ' + this.state.length)
+      if (this.state.prologue && checkout < this.state.prologue.length) {
+        throw ASSERTION('Invalid checkout ' + checkout + ' for ' + opts.name + ', prologue length is ' + this.state.prologue.length)
+      }
       if (checkout < this.state.length) await this.state.truncate(checkout, this.fork)
     }
 

package/lib/core.js

@@ -171,6 +171,7 @@ module.exports = class Core {
         key: opts.key || (compat ? manifest.signers[0].publicKey : Verifier.manifestHash(manifest)),
         manifest,
         keyPair: keyPair ? { publicKey: keyPair.publicKey, secretKey: keyPair.secretKey || null } : null,
+        frozen: false,
         userData: [],
         tree: {
           fork: 0,
@@ -190,6 +191,7 @@ module.exports = class Core {
         key: header.key,
         manifest,
         keyPair,
+        frozen: false,
         discoveryKey,
         userData: opts.userData || [],
         alias: opts.alias || null
@@ -494,6 +496,11 @@ module.exports = class Core {
       return false
     }
 
+    // sanity check -> no manifest, no way to verify
+    if (!this.header.manifest) {
+      return false
+    }
+
     const batch = MerkleTree.verifyFullyRemote(this.state, proof)
 
     try {
@@ -502,24 +509,57 @@ module.exports = class Core {
       return true
     }
 
+    const roots = await MerkleTree.getRootsFromStorage(this.storage, proof.upgrade.length)
+    const remoteTreeHash = crypto.tree(proof.upgrade.nodes)
+    const localTreeHash = crypto.tree(roots)
+
+    try {
+      const rx = this.state.storage.read()
+      const treeProofPromise = MerkleTree.proof(this.state, rx, {
+        block: null,
+        hash: null,
+        seek: null,
+        upgrade: {
+          start: 0,
+          length: proof.upgrade.length
+        }
+      })
+
+      rx.tryFlush()
+
+      const treeProof = await treeProofPromise
+
+      const verifyBatch = MerkleTree.verifyFullyRemote(this.state, await treeProof.settle())
+      this._verifyBatchUpgrade(verifyBatch, this.header.manifest)
+    } catch {
+      return true
+    }
+
+    // both proofs are valid, now check if they forked
+    if (b4a.equals(localTreeHash, remoteTreeHash)) return false
+
     await this.state.mutex.lock()
 
     try {
       const tx = this.state.createWriteBatch()
-      if (this.header.manifest === null && proof.manifest) {
-        this._setManifest(tx, proof.manifest, null)
-      }
+
+      this.header.frozen = true
+
+      tx.setAuth({
+        key: this.header.key,
+        discoveryKey: this.discoveryKey,
+        manifest: this.header.manifest,
+        keyPair: this.header.keyPair,
+        frozen: true
+      })
 
       await this.state.flush()
     } finally {
       this.state.mutex.unlock()
     }
 
-    const remoteTreeHash = crypto.tree(proof.upgrade.nodes)
-    const localTreeHash = crypto.tree(await MerkleTree.getRootsFromStorage(this.storage, proof.upgrade.length))
-
-    if (b4a.equals(localTreeHash, remoteTreeHash)) return false
-
+    // tmp log so we can see these
+    console.log('[hypercore] conflict detected in ' + this.id + ' (writable=' + !!this.header.keyPair + ',quorum=' + this.header.manifest.quorum + ')')
     await this._onconflict(proof)
     return true
   }

package/lib/merkle-tree.js

@@ -148,7 +148,7 @@ class MerkleTreeBatch {
     return caps.treeSignableCompat(this.hash(), this.length, this.fork, noHeader)
   }
 
-  get (index, error) {
+  get (index) {
     if (index >= this.length * 2) {
       return null
     }
@@ -157,7 +157,7 @@ class MerkleTreeBatch {
       if (n.index === index) return n
     }
 
-    return MerkleTree.get(this.session, index, error)
+    return getTreeNodeFromStorage(this.session.storage, index)
   }
 
   // deprecated, use sssion proof instead
@@ -267,9 +267,9 @@ class MerkleTreeBatch {
   }
 
   byteRange (index) {
-    const readBatch = this.storage.read()
-    const range = getByteRange(this, index, readBatch)
-    readBatch.tryFlush()
+    const rx = this.storage.read()
+    const range = getByteRange(this, index, rx)
+    rx.tryFlush()
 
     return range
   }
@@ -277,9 +277,9 @@ class MerkleTreeBatch {
   byteOffset (index) {
     if (index === 2 * this.length) return this.byteLength
 
-    const readBatch = this.storage.read()
-    const offset = getByteOffset(this, index, readBatch)
-    readBatch.tryFlush()
+    const rx = this.storage.read()
+    const offset = getByteOffset(this, index, rx)
+    rx.tryFlush()
 
     return offset
   }
@@ -347,7 +347,7 @@ class ReorgBatch extends MerkleTreeBatch {
       const left = n.get(ite.leftChild())
       if (!left) break
 
-      const existing = await MerkleTree.get(this.session, left.index, false)
+      const existing = await getTreeNodeFromStorage(this.session.storage, left.index)
       if (!existing || !b4a.equals(existing.hash, left.hash)) {
         diff = left
       } else {
@@ -413,7 +413,7 @@ class ByteSeeker {
       const ite = flat.iterator(node.index)
 
       while ((ite.index & 1) !== 0) {
-        const l = await MerkleTree.get(this.session, ite.leftChild(), false)
+        const l = await getTreeNodeFromStorage(this.session.storage, ite.leftChild())
 
         if (l) {
           const size = getUnpaddedSize(l, this.padding, ite)
@@ -530,51 +530,27 @@ class MerkleTree {
   static getRootsFromStorage (storage, length) {
     const indexes = flat.fullRoots(2 * length)
     const roots = new Array(indexes.length)
-    const readBatch = storage.read()
+    const rx = storage.read()
 
     for (let i = 0; i < indexes.length; i++) {
-      roots[i] = readBatch.getTreeNode(indexes[i], true)
+      roots[i] = getTreeNodeOrError(rx, indexes[i])
     }
 
-    readBatch.tryFlush()
+    rx.tryFlush()
 
     return Promise.all(roots)
   }
 
-  static getNeededNodes (session, length, start, end) {
-    const nodes = new Map()
-    const head = length * 2
-
-    for (let i = start; i < end; i++) {
-      const ite = flat.iterator(i * 2)
-
-      while (true) {
-        if (nodes.has(ite.index)) break
-        nodes.set(ite.index, this.get(session, ite.index, true))
-
-        const sibling = ite.sibling()
-
-        ite.parent()
-        if (ite.contains(head)) break
-
-        if (nodes.has(sibling)) break
-        nodes.set(sibling, this.get(session, sibling, true))
-      }
-    }
-
-    return Promise.all([...nodes.values()])
-  }
-
   static async upgradeable (session, length) {
     const indexes = flat.fullRoots(2 * length)
     const roots = new Array(indexes.length)
-    const readBatch = session.storage.read()
+    const rx = session.storage.read()
 
     for (let i = 0; i < indexes.length; i++) {
-      roots[i] = readBatch.getTreeNode(indexes[i], false)
+      roots[i] = rx.getTreeNode(indexes[i])
     }
 
-    readBatch.tryFlush()
+    rx.tryFlush()
 
     for (const node of await Promise.all(roots)) {
       if (node === null) return false
@@ -587,10 +563,8 @@ class MerkleTree {
     return new ByteSeeker(session, bytes, padding)
   }
 
-  static get (session, index, error = true, readBatch = null) {
-    if (readBatch) return readBatch.getTreeNode(index, error)
-
-    return getTreeNode(session.storage, index, error)
+  static get (session, index) {
+    return getTreeNodeFromStorage(session.storage, index)
   }
 
   static async truncate (session, length, batch, fork = batch.fork) {
@@ -602,7 +576,7 @@ class MerkleTree {
       if (i < batch.roots.length && batch.roots[i].index === root) continue
 
       while (batch.roots.length > i) batch.roots.pop()
-      batch.roots.push(unslabNode(await this.get(session, root)))
+      batch.roots.push(unslabNode(await getTreeNodeFromStorageOrError(session.storage, root)))
     }
 
     while (batch.roots.length > fullRoots.length) {
@@ -630,7 +604,7 @@ class MerkleTree {
     }
 
     for (const root of batch.roots) {
-      const existing = await MerkleTree.get(session, root.index, false)
+      const existing = await getTreeNodeFromStorage(session.storage, root.index)
       if (existing && b4a.equals(existing.hash, root.hash)) continue
       batch._updateDiffRoot(root)
       break
@@ -679,7 +653,7 @@ class MerkleTree {
     }
 
     if (unverified) {
-      const verified = await MerkleTree.get(session, unverified.index)
+      const verified = await getTreeNodeFromStorageOrError(session.storage, unverified.index)
       if (!b4a.equals(verified.hash, unverified.hash)) {
         throw INVALID_CHECKSUM('Invalid checksum at node ' + unverified.index)
       }
@@ -730,16 +704,16 @@ module.exports = {
   MerkleTree
 }
 
-async function getNodeSize (index, readBatch) {
-  return (await readBatch.getTreeNode(index, true)).size
+async function getNodeSize (index, rx) {
+  return (await getTreeNodeOrError(rx, index)).size
 }
 
-async function getByteOffsetSession (session, index, readBatch) {
+async function getByteOffsetSession (session, index, rx) {
   if (index === 2 * session.length) return session.byteLength
 
-  const treeNodes = readBatch === null
+  const treeNodes = rx === null
     ? await getByteOffsetBatchFlush(session.roots, index, session.storage.read())
-    : await getByteOffsetBatch(session.roots, index, readBatch)
+    : await getByteOffsetBatch(session.roots, index, rx)
 
   let offset = 0
   for (const node of treeNodes) offset += node.size
@@ -747,10 +721,10 @@ async function getByteOffsetSession (session, index, readBatch) {
   return offset
 }
 
-async function getByteOffset (tree, index, readBatch) {
+async function getByteOffset (tree, index, rx) {
   if (index === 2 * tree.length) return tree.byteLength
 
-  const treeNodes = await getByteOffsetBatch(tree.roots, index, readBatch)
+  const treeNodes = await getByteOffsetBatch(tree.roots, index, rx)
 
   let offset = 0
   for (const node of treeNodes) offset += node.size
@@ -758,13 +732,13 @@ async function getByteOffset (tree, index, readBatch) {
   return offset
 }
 
-function getByteOffsetBatchFlush (roots, index, readBatch) {
-  const treeNodes = getByteOffsetBatch(roots, index, readBatch)
-  readBatch.tryFlush()
+function getByteOffsetBatchFlush (roots, index, rx) {
+  const treeNodes = getByteOffsetBatch(roots, index, rx)
+  rx.tryFlush()
   return treeNodes
 }
 
-function getByteOffsetBatch (roots, index, readBatch) {
+function getByteOffsetBatch (roots, index, rx) {
   if ((index & 1) === 1) index = flat.leftSpan(index)
 
   let head = 0
@@ -785,7 +759,7 @@ function getByteOffsetBatch (roots, index, readBatch) {
       if (index < ite.index) {
         ite.leftChild()
       } else {
-        promises.push(readBatch.getTreeNode(ite.leftChild(), true))
+        promises.push(getTreeNodeOrError(rx, ite.leftChild()))
         ite.sibling()
       }
     }
@@ -796,14 +770,14 @@ function getByteOffsetBatch (roots, index, readBatch) {
   throw ASSERTION('Failed to find offset')
 }
 
-function getByteRange (tree, index, readBatch) {
+function getByteRange (tree, index, rx) {
   const head = 2 * tree.length
   if (((index & 1) === 0 ? index : flat.rightSpan(index)) >= head) {
     throw BAD_ARGUMENT('Index is out of bounds')
   }
 
-  const offset = getByteOffset(tree, index, readBatch)
-  const size = getNodeSize(index, readBatch)
+  const offset = getByteOffset(tree, index, rx)
+  const size = getNodeSize(index, rx)
 
   return Promise.all([offset, size])
 }
@@ -935,8 +909,7 @@ async function seekFromHead (session, head, bytes, padding) {
 
   for (let i = 0; i < roots.length; i++) {
     const root = roots[i]
-    const node = await MerkleTree.get(session, root, false)
-    if (node === null) throw new Error('Tree node missing')
+    const node = await getTreeNodeFromStorage(session.storage, root)
     const size = getUnpaddedSize(node, padding, null)
 
     if (bytes === size) return root
@@ -959,7 +932,7 @@ async function seekTrustedTree (session, root, bytes, padding) {
   const ite = flat.iterator(root)
 
   while ((ite.index & 1) !== 0) {
-    const l = await MerkleTree.get(session, ite.leftChild(), false)
+    const l = await getTreeNodeFromStorage(session.storage, ite.leftChild())
 
     if (l) {
       const size = getUnpaddedSize(l, padding, ite)
@@ -986,8 +959,7 @@ async function seekUntrustedTree (session, root, bytes, padding) {
 
   bytes -= offset
 
-  const node = await MerkleTree.get(session, root, false)
-  if (node === null) throw new Error('Tree node missing')
+  const node = await getTreeNodeFromStorageOrError(session.storage, root)
 
   if (getUnpaddedSize(node, padding, null) <= bytes) throw INVALID_OPERATION('Invalid seek')
 
@@ -998,41 +970,41 @@ async function seekUntrustedTree (session, root, bytes, padding) {
 // Note, that all these methods are sync as we can statically infer which nodes
 // are needed for the remote to verify given they arguments they passed us
 
-function seekProof (session, batch, seekRoot, root, p) {
+function seekProof (session, rx, seekRoot, root, p) {
   const ite = flat.iterator(seekRoot)
 
   p.seek = []
-  p.seek.push(MerkleTree.get(session, ite.index, true, batch))
+  p.seek.push(getTreeNodeOrError(rx, ite.index))
 
   while (ite.index !== root) {
     ite.sibling()
-    p.seek.push(MerkleTree.get(session, ite.index, true, batch))
+    p.seek.push(getTreeNodeOrError(rx, ite.index))
     ite.parent()
   }
 }
 
-function blockAndSeekProof (session, batch, node, seek, seekRoot, root, p) {
-  if (!node) return seekProof(session, batch, seekRoot, root, p)
+function blockAndSeekProof (session, rx, node, seek, seekRoot, root, p) {
+  if (!node) return seekProof(session, rx, seekRoot, root, p)
 
   const ite = flat.iterator(node.index)
 
   p.node = []
-  if (!node.value) p.node.push(MerkleTree.get(session, ite.index, true, batch))
+  if (!node.value) p.node.push(getTreeNodeOrError(rx, ite.index))
 
   while (ite.index !== root) {
     ite.sibling()
 
     if (seek && ite.contains(seekRoot) && ite.index !== seekRoot) {
-      seekProof(session, batch, seekRoot, ite.index, p)
+      seekProof(session, rx, seekRoot, ite.index, p)
     } else {
-      p.node.push(MerkleTree.get(session, ite.index, true, batch))
+      p.node.push(getTreeNodeOrError(rx, ite.index))
     }
 
     ite.parent()
   }
 }
 
-function upgradeProof (session, batch, node, seek, from, to, subTree, p) {
+function upgradeProof (session, rx, node, seek, from, to, subTree, p) {
   if (from === 0) p.upgrade = []
 
   for (const ite = flat.iterator(0); ite.fullRoot(to); ite.nextTree()) {
@@ -1052,9 +1024,9 @@ function upgradeProof (session, batch, node, seek, from, to, subTree, p) {
         ite.sibling()
         if (ite.index > target) {
           if (p.node === null && p.seek === null && ite.contains(subTree)) {
-            blockAndSeekProof(session, batch, node, seek, subTree, ite.index, p)
+            blockAndSeekProof(session, rx, node, seek, subTree, ite.index, p)
           } else {
-            p.upgrade.push(batch.getTreeNode(ite.index, true))
+            p.upgrade.push(getTreeNodeOrError(rx, ite.index))
           }
         }
         ite.parent()
@@ -1070,16 +1042,16 @@ function upgradeProof (session, batch, node, seek, from, to, subTree, p) {
     // if the subtree included is a child of this tree, include that one
     // instead of a dup node
     if (p.node === null && p.seek === null && ite.contains(subTree)) {
-      blockAndSeekProof(session, batch, node, seek, subTree, ite.index, p)
+      blockAndSeekProof(session, rx, node, seek, subTree, ite.index, p)
       continue
     }
 
     // add root (can be optimised since the root might be in tree.roots)
-    p.upgrade.push(MerkleTree.get(session, ite.index, true, batch))
+    p.upgrade.push(getTreeNodeOrError(rx, ite.index))
   }
 }
 
-function additionalUpgradeProof (session, batch, from, to, p) {
+function additionalUpgradeProof (session, rx, from, to, p) {
   if (from === 0) p.additionalUpgrade = []
 
   for (const ite = flat.iterator(0); ite.fullRoot(to); ite.nextTree()) {
@@ -1098,7 +1070,7 @@ function additionalUpgradeProof (session, batch, from, to, p) {
       while (ite.index !== root) {
         ite.sibling()
         if (ite.index > target) {
-          p.additionalUpgrade.push(MerkleTree.get(session, ite.index, true, batch))
+          p.additionalUpgrade.push(getTreeNodeOrError(rx, ite.index))
         }
         ite.parent()
       }
@@ -1111,7 +1083,7 @@ function additionalUpgradeProof (session, batch, from, to, p) {
     }
 
     // add root (can be optimised since the root is in tree.roots)
-    p.additionalUpgrade.push(MerkleTree.get(session, ite.index, true, batch))
+    p.additionalUpgrade.push(getTreeNodeOrError(rx, ite.index))
   }
 }
 
@@ -1163,17 +1135,30 @@ function normalizeIndexed (block, hash) {
   return null
 }
 
-function getTreeNode (storage, index, error) {
-  const batch = storage.read()
-  const node = batch.getTreeNode(index, error)
-  batch.tryFlush()
+async function getTreeNodeOrError (rx, index) {
+  const node = await rx.getTreeNode(index)
+  if (node === null) throw INVALID_OPERATION('Expected tree node ' + index + ' from storage, got (nil)')
+  return node
+}
+
+function getTreeNodeFromStorageOrError (storage, index) {
+  const rx = storage.read()
+  const p = getTreeNodeOrError(rx, index)
+  rx.tryFlush()
+  return p
+}
+
+function getTreeNodeFromStorage (storage, index) {
+  const rx = storage.read()
+  const node = rx.getTreeNode(index)
+  rx.tryFlush()
   return node
 }
 
 function hasTreeNode (storage, index) {
-  const batch = storage.read()
-  const has = batch.hasTreeNode(index)
-  batch.tryFlush()
+  const rx = storage.read()
+  const has = rx.hasTreeNode(index)
+  rx.tryFlush()
   return has
 }
 
@@ -1197,7 +1182,7 @@ async function settleProof (p) {
 }
 
 // tree can be either the merkle tree or a merkle tree batch
-async function generateProof (session, readBatch, block, hash, seek, upgrade) {
+async function generateProof (session, rx, block, hash, seek, upgrade) {
   // Important that this does not throw inbetween making the promise arrays
   // and finalise being called, otherwise there will be lingering promises in the background
 
@@ -1228,14 +1213,14 @@ async function generateProof (session, readBatch, block, hash, seek, upgrade) {
   if (node !== null && (!upgrade || node.lastIndex < upgrade.start)) {
     subTree = nodesToRoot(node.index, node.nodes, to)
     const seekRoot = seek ? await seekUntrustedTree(session, subTree, seek.bytes, seek.padding) : head
-    blockAndSeekProof(session, readBatch, node, seek, seekRoot, subTree, p.pending)
+    blockAndSeekProof(session, rx, node, seek, seekRoot, subTree, p.pending)
   } else if ((node || seek) && upgrade) {
     subTree = seek ? await seekFromHead(session, to, seek.bytes, seek.padding) : node.index
   }
 
   if (upgrade) {
-    upgradeProof(session, readBatch, node, seek, from, to, subTree, p.pending)
-    if (head > to) additionalUpgradeProof(session, readBatch, to, head, p.pending)
+    upgradeProof(session, rx, node, seek, from, to, subTree, p.pending)
+    if (head > to) additionalUpgradeProof(session, rx, to, head, p.pending)
   }
 
   return p

package/lib/replicator.js

@@ -475,6 +475,8 @@ class Peer {
   }
 
   broadcastRange (start, length, drop) {
+    if (!this.isActive()) return
+
     if (drop) this._unclearLocalRange(start, length)
     else this._clearLocalRange(start, length)
 
@@ -779,6 +781,10 @@ class Peer {
     // sync from now on, so safe to delete from the map
     this.remoteRequests.delete(req.msg.id)
 
+    if (!this.isActive() && proof.block !== null) {
+      return
+    }
+
     if (proof === null) {
       if (req.msg.manifest && this.core.header.manifest) {
         const manifest = this.core.header.manifest
@@ -1389,7 +1395,7 @@ class Peer {
   }
 
   isActive () {
-    if (this.paused || this.removed) return false
+    if (this.paused || this.removed || this.core.header.frozen) return false
     return true
   }
 
@@ -2120,6 +2126,8 @@ module.exports = class Replicator {
   }
 
   _onwant (peer, start, length) {
+    if (!peer.isActive()) return
+
     const contig = Math.min(this.core.state.length, this.core.header.hints.contiguousLength)
 
     if (start + length < contig || (this.core.state.length === contig)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.0.43",
+  "version": "11.0.45",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {