hypercore 11.12.2 → 11.13.1

package/lib/merkle-tree.js

@@ -679,6 +679,7 @@ class MerkleTree {
     // TODO: we could prop use a read batch here and do this in blocks of X for perf
     while (!ite.contains(head) && !(await hasTreeNode(session.storage, ite.index))) {
       cnt++
+      if (cnt >= 1024) throw ASSERTION('Bad arguments to missingNodes index=' + index + ' at length=' + length)
       ite.parent()
     }
 
@@ -742,6 +743,7 @@ function getByteOffsetBatch (roots, index, rx) {
   if ((index & 1) === 1) index = flat.leftSpan(index)
 
   let head = 0
+  let cnt = 0
 
   const promises = []
 
@@ -756,6 +758,8 @@ function getByteOffsetBatch (roots, index, rx) {
     const ite = flat.iterator(node.index)
 
     while (ite.index !== index) {
+      if (++cnt >= 1024) throw ASSERTION('Bad getByteOffsetSession index=' + index)
+
       if (index < ite.index) {
         ite.leftChild()
       } else {
@@ -930,8 +934,10 @@ async function seekTrustedTree (session, root, bytes, padding) {
   if (!bytes) return root
 
   const ite = flat.iterator(root)
+  let cnt = 0
 
   while ((ite.index & 1) !== 0) {
+    if (++cnt >= 1024) throw ASSERTION('Bad seekTrusted bytes=' + bytes + ', paddding=' + padding)
     const l = await getTreeNodeFromStorage(session.storage, ite.leftChild())
 
     if (l) {
@@ -972,11 +978,13 @@ async function seekUntrustedTree (session, root, bytes, padding) {
 
 function seekProof (session, rx, seekRoot, root, p) {
   const ite = flat.iterator(seekRoot)
+  let cnt = 0
 
   p.seek = []
   p.seek.push(getTreeNodeOrError(rx, ite.index))
 
   while (ite.index !== root) {
+    if (++cnt >= 1024) throw ASSERTION('Bad seekProof seekRoot=' + seekRoot + ', root=' + root)
     ite.sibling()
     p.seek.push(getTreeNodeOrError(rx, ite.index))
     ite.parent()
@@ -987,11 +995,13 @@ function blockAndSeekProof (session, rx, node, seek, seekRoot, root, p) {
   if (!node) return seekProof(session, rx, seekRoot, root, p)
 
   const ite = flat.iterator(node.index)
+  let cnt = 0
 
   p.node = []
   if (!node.value) p.node.push(getTreeNodeOrError(rx, ite.index))
 
   while (ite.index !== root) {
+    if (++cnt >= 1024) throw ASSERTION('Bad blockAndSeekProof seekRoot=' + seekRoot + ', root=' + root)
     ite.sibling()
 
     if (seek && ite.contains(seekRoot) && ite.index !== seekRoot) {
@@ -1017,10 +1027,13 @@ function upgradeProof (session, rx, node, seek, from, to, subTree, p) {
 
       const root = ite.index
       const target = from - 2
+      let cnt = 0
 
       ite.seek(target)
 
       while (ite.index !== root) {
+        if (++cnt >= 1024) throw ASSERTION('Bad upgradeProof target=' + target + ', root=' + root)
+
         ite.sibling()
         if (ite.index > target) {
           if (p.node === null && p.seek === null && ite.contains(subTree)) {
@@ -1064,10 +1077,12 @@ function additionalUpgradeProof (session, rx, from, to, p) {
 
       const root = ite.index
       const target = from - 2
+      let cnt = 0
 
       ite.seek(target)
 
       while (ite.index !== root) {
+        if (++cnt >= 1024) throw ASSERTION('Bad arguments to additionalUpgradeProof root=' + root + ' target=' + target)
         ite.sibling()
         if (ite.index > target) {
           p.additionalUpgrade.push(getTreeNodeOrError(rx, ite.index))

package/lib/messages.js

@@ -514,13 +514,20 @@ wire.data = {
 wire.noData = {
   preencode (state, m) {
     c.uint.preencode(state, m.request)
+    c.uint.preencode(state, m.reason ? 1 : 0)
+    if (m.reason) c.uint.preencode(state, m.reason)
   },
   encode (state, m) {
     c.uint.encode(state, m.request)
+    c.uint.encode(state, m.reason ? 1 : 0)
+    if (m.reason) c.uint.encode(state, m.reason)
   },
   decode (state, m) {
+    const request = c.uint.decode(state)
+    const flags = state.start < state.end ? c.uint.decode(state) : 0
     return {
-      request: c.uint.decode(state)
+      request,
+      reason: flags & 1 ? c.uint.decode(state) : 0
     }
   }
 }

package/lib/replicator.js

@@ -44,6 +44,11 @@ const MAX_REMOTE_SEGMENTS = 2048
 
 const MAX_RANGES = 64
 
+const NOT_AVAILABLE = 1
+const INVALID_REQUEST = 2
+const MAX_INVALID_REQUESTS = 64
+const MAX_BACKOFFS = MAX_INVALID_REQUESTS / 2
+
 const PRIORITY = {
   NORMAL: 0,
   HIGH: 1,
@@ -407,7 +412,8 @@ class Peer {
       wireExtension: { tx: 0, rx: 0 },
       hotswaps: 0,
       invalidData: 0,
-      invalidRequests: 0
+      invalidRequests: 0,
+      backoffs: 0
     }
 
     this.receiverQueue = new ReceiverQueue()
@@ -730,9 +736,13 @@ class Peer {
       return new ProofRequest(msg, proof, block, manifest)
     } catch (err) {
       batch.destroy()
-      this.stats.invalidRequests++
-      this.replicator.stats.invalidRequests++
-      throw err
+
+      this.replicator._oninvalidrequest(err, msg, this)
+
+      if (this.stats.invalidRequests >= MAX_INVALID_REQUESTS) throw err
+
+      await backoff(this.stats.invalidRequests)
+      return null
     }
   }
 
@@ -791,6 +801,11 @@ class Peer {
       ? await this._getProof(batch, msg)
       : new ProofRequest(msg, null, null, null)
 
+    if (req === null) {
+      this.wireNoData.send({ request: msg.id, reason: INVALID_REQUEST })
+      return
+    }
+
     batch.tryFlush()
 
     await this._fulfillRequest(req)
@@ -819,7 +834,7 @@ class Peer {
         return
       }
 
-      this.wireNoData.send({ request: req.msg.id })
+      this.wireNoData.send({ request: req.msg.id, reason: NOT_AVAILABLE })
       return
     }
 
@@ -904,7 +919,7 @@ class Peer {
       if (isBlockRequest(req)) this.replicator._unmarkInflight(req.block.index)
 
       this.paused = true
-      this.replicator._oninvalid(err, req, data, this)
+      this.replicator._oninvaliddata(err, req, data, this)
       return
     }
 
@@ -913,7 +928,7 @@ class Peer {
 
     try {
       if (!matchingRequest(req, data) || !(await this.core.verify(data, this))) {
-        this.replicator._onnodata(this, req)
+        this.replicator._onnodata(this, req, 0)
         return
       }
     } catch (err) {
@@ -935,8 +950,8 @@ class Peer {
       }
 
       this.paused = true
-      this.replicator._onnodata(this, req)
-      this.replicator._oninvalid(err, req, data, this)
+      this.replicator._onnodata(this, req, 0)
+      this.replicator._oninvaliddata(err, req, data, this)
       return
     } finally {
       if (isBlockRequest(req)) this.replicator._markProcessed(req.block.index)
@@ -950,13 +965,12 @@ class Peer {
     }
   }
 
-  onnodata ({ request }) {
+  onnodata ({ request, reason }) {
     const req = request > 0 ? this.replicator._inflight.get(request) : null
-
     if (req === null || req.peer !== this) return
 
     this._onrequestroundtrip(req)
-    this.replicator._onnodata(this, req)
+    this.replicator._onnodata(this, req, reason)
   }
 
   _onrequestroundtrip (req) {
@@ -1505,7 +1519,8 @@ module.exports = class Replicator {
       wireExtension: { tx: 0, rx: 0 },
       hotswaps: 0,
       invalidData: 0,
-      invalidRequests: 0
+      invalidRequests: 0,
+      backoffs: 0
     }
 
     this._attached = new Set()
@@ -2086,7 +2101,16 @@ module.exports = class Replicator {
     }
   }
 
-  _onnodata (peer, req) {
+  _onnodata (peer, req, reason) {
+    if (reason === INVALID_REQUEST) {
+      peer.stats.backoffs++
+      this.stats.backoffs++
+
+      if (peer.stats.backoffs >= MAX_BACKOFFS) {
+        peer.paused = true
+      }
+    }
+
     this._clearRequest(peer, req)
     this.updateAll()
   }
@@ -2547,7 +2571,7 @@ module.exports = class Replicator {
     }
   }
 
-  _oninvalid (err, req, res, from) {
+  _oninvaliddata (err, req, res, from) {
     const sessions = this.core.monitors
 
     this.stats.invalidData++
@@ -2556,6 +2580,16 @@ module.exports = class Replicator {
       sessions[i].emit('verification-error', err, req, res, from)
     }
   }
+
+  _oninvalidrequest (err, req, from) {
+    const sessions = this.core.monitors
+
+    from.stats.invalidRequests++
+    this.stats.invalidRequests++
+    for (let i = 0; i < sessions.length; i++) {
+      sessions[i].emit('invalid-request', err, req, from)
+    }
+  }
 }
 
 function matchingRequest (req, data) {
@@ -2707,3 +2741,8 @@ function incrementRx (stats1, stats2) {
 }
 
 function noop () {}
+
+function backoff (times) {
+  const sleep = times < 2 ? 200 : times < 5 ? 500 : times < 40 ? 1000 : 5000
+  return new Promise(resolve => setTimeout(resolve, sleep))
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.12.2",
+  "version": "11.13.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {