hypercore 10.28.11 → 10.29.1

package/lib/merkle-tree.js

@@ -345,8 +345,7 @@ class ByteSeeker {
     if (!bytes) return [0, 0]
 
     for (const node of this.tree.roots) { // all async ticks happen once we find the root so safe
-      let size = node.size
-      if (this.padding > 0) size -= this.padding * flat.countLeaves(node.index)
+      const size = getUnpaddedSize(node, this.padding, null)
 
       if (bytes === size) return [flat.rightSpan(node.index) + 2, 0]
       if (bytes > size) {
@@ -359,8 +358,7 @@ class ByteSeeker {
       while ((ite.index & 1) !== 0) {
         const l = await this.tree.get(ite.leftChild(), false)
         if (l) {
-          let size = l.size
-          if (this.padding > 0) size -= this.padding * ite.countLeaves()
+          const size = getUnpaddedSize(l, this.padding, ite)
 
           if (size === bytes) return [ite.rightSpan() + 2, 0]
           if (size > bytes) continue
@@ -875,20 +873,21 @@ function verifyUpgrade ({ fork, upgrade }, blockRoot, batch) {
   return q.extra === null
 }
 
-async function seekFromHead (tree, head, bytes) {
+async function seekFromHead (tree, head, bytes, padding) {
   const roots = flat.fullRoots(head)
 
   for (let i = 0; i < roots.length; i++) {
     const root = roots[i]
     const node = await tree.get(root)
+    const size = getUnpaddedSize(node, padding, null)
 
-    if (bytes === node.size) return root
-    if (bytes > node.size) {
-      bytes -= node.size
+    if (bytes === size) return root
+    if (bytes > size) {
+      bytes -= size
       continue
     }
 
-    return seekTrustedTree(tree, root, bytes)
+    return seekTrustedTree(tree, root, bytes, padding)
   }
 
   return head
@@ -896,7 +895,7 @@ async function seekFromHead (tree, head, bytes) {
 
 // trust that bytes are within the root tree and find the block at bytes
 
-async function seekTrustedTree (tree, root, bytes) {
+async function seekTrustedTree (tree, root, bytes, padding) {
   if (!bytes) return root
 
   const ite = flat.iterator(root)
@@ -904,9 +903,10 @@ async function seekTrustedTree (tree, root, bytes) {
   while ((ite.index & 1) !== 0) {
     const l = await tree.get(ite.leftChild(), false)
     if (l) {
-      if (l.size === bytes) return ite.index
-      if (l.size > bytes) continue
-      bytes -= l.size
+      const size = getUnpaddedSize(l, padding, ite)
+      if (size === bytes) return ite.index
+      if (size > bytes) continue
+      bytes -= size
       ite.sibling()
     } else {
       ite.parent()
@@ -919,8 +919,8 @@ async function seekTrustedTree (tree, root, bytes) {
 
 // try to find the block at bytes without trusting that is *is* within the root passed
 
-async function seekUntrustedTree (tree, root, bytes) {
-  const offset = await tree.byteOffset(root)
+async function seekUntrustedTree (tree, root, bytes, padding) {
+  const offset = await tree.byteOffset(root) - (padding ? padding * flat.leftSpan(root) / 2 : 0)
 
   if (offset > bytes) throw INVALID_OPERATION('Invalid seek')
   if (offset === bytes) return root
@@ -929,9 +929,9 @@ async function seekUntrustedTree (tree, root, bytes) {
 
   const node = await tree.get(root)
 
-  if (node.size <= bytes) throw INVALID_OPERATION('Invalid seek')
+  if (getUnpaddedSize(node, padding, null) <= bytes) throw INVALID_OPERATION('Invalid seek')
 
-  return seekTrustedTree(tree, root, bytes)
+  return seekTrustedTree(tree, root, bytes, padding)
 }
 
 // Below is proof production, ie, construct proofs to verify a request
@@ -1212,10 +1212,10 @@ async function generateProof (tree, block, hash, seek, upgrade) {
 
   if (node !== null && (!upgrade || node.lastIndex < upgrade.start)) {
     subTree = nodesToRoot(node.index, node.nodes, to)
-    const seekRoot = seek ? await seekUntrustedTree(tree, subTree, seek.bytes) : head
+    const seekRoot = seek ? await seekUntrustedTree(tree, subTree, seek.bytes, seek.padding) : head
     blockAndSeekProof(tree, node, seek, seekRoot, subTree, p)
   } else if ((node || seek) && upgrade) {
-    subTree = seek ? await seekFromHead(tree, to, seek.bytes) : node.index
+    subTree = seek ? await seekFromHead(tree, to, seek.bytes, seek.padding) : node.index
   }
 
   if (upgrade) {
@@ -1257,3 +1257,7 @@ async function generateProof (tree, block, hash, seek, upgrade) {
 
   return result
 }
+
+function getUnpaddedSize (node, padding, ite) {
+  return padding === 0 ? node.size : node.size - padding * (ite ? ite.countLeaves() : flat.countLeaves(node.index))
+}

package/lib/messages.js

@@ -165,16 +165,17 @@ const wire = exports.wire = {}
 
 wire.handshake = {
   preencode (state, m) {
-    c.uint.preencode(state, 0)
+    c.uint.preencode(state, 1)
     c.fixed32.preencode(state, m.capability)
   },
   encode (state, m) {
-    c.uint.encode(state, 0) // flags
+    c.uint.encode(state, m.seeks ? 1 : 0)
     c.fixed32.encode(state, m.capability)
   },
   decode (state) {
-    c.uint.decode(state) // flags
+    const flags = c.uint.decode(state)
     return {
+      seeks: (flags & 1) !== 0,
       capability: c.fixed32.decode(state)
     }
   }
@@ -200,13 +201,16 @@ const requestBlock = {
 const requestSeek = {
   preencode (state, s) {
     c.uint.preencode(state, s.bytes)
+    c.uint.preencode(state, s.padding)
   },
   encode (state, s) {
     c.uint.encode(state, s.bytes)
+    c.uint.encode(state, s.padding)
   },
   decode (state) {
     return {
-      bytes: c.uint.decode(state)
+      bytes: c.uint.decode(state),
+      padding: c.uint.decode(state)
     }
   }
 }

package/lib/replicator.js

@@ -269,6 +269,7 @@ class Peer {
     this.stream = protomux.stream
     this.protomux = protomux
     this.remotePublicKey = this.stream.remotePublicKey
+    this.remoteSupportsSeeks = false
 
     this.paused = false
 
@@ -389,7 +390,7 @@ class Peer {
     })
   }
 
-  onopen ({ capability }) {
+  onopen ({ seeks, capability }) {
     const expected = caps.replicate(this.stream.isInitiator === false, this.replicator.key, this.stream.handshakeHash)
 
     if (b4a.equals(capability, expected) !== true) { // TODO: change this to a rejection instead, less leakage
@@ -398,6 +399,7 @@ class Peer {
 
     if (this.remoteOpened === true) return
     this.remoteOpened = true
+    this.remoteSupportsSeeks = seeks
 
     this.protomux.cork()
 
@@ -811,6 +813,9 @@ class Peer {
   }
 
   _requestSeek (s) {
+    // if replicator is updating the seeks etc, bail and wait for it to drain
+    if (this.replicator._updatesPending > 0) return false
+
     const { length, fork } = this.core.tree
 
     if (fork !== this.remoteFork) return false
@@ -821,7 +826,7 @@ class Peer {
       // We need an upgrade for the seek, if non can be provided, skip
       if (req === null) return false
 
-      req.seek = { bytes: s.seeker.padding * s.seeker.start + s.seeker.bytes }
+      req.seek = this.remoteSupportsSeeks ? { bytes: s.seeker.bytes, padding: s.seeker.padding } : null
 
       s.inflight.push(req)
       this._send(req)
@@ -849,9 +854,10 @@ class Peer {
       if (h.inflight.length > 0) continue
 
       const req = this._makeRequest(false, h.priority)
+      const nodes = flatTree.depth(s.seeker.start + s.seeker.end - 1)
 
-      req.hash = { index: 2 * index, nodes: 0 }
-      req.seek = { bytes: s.seeker.padding * s.seeker.start + s.seeker.bytes }
+      req.hash = { index: 2 * index, nodes }
+      req.seek = this.remoteSupportsSeeks ? { bytes: s.seeker.bytes, padding: s.seeker.padding } : null
 
       s.inflight.push(req)
       h.inflight.push(req)
@@ -1059,23 +1065,30 @@ class Peer {
     this.inflight++
     this.replicator._inflight.add(req)
 
-    if (req.upgrade !== null && req.fork === fork) {
-      const u = this.replicator._addUpgrade()
-      u.inflight.push(req)
-    }
-
     try {
       if (req.block !== null && req.fork === fork) {
         req.block.nodes = await this.core.tree.missingNodes(2 * req.block.index)
       }
-      if (req.hash !== null && req.fork === fork) {
+      if (req.hash !== null && req.fork === fork && req.hash.nodes === 0) {
         req.hash.nodes = await this.core.tree.missingNodes(req.hash.index)
+
+        // nodes === 0, we already have it, bail
+        if (req.hash.nodes === 0 && (req.hash.index & 1) === 0) {
+          this.inflight--
+          this.replicator._resolveHashLocally(this, req)
+          return
+        }
       }
     } catch (err) {
       this.stream.destroy(err)
       return
     }
 
+    if (req.upgrade !== null && req.fork === fork) {
+      const u = this.replicator._addUpgrade()
+      u.inflight.push(req)
+    }
+
     this.wireRequest.send(req)
   }
 }
@@ -1186,7 +1199,7 @@ module.exports = class Replicator {
     for (const peer of this.peers) peer.signalUpgrade()
     if (this._blocks.isEmpty() === false) this._resolveBlocksLocally()
     if (this._upgrade !== null) this._resolveUpgradeRequest(null)
-    if (this._ranges.length !== 0 || this._seeks.length !== 0) this._updateNonPrimary()
+    if (this._ranges.length !== 0 || this._seeks.length !== 0) this._updateNonPrimary(true)
   }
 
   // Called externally when a conflict has been detected and verified
@@ -1270,7 +1283,7 @@ module.exports = class Replicator {
 
     // Trigger this to see if this is already resolved...
     // Also auto compresses the range based on local bitfield
-    this._updateNonPrimary()
+    this._updateNonPrimary(true)
 
     return ref
   }
@@ -1420,6 +1433,12 @@ module.exports = class Replicator {
     this._queued.push(b)
   }
 
+  _resolveHashLocally (peer, req) {
+    this._removeInflight(req.id)
+    this._resolveBlockRequest(this._hashes, req.hash.index / 2, null, req)
+    this.updatePeer(peer)
+  }
+
   // Runs in the background - not allowed to throw
   async _resolveBlocksLocally () {
     // TODO: check if fork compat etc. Requires that we pass down truncation info
@@ -1528,7 +1547,7 @@ module.exports = class Replicator {
   }
 
   // "slow" updates here - async but not allowed to ever throw
-  async _updateNonPrimary () {
+  async _updateNonPrimary (updateAll) {
     // Check if running, if so skip it and the running one will issue another update for us (debounce)
     while (++this._updatesPending === 1) {
       for (let i = 0; i < this._ranges.length; i++) {
@@ -1564,13 +1583,13 @@ module.exports = class Replicator {
         else s.resolve(res)
       }
 
-      if (this._inflight.idle) this.updateAll()
-
-      // No additional updates scheduled - return
-      if (--this._updatesPending === 0) return
+      // No additional updates scheduled - break
+      if (--this._updatesPending === 0) break
       // Debounce the additional updates - continue
       this._updatesPending = 0
     }
+
+    if (this._inflight.idle || updateAll) this.updateAll()
   }
 
   _maybeResolveIfAvailableRanges () {
@@ -1641,8 +1660,8 @@ module.exports = class Replicator {
       this._manifestPeer = null
     }
 
-    if (this._seeks.length > 0 || this._ranges.length > 0) this._updateNonPrimary()
-    else this.updatePeer(peer)
+    if (this._seeks.length > 0 || this._ranges.length > 0) this._updateNonPrimary(this._seeks.length > 0)
+    this.updatePeer(peer)
   }
 
   _onwant (peer, start, length) {
@@ -1921,6 +1940,7 @@ module.exports = class Replicator {
     const stream = protomux.stream
 
     peer.channel.open({
+      seeks: true,
       capability: caps.replicate(stream.isInitiator, this.key, stream.handshakeHash)
     })
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.28.11",
+  "version": "10.29.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {