hypercore 10.16.1 → 10.18.0

package/index.js

@@ -359,7 +359,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.id = z32.encode(this.key)
 
     this.replicator = new Replicator(this.core, this.key, {
-      eagerUpdate: true,
+      eagerUpgrade: true,
       allowFork: opts.allowFork !== false,
       onpeerupdate: this._onpeerupdate.bind(this),
       onupload: this._onupload.bind(this)

package/lib/messages.js

@@ -103,9 +103,10 @@ wire.request = {
     if (m.hash) requestBlock.preencode(state, m.hash)
     if (m.seek) requestSeek.preencode(state, m.seek)
     if (m.upgrade) requestUpgrade.preencode(state, m.upgrade)
+    if (m.priority) c.uint.preencode(state, m.priority)
   },
   encode (state, m) {
-    const flags = (m.block ? 1 : 0) | (m.hash ? 2 : 0) | (m.seek ? 4 : 0) | (m.upgrade ? 8 : 0)
+    const flags = (m.block ? 1 : 0) | (m.hash ? 2 : 0) | (m.seek ? 4 : 0) | (m.upgrade ? 8 : 0) | (m.priority ? 16 : 0)
 
     c.uint.encode(state, flags)
     c.uint.encode(state, m.id)
@@ -115,6 +116,7 @@ wire.request = {
     if (m.hash) requestBlock.encode(state, m.hash)
     if (m.seek) requestSeek.encode(state, m.seek)
     if (m.upgrade) requestUpgrade.encode(state, m.upgrade)
+    if (m.priority) c.uint.encode(state, m.priority)
   },
   decode (state) {
     const flags = c.uint.decode(state)
@@ -125,7 +127,8 @@ wire.request = {
       block: flags & 1 ? requestBlock.decode(state) : null,
       hash: flags & 2 ? requestBlock.decode(state) : null,
       seek: flags & 4 ? requestSeek.decode(state) : null,
-      upgrade: flags & 8 ? requestUpgrade.decode(state) : null
+      upgrade: flags & 8 ? requestUpgrade.decode(state) : null,
+      priority: flags & 16 ? c.uint.decode(state) : 0
     }
   }
 }

package/lib/receiver-queue.js

@@ -0,0 +1,63 @@
+const FIFO = require('fast-fifo')
+
+module.exports = class ReceiverQueue {
+  constructor () {
+    this.queue = new FIFO()
+    this.priority = []
+    this.requests = new Map()
+    this.length = 0
+  }
+
+  push (req) {
+    // TODO: use a heap at some point if we wanna support multiple prios
+    if (req.priority > 0) this.priority.push(req)
+    else this.queue.push(req)
+
+    this.requests.set(req.id, req)
+    this.length++
+  }
+
+  shift () {
+    while (this.priority.length > 0) {
+      const msg = this.priority.pop()
+      const req = this._processRequest(msg)
+      if (req !== null) return req
+    }
+
+    while (this.queue.length > 0) {
+      const msg = this.queue.shift()
+      const req = this._processRequest(msg)
+      if (req !== null) return req
+    }
+
+    return null
+  }
+
+  _processRequest (req) {
+    if (req.block || req.hash || req.seek || req.upgrade) {
+      this.requests.delete(req.id)
+      this.length--
+      return req
+    }
+
+    return null
+  }
+
+  delete (id) {
+    const req = this.requests.get(id)
+    if (!req) return
+
+    req.block = null
+    req.hash = null
+    req.seek = null
+    req.upgrade = null
+
+    this.requests.delete(id)
+    this.length--
+
+    if (this.length === 0) {
+      this.queue.clear()
+      this.priority = []
+    }
+  }
+}

package/lib/replicator.js

@@ -1,14 +1,22 @@
 const b4a = require('b4a')
 const safetyCatch = require('safety-catch')
 const RandomIterator = require('random-array-iterator')
+const ReceiverQueue = require('./receiver-queue')
 const RemoteBitfield = require('./remote-bitfield')
 const { REQUEST_CANCELLED, REQUEST_TIMEOUT, INVALID_CAPABILITY, SNAPSHOT_NOT_AVAILABLE } = require('./errors')
 const m = require('./messages')
 const caps = require('./caps')
 
-const DEFAULT_MAX_INFLIGHT = 32
+const DEFAULT_MAX_INFLIGHT = [32, 512]
+const SCALE_LATENCY = 50
 const DEFAULT_SEGMENT_SIZE = 128 * 1024 // 128 KiB
 
+const PRIORITY = {
+  NORMAL: 0,
+  HIGH: 1,
+  VERY_HIGH: 2
+}
+
 class Attachable {
   constructor () {
     this.resolved = false
@@ -94,17 +102,21 @@ class Attachable {
 }
 
 class BlockRequest extends Attachable {
-  constructor (tracker, index) {
+  constructor (tracker, index, priority) {
     super()
 
     this.index = index
+    this.priority = priority
     this.inflight = []
     this.queued = false
     this.tracker = tracker
   }
 
   _unref () {
-    if (this.inflight.length > 0) return
+    for (const req of this.inflight) {
+      req.peer._cancelRequest(req.id)
+    }
+
     this.tracker.remove(this.index)
   }
 }
@@ -232,11 +244,11 @@ class BlockTracker {
     return this._map.get(index) || null
   }
 
-  add (index) {
+  add (index, priority) {
     let b = this._map.get(index)
     if (b) return b
 
-    b = new BlockRequest(this, index)
+    b = new BlockRequest(this, index, priority)
     this._map.set(index, b)
 
     return b
@@ -266,7 +278,7 @@ class Peer {
 
     this.wireSync = this.channel.messages[0]
     this.wireRequest = this.channel.messages[1]
-    this.wireCancel = null
+    this.wireCancel = this.channel.messages[2]
     this.wireData = this.channel.messages[3]
     this.wireNoData = this.channel.messages[4]
     this.wireWant = this.channel.messages[5]
@@ -275,8 +287,11 @@ class Peer {
     this.wireRange = this.channel.messages[8]
     this.wireExtension = this.channel.messages[9]
 
+    this.receiverQueue = new ReceiverQueue()
+    this.receiverBusy = false
+
     this.inflight = 0
-    this.maxInflight = DEFAULT_MAX_INFLIGHT
+    this.inflightRange = DEFAULT_MAX_INFLIGHT
     this.dataProcessing = 0
 
     this.canUpgrade = true
@@ -310,6 +325,18 @@ class Peer {
     replicator._ifAvailable++
   }
 
+  get remoteContiguousLength () {
+    return this.remoteBitfield.findFirst(false, 0)
+  }
+
+  getMaxInflight () {
+    const stream = this.stream.rawStream
+    if (!stream.udx) return Math.min(this.inflightRange[1], this.inflightRange[0] * 3)
+
+    const scale = stream.rtt <= SCALE_LATENCY ? 1 : stream.rtt / SCALE_LATENCY
+    return Math.round(Math.min(this.inflightRange[1], this.inflightRange[0] * scale))
+  }
+
   signalUpgrade () {
     if (this._shouldUpdateCanUpgrade() === true) this._updateCanUpgradeAndSync()
     else this.sendSync()
@@ -495,6 +522,35 @@ class Peer {
   }
 
   async onrequest (msg) {
+    if (!this.protomux.drained || this.receiverQueue.length) {
+      this.receiverQueue.push(msg)
+      return
+    }
+
+    await this._handleRequest(msg)
+  }
+
+  oncancel (msg) {
+    this.receiverQueue.delete(msg.request)
+  }
+
+  ondrain () {
+    return this._handleRequests()
+  }
+
+  async _handleRequests () {
+    if (this.receiverBusy) return
+    this.receiverBusy = true
+
+    while (this.remoteOpened && this.protomux.drained && this.receiverQueue.length > 0) {
+      const msg = this.receiverQueue.shift()
+      await this._handleRequest(msg)
+    }
+
+    this.receiverBusy = false
+  }
+
+  async _handleRequest (msg) {
     let proof = null
 
     // TODO: could still be answerable if (index, fork) is an ancestor of the current fork
@@ -527,6 +583,16 @@ class Peer {
     })
   }
 
+  _cancelRequest (id) {
+    const exists = this.replicator._inflight.get(id)
+    if (!exists) return
+
+    this.inflight--
+    this.replicator._inflight.remove(id)
+
+    this.wireCancel.send({ request: id })
+  }
+
   _checkIfConflict (err) {
     this.paused = true
 
@@ -619,7 +685,16 @@ class Peer {
   onrange ({ drop, start, length }) {
     const has = drop === false
 
-    this.remoteBitfield.setRange(start, length, has)
+    if (length === 1) {
+      this.remoteBitfield.setRange(start, length, has)
+    } else {
+      const rangeStart = this.remoteBitfield.findFirst(!has, start)
+      const rangeLength = length - (rangeStart - start)
+
+      if (rangeLength > 0) {
+        this.remoteBitfield.setRange(rangeStart, rangeLength, has)
+      }
+    }
 
     if (drop === false) this._update()
   }
@@ -653,7 +728,7 @@ class Peer {
     this.protomux.uncork()
   }
 
-  _makeRequest (needsUpgrade) {
+  _makeRequest (needsUpgrade, priority) {
     if (needsUpgrade === true && this.replicator._shouldUpgrade(this) === false) {
       return null
     }
@@ -671,7 +746,8 @@ class Peer {
       seek: null,
       upgrade: needsUpgrade === false
         ? null
-        : { start: this.core.tree.length, length: this.remoteLength - this.core.tree.length }
+        : { start: this.core.tree.length, length: this.remoteLength - this.core.tree.length },
+      priority
     }
   }
 
@@ -718,10 +794,10 @@ class Peer {
       if (b !== null && b.inflight.length > 0) continue
 
       // Block is not inflight, but we only want the hash, check if that is inflight
-      const h = this.replicator._hashes.add(index)
+      const h = this.replicator._hashes.add(index, PRIORITY.NORMAL)
       if (h.inflight.length > 0) continue
 
-      const req = this._makeRequest(false)
+      const req = this._makeRequest(false, h.priority)
 
       req.hash = { index: 2 * index, nodes: 0 }
       req.seek = { bytes: s.seeker.bytes }
@@ -750,7 +826,7 @@ class Peer {
       return false
     }
 
-    const req = this._makeRequest(b.index >= length)
+    const req = this._makeRequest(b.index >= length, b.priority)
     if (req === null) return false
 
     req.block = { index: b.index, nodes: 0 }
@@ -783,10 +859,10 @@ class Peer {
       if (this.remoteBitfield.get(index) === false) continue
       if (this.core.bitfield.get(index) === true) continue
 
-      const b = this.replicator._blocks.add(index)
+      const b = this.replicator._blocks.add(index, PRIORITY.NORMAL)
       if (b.inflight.length > 0) continue
 
-      const req = this._makeRequest(index >= length)
+      const req = this._makeRequest(index >= length, b.priority)
 
       // If the request cannot be satisfied, dealloc the block request if no one is subscribed to it
       if (req === null) {
@@ -849,6 +925,8 @@ class Peer {
   }
 
   _maybeWant (start, length = 1) {
+    if (start + length <= this.remoteContiguousLength) return
+
     let i = Math.floor(start / DEFAULT_SEGMENT_SIZE)
     const n = Math.ceil((start + length) / DEFAULT_SEGMENT_SIZE)
 
@@ -902,6 +980,7 @@ module.exports = class Replicator {
     this.peers = []
     this.findingPeers = 0 // updateable from the outside
 
+    this._attached = new Set()
     this._inflight = new InflightTracker()
     this._blocks = new BlockTracker()
     this._hashes = new BlockTracker()
@@ -917,6 +996,13 @@ module.exports = class Replicator {
     this._ifAvailable = 0
     this._updatesPending = 0
     this._applyingReorg = null
+
+    const self = this
+    this._onstreamclose = onstreamclose
+
+    function onstreamclose () {
+      self.detachFrom(this.userData)
+    }
   }
 
   cork () {
@@ -998,7 +1084,7 @@ module.exports = class Replicator {
   }
 
   addBlock (session, index) {
-    const b = this._blocks.add(index)
+    const b = this._blocks.add(index, PRIORITY.HIGH)
     const ref = b.attach(session)
 
     this._queueBlock(b)
@@ -1492,7 +1578,7 @@ module.exports = class Replicator {
   }
 
   _updatePeer (peer) {
-    if (peer.paused || peer.inflight >= peer.maxInflight) {
+    if (peer.paused || peer.inflight >= peer.getMaxInflight()) {
       return false
     }
 
@@ -1518,7 +1604,7 @@ module.exports = class Replicator {
   }
 
   _updatePeerNonPrimary (peer) {
-    if (peer.paused || peer.inflight >= peer.maxInflight) {
+    if (peer.paused || peer.inflight >= peer.getMaxInflight()) {
       return false
     }
 
@@ -1587,21 +1673,36 @@ module.exports = class Replicator {
   attachTo (protomux, session) {
     const makePeer = this._makePeer.bind(this, protomux, session)
 
+    this._attached.add(protomux)
     protomux.pair({ protocol: 'hypercore/alpha', id: this.discoveryKey }, makePeer)
+    protomux.stream.setMaxListeners(0)
+    protomux.stream.on('close', this._onstreamclose)
+
     this._ifAvailable++
     protomux.stream.opened.then((opened) => {
       this._ifAvailable--
+
       if (opened) makePeer()
       else if (session) session.close().catch(noop)
       this._checkUpgradeIfAvailable()
     })
   }
 
+  detachFrom (protomux) {
+    if (this._attached.delete(protomux)) {
+      protomux.stream.removeListener('close', this._onstreamclose)
+      protomux.unpair({ protocol: 'hypercore/alpha', id: this.discoveryKey })
+    }
+  }
+
   destroy () {
     for (const peer of this.peers) {
-      peer.protomux.unpair({ protocol: 'hypercore/alpha', id: this.discoveryKey })
+      this.detachFrom(peer.protomux)
       peer.channel.close()
     }
+    for (const protomux of this._attached) {
+      this.detachFrom(protomux)
+    }
   }
 
   _makePeer (protomux, session) {
@@ -1616,7 +1717,7 @@ module.exports = class Replicator {
       messages: [
         { encoding: m.wire.sync, onmessage: onwiresync },
         { encoding: m.wire.request, onmessage: onwirerequest },
-        null, // oncancel
+        { encoding: m.wire.cancel, onmessage: onwirecancel },
         { encoding: m.wire.data, onmessage: onwiredata },
         { encoding: m.wire.noData, onmessage: onwirenodata },
         { encoding: m.wire.want, onmessage: onwirewant },
@@ -1626,7 +1727,8 @@ module.exports = class Replicator {
         { encoding: m.wire.extension, onmessage: onwireextension }
       ],
       onopen: onwireopen,
-      onclose: onwireclose
+      onclose: onwireclose,
+      ondrain: onwiredrain
     })
 
     if (channel === null) return onnochannel()
@@ -1708,6 +1810,10 @@ function onwireclose (isRemote, c) {
   return c.userData.onclose(isRemote)
 }
 
+function onwiredrain (c) {
+  return c.userData.ondrain()
+}
+
 function onwiresync (m, c) {
   return c.userData.onsync(m)
 }
@@ -1716,6 +1822,10 @@ function onwirerequest (m, c) {
   return c.userData.onrequest(m)
 }
 
+function onwirecancel (m, c) {
+  return c.userData.oncancel(m)
+}
+
 function onwiredata (m, c) {
   return c.userData.ondata(m)
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.16.1",
+  "version": "10.18.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -41,10 +41,11 @@
     "compact-encoding": "^2.11.0",
     "crc-universal": "^1.0.2",
     "events": "^3.3.0",
+    "fast-fifo": "^1.3.0",
     "flat-tree": "^1.9.0",
     "hypercore-crypto": "^3.2.1",
     "is-options": "^1.0.1",
-    "protomux": "^3.4.0",
+    "protomux": "^3.5.0",
     "quickbit-universal": "^2.1.1",
     "random-access-file": "^4.0.0",
     "random-array-iterator": "^1.0.0",
@@ -60,7 +61,10 @@
     "random-access-memory": "^6.1.0",
     "random-access-memory-overlay": "^3.0.0",
     "range-parser": "^1.2.1",
+    "speedometer": "^1.1.0",
     "standard": "^17.0.0",
-    "tmp-promise": "^3.0.2"
+    "tiny-byte-size": "^1.1.0",
+    "tmp-promise": "^3.0.2",
+    "udx-native": "^1.6.1"
   }
 }