hypercore 10.0.0-alpha.5 → 10.0.0-alpha.50

package/lib/replicator.js

@@ -1,838 +1,1607 @@
-const Protocol = require('./protocol')
-const RemoteBitfield = require('./remote-bitfield')
+const b4a = require('b4a')
+const safetyCatch = require('safety-catch')
 const RandomIterator = require('random-array-iterator')
+const RemoteBitfield = require('./remote-bitfield')
+const { REQUEST_CANCELLED, INVALID_CAPABILITY, SNAPSHOT_NOT_AVAILABLE } = require('./errors')
+const m = require('./messages')
+const caps = require('./caps')
+
+const DEFAULT_MAX_INFLIGHT = 32
+
+class Attachable {
+  constructor () {
+    this.resolved = false
+    this.refs = []
+  }
+
+  attach (session) {
+    const r = {
+      context: this,
+      session,
+      sindex: 0,
+      rindex: 0,
+      snapshot: true,
+      resolve: null,
+      reject: null,
+      promise: null
+    }
 
-const PKG = require('../package.json')
-const USER_AGENT = PKG.name + '/' + PKG.version + '@nodejs'
+    r.sindex = session.push(r) - 1
+    r.rindex = this.refs.push(r) - 1
+    r.promise = new Promise((resolve, reject) => {
+      r.resolve = resolve
+      r.reject = reject
+    })
 
-class RemoteState {
-  constructor (core) {
-    this.receivedInfo = false
-    this.inflight = 0
-    this.maxInflight = 16
-    this.bitfield = new RemoteBitfield()
-    this.length = 0
-    this.fork = 0
+    return r
   }
-}
 
-class InvertedPromise {
-  constructor (resolve, reject, index) {
-    this.index = index
-    this.resolve = resolve
-    this.reject = reject
+  detach (r, err = null) {
+    if (r.context !== this) return false
+
+    this._detach(r)
+    this._cancel(r, err)
+    this.gc()
+
+    return true
   }
-}
 
-class Request {
-  constructor (index, seek, nodes) {
-    this.peer = null
-    this.index = index
-    this.seek = seek
-    this.value = seek === 0
-    this.nodes = nodes
-    this.promises = []
+  _detach (r) {
+    const rh = this.refs.pop()
+    const sh = r.session.pop()
+
+    if (r.rindex < this.refs.length) this.refs[rh.rindex = r.rindex] = rh
+    if (r.sindex < r.session.length) r.session[sh.sindex = r.sindex] = sh
+
+    r.context = null
+
+    return r
   }
 
-  createPromise () {
-    return new Promise((resolve, reject) => {
-      this.promises.push(new InvertedPromise(resolve, reject, this.promises.length))
-    })
+  gc () {
+    if (this.refs.length === 0) this._unref()
+  }
+
+  _cancel (r, err) {
+    r.reject(err || REQUEST_CANCELLED())
+  }
+
+  _unref () {
+    // overwrite me
   }
 
   resolve (val) {
-    for (let i = 0; i < this.promises.length; i++) {
-      this.promises[i].resolve(val)
+    this.resolved = true
+    while (this.refs.length > 0) {
+      this._detach(this.refs[this.refs.length - 1]).resolve(val)
     }
   }
 
   reject (err) {
-    for (let i = 0; i < this.promises.length; i++) {
-      this.promises[i].reject(err)
+    this.resolved = true
+    while (this.refs.length > 0) {
+      this._detach(this.refs[this.refs.length - 1]).reject(err)
     }
   }
 }
 
-class Upgrade {
-  constructor (minLength) {
-    this.minLength = minLength
-    this.promises = []
-  }
+class BlockRequest extends Attachable {
+  constructor (tracker, index) {
+    super()
 
-  update (peers, fork) {
-    for (const peer of peers) {
-      if (peer.state.length >= this.minLength && !paused(peer, fork)) return true
-      if (!peer.state.receivedInfo) return true
-    }
+    this.index = index
+    this.inflight = []
+    this.queued = false
+    this.tracker = tracker
+  }
 
-    return false
+  _unref () {
+    if (this.inflight.length > 0) return
+    this.tracker.remove(this.index)
   }
+}
 
-  createPromise () {
-    return new Promise((resolve, reject) => {
-      this.promises.push(new InvertedPromise(resolve, reject, this.promises.length))
-    })
+class RangeRequest extends Attachable {
+  constructor (ranges, start, end, linear, ifAvailable, blocks) {
+    super()
+
+    this.start = start
+    this.end = end
+    this.linear = linear
+    this.ifAvailable = ifAvailable
+    this.blocks = blocks
+    this.ranges = ranges
+
+    // As passed by the user, immut
+    this.userStart = start
+    this.userEnd = end
   }
 
-  resolve (val) {
-    for (let i = 0; i < this.promises.length; i++) {
-      this.promises[i].resolve(val)
-    }
+  _unref () {
+    const i = this.ranges.indexOf(this)
+    if (i === -1) return
+    const h = this.ranges.pop()
+    if (i < this.ranges.length - 1) this.ranges[i] = h
   }
 
-  reject (err) {
-    for (let i = 0; i < this.promises.length; i++) {
-      this.promises[i].reject(err)
-    }
+  _cancel (r) {
+    r.resolve(false)
   }
 }
 
-class UpgradeLock {
-  constructor (peer, length) {
-    this.peer = peer
+class UpgradeRequest extends Attachable {
+  constructor (replicator, fork, length) {
+    super()
+
+    this.fork = fork
     this.length = length
-    this.resolve = null
-    this.promise = new Promise((resolve) => { this.resolve = resolve })
+    this.inflight = []
+    this.replicator = replicator
+  }
+
+  _unref () {
+    if (this.replicator.eagerUpgrade === true || this.inflight.length > 0) return
+    this.replicator._upgrade = null
+  }
+
+  _cancel (r) {
+    r.resolve(false)
   }
 }
 
-class Seek {
-  constructor (seeker) {
-    this.request = null
+class SeekRequest extends Attachable {
+  constructor (seeks, seeker) {
+    super()
+
     this.seeker = seeker
-    this.promise = null
-    this.finished = false
+    this.inflight = []
+    this.seeks = seeks
   }
 
-  async update () {
-    const res = await this.seeker.update()
-    if (!res) return false
-    this.finished = true
-    this.promise.resolve(res)
-    return true
+  _unref () {
+    if (this.inflight.length > 0) return
+    const i = this.seeks.indexOf(this)
+    if (i === -1) return
+    const h = this.seeks.pop()
+    if (i < this.seeks.length - 1) this.seeks[i] = h
   }
+}
 
-  createPromise () {
-    return new Promise((resolve, reject) => {
-      this.promise = new InvertedPromise(resolve, reject, 0)
-    })
+class InflightTracker {
+  constructor () {
+    this._requests = []
+    this._free = []
   }
-}
 
-class Range {
-  constructor (start, end, linear) {
-    this.start = start
-    this.end = end
-    this.linear = !!linear
-    this.promise = null
-    this.done = false
+  get idle () {
+    return this._requests.length === this._free.length
+  }
 
-    this._inv = null
-    this._start = start // can be updated
-    this._ranges = null // set be replicator
-    this._resolved = false
-    this._error = null
+  * [Symbol.iterator] () {
+    for (const req of this._requests) {
+      if (req !== null) yield req
+    }
   }
 
-  contains (req) {
-    return this._start <= req.index && req.index < this.end
+  add (req) {
+    const id = this._free.length ? this._free.pop() : this._requests.push(null)
+
+    req.id = id
+    this._requests[id - 1] = req
+    return req
   }
 
-  update (bitfield) {
-    if (this.end === -1) {
-      while (bitfield.get(this._start)) this._start++
-      return false
-    }
+  get (id) {
+    return id <= this._requests.length ? this._requests[id - 1] : null
+  }
 
-    for (; this._start < this.end; this._start++) {
-      if (!bitfield.get(this._start)) return false
+  remove (id) {
+    if (id <= this._requests.length) {
+      this._requests[id - 1] = null
+      this._free.push(id)
     }
-
-    return true
   }
+}
 
-  resolve (done) {
-    this._done(null, done)
+class BlockTracker {
+  constructor () {
+    this._map = new Map()
   }
 
-  destroy (err) {
-    this._done(err, false)
+  [Symbol.iterator] () {
+    return this._map.values()
   }
 
-  downloaded () {
-    if (this.promise) return this.promise
-    if (!this.done) return this._makePromise()
-    if (this._error !== null) return Promise.reject(this._error)
-    return Promise.resolve(this._resolved)
+  isEmpty () {
+    return this._map.size === 0
   }
 
-  _done (err, done) {
-    if (this.done) return
-    this.done = true
+  has (index) {
+    return this._map.has(index)
+  }
 
-    if (this._ranges) {
-      const i = this._ranges.indexOf(this)
+  get (index) {
+    return this._map.get(index) || null
+  }
 
-      if (i === this._ranges.length - 1) this._ranges.pop()
-      else if (i > -1) this._ranges[i] = this._ranges.pop()
-    }
+  add (index) {
+    let b = this._map.get(index)
+    if (b) return b
 
-    this._ranges = null
-    this._resolved = done
-    this._error = err
+    b = new BlockRequest(this, index)
+    this._map.set(index, b)
 
-    if (this._inv === null) return
-    if (err) this._inv.reject(err)
-    else this._inv.resolve(done)
+    return b
   }
 
-  _makePromise () {
-    this.promise = new Promise((resolve, reject) => {
-      this._inv = new InvertedPromise(resolve, reject, 0)
-    })
-
-    return this.promise
+  remove (index) {
+    const b = this.get(index)
+    this._map.delete(index)
+    return b
   }
 }
 
-class RequestPool {
-  constructor (replicator, core) {
+class Peer {
+  constructor (replicator, protomux, channel) {
+    this.core = replicator.core
     this.replicator = replicator
-    this.core = core
-    this.pending = []
-    this.seeks = []
-    this.ranges = []
-    this.requests = new Map()
-    this.upgrading = null
-    this.unforking = null
-    this.eagerUpgrades = true
-    this.paused = false
-  }
-
-  // We could make this faster by using some memory of each peer to store what they are involved in,
-  // but that might not be worth the cost/complexity.
-  clear (peer) {
-    peer.state.inflight = 0
-
-    for (const seek of this.seeks) {
-      if (seek.request && seek.request.peer === peer) {
-        // TODO: should prob remove cancel this request all together if no one else wants it
-        // and nothing is inflight
-        seek.request = null
-      }
-    }
-    for (const req of this.requests.values()) {
-      if (req.peer === peer) {
-        req.peer = null
-        this.pending.push(req)
-      }
-    }
-    if (this.upgrading && this.upgrading.peer === peer) {
-      this.upgrading.resolve()
-      this.upgrading = null
-    }
-  }
+    this.stream = protomux.stream
+    this.protomux = protomux
+    this.remotePublicKey = this.stream.remotePublicKey
+
+    this.channel = channel
+    this.channel.userData = this
+
+    this.wireSync = this.channel.messages[0]
+    this.wireRequest = this.channel.messages[1]
+    this.wireCancel = null
+    this.wireData = this.channel.messages[3]
+    this.wireNoData = this.channel.messages[4]
+    this.wireWant = this.channel.messages[5]
+    this.wireUnwant = this.channel.messages[6]
+    this.wireBitfield = this.channel.messages[7]
+    this.wireRange = this.channel.messages[8]
+    this.wireExtension = this.channel.messages[9]
 
-  isRequesting (index) {
-    return this.requests.has(index)
-  }
+    this.inflight = 0
+    this.maxInflight = DEFAULT_MAX_INFLIGHT
+    this.dataProcessing = 0
 
-  update (peer) {
-    if (peer.state.inflight >= peer.state.maxInflight) return false
-    if (this.paused) return false
+    this.canUpgrade = true
 
-    if (peer.state.fork > this.core.tree.fork) {
-      // we message them in the info recv
-      return true
-    }
+    this.needsSync = false
+    this.syncsProcessing = 0
 
-    // technically we'd like to run seeks at the same custom prio as reqs
-    // but this is a lot simpler and they should run asap anyway as they
-    // are super low cost (hash only request)
-    for (const seek of this.seeks) {
-      if (this._updateSeek(peer, seek)) return true
-    }
+    // TODO: tweak pipelining so that data sent BEFORE remoteOpened is not cap verified!
+    // we might wanna tweak that with some crypto, ie use the cap to encrypt it...
+    // or just be aware of that, to only push non leaky data
 
-    if (this.pendingUpgrade) {
-      if (this._updateUpgrade(peer)) return true
-    }
+    this.remoteOpened = false
+    this.remoteBitfield = new RemoteBitfield()
 
-    const pending = new RandomIterator(this.pending) // can be cached
-    for (const req of pending) {
-      if (this._updatePeer(peer, req)) {
-        pending.dequeue()
-        return true
-      }
-    }
+    this.remoteFork = 0
+    this.remoteLength = 0
+    this.remoteCanUpgrade = false
+    this.remoteUploading = true
+    this.remoteDownloading = true
+    this.remoteSynced = false
 
-    const ranges = new RandomIterator(this.ranges) // can be cached
-    for (const range of ranges) {
-      if (this._updateRange(peer, range)) return true
-    }
+    this.lengthAcked = 0
 
-    if (this.eagerUpgrades && !this.upgrading) {
-      return this._updateUpgrade(peer)
-    }
+    this.extensions = new Map()
+    this.lastExtensionSent = ''
+    this.lastExtensionRecv = ''
 
-    return false
+    replicator._ifAvailable++
   }
 
-  _updateSeek (peer, seek) {
-    if (seek.request) return false
-    // We have to snapshot the nodes here now, due to the caching of the request...
-    const nodes = log2(seek.seeker.end - seek.seeker.start)
-    seek.request = this._requestRange(peer, seek.seeker.start, seek.seeker.end, seek.seeker.bytes, nodes)
-    return seek.request !== null
+  signalUpgrade () {
+    if (this._shouldUpdateCanUpgrade() === true) this._updateCanUpgradeAndSync()
+    else this.sendSync()
   }
 
-  _updatePeer (peer, req) {
-    const remote = peer.state.bitfield
-    const local = this.core.bitfield
+  broadcastRange (start, length, drop) {
+    this.wireRange.send({
+      drop,
+      start,
+      length
+    })
+  }
 
-    if (!remote.get(req.index) || local.get(req.index)) return false
+  extension (name, message) {
+    this.wireExtension.send({ name: name === this.lastExtensionSent ? '' : name, message })
+    this.lastExtensionSent = name
+  }
 
-    this.send(peer, req)
-    return true
+  onextension (message) {
+    const name = message.name || this.lastExtensionRecv
+    this.lastExtensionRecv = name
+    const ext = this.extensions.get(name)
+    if (ext) ext._onmessage({ start: 0, end: message.byteLength, buffer: message.message }, this)
   }
 
-  _updateRange (peer, range) {
-    const end = range.end === -1 ? peer.state.length : range.end
-    if (end <= range._start) return false
+  sendSync () {
+    if (this.syncsProcessing !== 0) {
+      this.needsSync = true
+      return
+    }
+
+    if (this.core.tree.fork !== this.remoteFork) {
+      this.canUpgrade = false
+    }
 
-    if (range.linear) return !!this._requestRange(peer, range._start, end, 0, 0)
+    this.needsSync = false
 
-    const r = range._start + Math.floor(Math.random() * (end - range._start))
-    return !!(this._requestRange(peer, r, end, 0, 0) || this._requestRange(peer, range._start, r, 0, 0))
+    this.wireSync.send({
+      fork: this.core.tree.fork,
+      length: this.core.tree.length,
+      remoteLength: this.core.tree.fork === this.remoteFork ? this.remoteLength : 0,
+      canUpgrade: this.canUpgrade,
+      uploading: true,
+      downloading: true
+    })
   }
 
-  _updateUpgrade (peer) {
-    const minLength = this.pendingUpgrade
-      ? this.pendingUpgrade.minLength
-      : this.core.tree.length + 1
+  onopen ({ capability }) {
+    const expected = caps.replicate(this.stream.isInitiator === false, this.replicator.key, this.stream.handshakeHash)
 
-    if (this.upgrading || peer.state.length < minLength) return false
-    this.upgrading = new UpgradeLock(peer, peer.state.length)
-
-    const data = {
-      fork: this.core.tree.fork,
-      seek: null,
-      block: null,
-      upgrade: { start: this.core.tree.length, length: peer.state.length - this.core.tree.length }
+    if (b4a.equals(capability, expected) !== true) { // TODO: change this to a rejection instead, less leakage
+      throw INVALID_CAPABILITY('Remote sent an invalid replication capability')
     }
 
-    peer.request(data)
-    return true
-  }
+    if (this.remoteOpened === true) return
+    this.remoteOpened = true
 
-  checkTimeouts (peers) {
-    if (!this.pendingUpgrade || this.upgrading) return
-    if (this.pendingUpgrade.update(peers, this.core.tree.fork)) return
-    this.pendingUpgrade.resolve(false)
-    this.pendingUpgrade = null
-  }
+    this.protomux.cork()
+
+    this.sendSync()
 
-  _requestRange (peer, start, end, seek, nodes) {
-    const remote = peer.state.bitfield
-    const local = this.core.bitfield
+    const p = pages(this.core)
+
+    for (let index = 0; index < p.length; index++) {
+      this.wireBitfield.send({
+        start: index * this.core.bitfield.pageSize,
+        bitfield: p[index]
+      })
+    }
 
-    // TODO: use 0 instead of -1 as end=0 should never be added!
-    if (end === -1) end = peer.state.length
+    this.replicator._ifAvailable--
+    this.replicator._addPeer(this)
 
-    for (let i = start; i < end; i++) {
-      if (!remote.get(i) || local.get(i)) continue
-      // TODO: if this was a NO_VALUE request, retry if no blocks can be found elsewhere
-      if (this.requests.has(i)) continue
+    this.protomux.uncork()
+  }
 
-      // TODO: if seeking and i >= core.length, let that takes precendance in the upgrade req
-      const req = new Request(i, i < this.core.tree.length ? seek : 0, nodes)
-      this.requests.set(i, req)
-      this.send(peer, req)
-      return req
+  onclose (isRemote) {
+    if (this.remoteOpened === false) {
+      this.replicator._ifAvailable--
+      this.replicator.updateAll()
+      return
     }
 
-    return null
+    this.remoteOpened = false
+    this.replicator._removePeer(this)
   }
 
-  // send handles it's own errors so we do not need to await/catch it
-  async send (peer, req) {
-    req.peer = peer
-    peer.state.inflight++ // TODO: a non value request should count less than a value one
+  async onsync ({ fork, length, remoteLength, canUpgrade, uploading, downloading }) {
+    const lengthChanged = length !== this.remoteLength
+    const sameFork = fork === this.core.tree.fork
 
-    // TODO: also check if remote can even upgrade us lol
-    let needsUpgrade = peer.state.length > this.core.tree.length || !!(!this.upgrading && this.pendingUpgrade)
-    const fork = this.core.tree.fork
-    let upgrading = false
+    this.remoteSynced = true
+    this.remoteFork = fork
+    this.remoteLength = length
+    this.remoteCanUpgrade = canUpgrade
+    this.remoteUploading = uploading
+    this.remoteDownloading = downloading
 
-    while (needsUpgrade) {
-      if (!this.upgrading) {
-        if (peer.state.length <= this.core.tree.length) {
-          needsUpgrade = false
-          break
-        }
-        // TODO: if the peer fails, we need to resolve the promise as well woop woop
-        // so we need some tracking mechanics for upgrades in general.
-        this.upgrading = new UpgradeLock(peer, peer.state.length)
-        upgrading = true
-        break
-      }
-      if (req.index < this.core.tree.length) {
-        needsUpgrade = false
-        break
-      }
+    this.lengthAcked = sameFork ? remoteLength : 0
+    this.syncsProcessing++
+
+    this.replicator._updateFork(this)
 
-      await this.upgrading.promise
-      needsUpgrade = peer.state.length > this.core.tree.length || !!(!this.upgrading && this.pendingUpgrade)
+    if (this.remoteLength > this.core.tree.length && this.lengthAcked === this.core.tree.length) {
+      if (this.replicator._addUpgradeMaybe() !== null) this._update()
     }
 
-    const data = {
-      fork,
-      seek: req.seek ? { bytes: req.seek } : null,
-      block: { index: req.index, value: req.value, nodes: 0 },
-      upgrade: needsUpgrade ? { start: this.core.tree.length, length: peer.state.length - this.core.tree.length } : null
+    const upgrade = (lengthChanged === false || sameFork === false)
+      ? this.canUpgrade && sameFork
+      : await this._canUpgrade(length, fork)
+
+    if (length === this.remoteLength && fork === this.core.tree.fork) {
+      this.canUpgrade = upgrade
     }
 
-    if (data.block.index < this.core.tree.length || this.core.truncating > 0) {
-      try {
-        data.block.nodes = Math.max(req.nodes, await this.core.tree.nodes(data.block.index * 2))
-      } catch (err) {
-        console.error('TODO handle me:', err.stack)
-      }
+    if (--this.syncsProcessing !== 0) return // ie not latest
+
+    if (this.needsSync === true || (this.core.tree.fork === this.remoteFork && this.core.tree.length > this.remoteLength)) {
+      this.signalUpgrade()
     }
 
-    if (peer.destroyed) {
-      req.peer = null
-      this.pending.push(req)
-      if (upgrading) {
-        this.upgrading.resolve()
-        this.upgrading = null
-      }
-      this.replicator.updateAll()
+    this._update()
+  }
+
+  _shouldUpdateCanUpgrade () {
+    return this.core.tree.fork === this.remoteFork &&
+      this.core.tree.length > this.remoteLength &&
+      this.canUpgrade === false &&
+      this.syncsProcessing === 0
+  }
+
+  async _updateCanUpgradeAndSync () {
+    const { length, fork } = this.core.tree
+
+    const canUpgrade = await this._canUpgrade(this.remoteLength, this.remoteFork)
+
+    if (this.syncsProcessing > 0 || length !== this.core.tree.length || fork !== this.core.tree.fork) {
       return
     }
-
-    if (fork !== this.core.tree.fork || paused(peer, this.core.tree.fork) || this.core.truncating > 0) {
-      if (peer.state.inflight > 0) peer.state.inflight--
-      if (req.promises.length) { // someone is eagerly waiting for this request
-        req.peer = null // resend on some other peer
-        this.pending.push(req)
-      } else { // otherwise delete the request
-        this.requests.delete(req.index)
-      }
-      if (upgrading) {
-        this.upgrading.resolve()
-        this.upgrading = null
-      }
+    if (canUpgrade === this.canUpgrade) {
       return
     }
 
-    peer.request(data)
+    this.canUpgrade = canUpgrade
+    this.sendSync()
   }
 
-  async _onupgrade (proof, peer) {
-    if (!this.upgrading || !proof.upgrade) return
-    if (this.unforking) return
-
-    await this.core.verify(proof, peer)
+  // Safe to call in the background - never fails
+  async _canUpgrade (remoteLength, remoteFork) {
+    if (remoteFork !== this.core.tree.fork) return false
 
-    // TODO: validate that we actually upgraded our length as well
-    this.upgrading.resolve()
-    this.upgrading = null
+    if (remoteLength === 0) return true
+    if (remoteLength >= this.core.tree.length) return false
 
-    if (this.pendingUpgrade) {
-      this.pendingUpgrade.resolve(true)
-      this.pendingUpgrade = null
-    }
+    try {
+      // Rely on caching to make sure this is cheap...
+      const canUpgrade = await this.core.tree.upgradeable(remoteLength)
 
-    if (this.seeks.length) await this._updateSeeks(null)
+      if (remoteFork !== this.core.tree.fork) return false
 
-    this.update(peer)
+      return canUpgrade
+    } catch {
+      return false
+    }
   }
 
-  async _onfork (proof, peer) {
-    // TODO: if proof is from a newer fork than currently unforking, restart
+  async _getProof (msg) {
+    const proof = await this.core.tree.proof(msg)
 
-    if (this.unforking) {
-      await this.unforking.update(proof)
-    } else {
-      const reorg = await this.core.tree.reorg(proof)
-      const verified = reorg.signedBy(this.core.header.signer.publicKey)
-      if (!verified) throw new Error('Remote signature could not be verified')
-      this.unforking = reorg
+    if (proof.block) {
+      if (msg.fork !== this.core.tree.fork) return null
+      proof.block.value = await this.core.blocks.get(msg.block.index)
     }
 
-    if (!this.unforking.finished) {
-      for (let i = this.unforking.want.start; i < this.unforking.want.end; i++) {
-        if (peer.state.bitfield.get(i)) {
-          const data = {
-            fork: this.unforking.fork,
-            seek: null,
-            block: { index: i, value: false, nodes: this.unforking.want.nodes },
-            upgrade: null
-          }
-          peer.request(data)
-          return
-        }
+    return proof
+  }
+
+  async onrequest (msg) {
+    let proof = null
+
+    // TODO: could still be answerable if (index, fork) is an ancestor of the current fork
+    if (msg.fork === this.core.tree.fork) {
+      try {
+        proof = await this._getProof(msg)
+      } catch (err) { // TODO: better error handling here, ie custom errors
+        safetyCatch(err)
       }
-      return
     }
 
-    const reorg = this.unforking
-    this.unforking = null
-    await this.core.reorg(reorg)
+    if (proof !== null) {
+      if (proof.block !== null) {
+        this.replicator.onupload(proof.block.index, proof.block.value, this)
+      }
 
-    // reset ranges, also need to reset seeks etc
-    for (const r of this.ranges) {
-      r._start = 0
+      this.wireData.send({
+        request: msg.id,
+        fork: msg.fork,
+        block: proof.block,
+        hash: proof.hash,
+        seek: proof.seek,
+        upgrade: proof.upgrade
+      })
+      return
     }
 
-    this.replicator.updateAll()
-
-    // TODO: we gotta clear out old requests as well here pointing at the old fork
+    this.wireNoData.send({
+      request: msg.id
+    })
   }
 
-  async ondata (proof, peer) {
-    // technically if the remote peer pushes a DATA someone else requested inflight can go to zero
-    if (peer.state.inflight > 0) peer.state.inflight--
-
-    // if we get a message from another fork, maybe "unfork".
-    if (peer.state.fork !== this.core.tree.fork) {
-      // TODO: user should opt-in to this behaivour
-      if (peer.state.fork > this.core.tree.fork) return this._onfork(proof, peer)
-      return
-    }
+  async ondata (data) {
+    const req = data.request > 0 ? this.replicator._inflight.get(data.request) : null
+    const reorg = data.fork > this.core.tree.fork
 
-    // ignore incoming messages during an unfork.
-    if (this.unforking) return
+    // no push atm, TODO: check if this satisfies another pending request
+    // allow reorg pushes tho as those are not written to storage so we'll take all the help we can get
+    if (req === null && reorg === false) return
 
-    if (!proof.block) return this._onupgrade(proof, peer)
+    if (req !== null) {
+      if (req.peer !== this) return
+      this.inflight--
+      this.replicator._inflight.remove(req.id)
+    }
 
-    const { index, value } = proof.block
-    const req = this.requests.get(index)
+    if (reorg === true) return this.replicator._onreorgdata(this, req, data)
 
-    // no push allowed, TODO: add flag to allow pushes
-    if (!req || req.peer !== peer || (value && !req.value) || (proof.upgrade && !this.upgrading)) return
+    this.dataProcessing++
 
     try {
-      await this.core.verify(proof, peer)
+      if (!matchingRequest(req, data) || !(await this.core.verify(data, this))) {
+        this.replicator._onnodata(this, req)
+        return
+      }
     } catch (err) {
-      this.requests.delete(index)
+      this.replicator._onnodata(this, req)
       throw err
+    } finally {
+      this.dataProcessing--
     }
 
-    // TODO: validate that we actually upgraded our length as well
-    if (proof.upgrade) {
-      this.upgrading.resolve()
-      this.upgrading = null
+    this.replicator._ondata(this, req, data)
 
-      if (this.pendingUpgrade) {
-        this.pendingUpgrade.resolve(true)
-        this.pendingUpgrade = null
-      }
+    if (this._shouldUpdateCanUpgrade() === true) {
+      this._updateCanUpgradeAndSync()
     }
+  }
 
-    await this._resolveRequest(req, index, value)
+  onnodata ({ request }) {
+    const req = request > 0 ? this.replicator._inflight.get(request) : null
 
-    this.update(peer)
-  }
+    if (req === null || req.peer !== this) return
 
-  async _resolveRequest (req, index, value) {
-    // if our request types match, clear inflight, otherwise we upgraded a hash req to a value req
-    const resolved = req.value === !!value
-    if (resolved) {
-      this.requests.delete(index)
-      req.resolve(value)
-    }
+    this.inflight--
+    this.replicator._inflight.remove(req.id)
+    this.replicator._onnodata(this, req)
+  }
 
-    if (this.seeks.length) await this._updateSeeks(req)
+  onwant () {
+    // TODO
+  }
 
-    // TODO: only do this for active ranges, ie ranges with inflight reqs...
-    for (let i = 0; i < this.ranges.length; i++) {
-      const r = this.ranges[i]
-      if (!r.contains(req)) continue
-      if (!r.update(this.core.bitfield)) continue
-      r.resolve(true)
-      i--
-    }
+  onunwant () {
+    // TODO
   }
 
-  async _updateSeeks (req) {
-    for (let i = 0; i < this.seeks.length; i++) {
-      await this.seeks[i].update()
+  onbitfield ({ start, bitfield }) {
+    // TODO: tweak this to be more generic
+
+    if (bitfield.length < 1024) {
+      const buf = b4a.from(bitfield.buffer, bitfield.byteOffset, bitfield.byteLength)
+      const bigger = b4a.concat([buf, b4a.alloc(4096 - buf.length)])
+      bitfield = new Uint32Array(bigger.buffer, bigger.byteOffset, 1024)
     }
 
-    for (let i = 0; i < this.seeks.length; i++) {
-      const seek = this.seeks[i]
+    this.remoteBitfield.pages.set(start / this.core.bitfield.pageSize, bitfield)
 
-      if (seek.finished) {
-        if (this.seeks.length > 1 && i < this.seeks.length - 1) {
-          this.seeks[i] = this.seeks[this.seeks.length - 1]
-          i--
-        }
-        this.seeks.pop()
-      }
-      if (req !== null && seek.request === req) seek.request = null
-    }
+    this._update()
   }
 
-  async resolveBlock (index, value) {
-    const req = this.requests.get(index)
-    if (req) await this._resolveRequest(req, index, value)
-  }
+  onrange ({ drop, start, length }) {
+    const has = drop === false
+
+    for (const end = start + length; start < end; start++) {
+      this.remoteBitfield.set(start, has)
+    }
 
-  upgrade () {
-    if (this.pendingUpgrade) return this.pendingUpgrade.createPromise()
-    this.pendingUpgrade = new Upgrade(this.core.tree.length + 1)
-    return this.pendingUpgrade.createPromise()
+    if (drop === false) this._update()
   }
 
-  range (range) {
-    if (range.ranges === null || this.ranges.index(range) > -1) return
-    this.ranges.push(range)
-    range.ranges = range
+  onreorghint () {
+    // TODO
   }
 
-  seek (seeker) {
-    const s = new Seek(seeker)
-    this.seeks.push(s)
-    return s.createPromise()
+  _update () {
+    // TODO: if this is in a batch or similar it would be better to defer it
+    // we could do that with nextTick/microtick mb? (combined with a property on the session to signal read buffer mb)
+    this.replicator.updatePeer(this)
   }
 
-  block (index) {
-    const e = this.requests.get(index)
+  _makeRequest (needsUpgrade) {
+    if (needsUpgrade === true && this.replicator._shouldUpgrade(this) === false) {
+      return null
+    }
 
-    if (e) {
-      if (!e.value) {
-        e.value = true
-        if (e.peer) this.send(e.peer, e)
-      }
+    if (needsUpgrade === false && this.replicator._autoUpgrade(this) === true) {
+      needsUpgrade = true
+    }
 
-      return e.createPromise()
+    return {
+      peer: this,
+      id: 0,
+      fork: this.remoteFork,
+      block: null,
+      hash: null,
+      seek: null,
+      upgrade: needsUpgrade === false
+        ? null
+        : { start: this.core.tree.length, length: this.remoteLength - this.core.tree.length }
     }
+  }
 
-    const r = new Request(index, 0, 0)
+  _requestUpgrade (u) {
+    const req = this._makeRequest(true)
+    if (req === null) return false
 
-    this.requests.set(index, r)
-    this.pending.push(r)
+    this._send(req)
 
-    return r.createPromise()
+    return true
   }
-}
 
-module.exports = class Replicator {
-  constructor (core, { onupdate }) {
-    this.core = core
-    this.peers = []
-    this.requests = new RequestPool(this, core)
-    this.updating = null
-    this.pendingPeers = new Set()
-    this.onupdate = onupdate
-  }
+  _requestSeek (s) {
+    const { length, fork } = this.core.tree
 
-  static createProtocol (noiseStream) {
-    return new Protocol(noiseStream, {
-      protocolVersion: 0,
-      userAgent: USER_AGENT
-    })
-  }
+    if (fork !== this.remoteFork) return false
 
-  joinProtocol (protocol, key, discoveryKey) {
-    if (protocol.isRegistered(discoveryKey)) return
-    const peer = protocol.registerPeer(key, discoveryKey, this, new RemoteState(this.core))
-    this.pendingPeers.add(peer)
-  }
+    if (s.seeker.start >= length) {
+      const req = this._makeRequest(true)
+
+      // We need an upgrade for the seek, if non can be provided, skip
+      if (req === null) return false
 
-  broadcastBlock (start) {
-    const msg = { start, length: 1 }
-    for (const peer of this.peers) peer.have(msg)
+      req.seek = { bytes: s.seeker.bytes }
 
-    // in case of writable peer waiting for the block itself
-    // we trigger the resolver to see if can't resolve it now
-    if (this.requests.isRequesting(start)) {
-      this._resolveBlock(start).then(noop, noop)
+      s.inflight.push(req)
+      this._send(req)
+
+      return true
     }
-  }
 
-  broadcastInfo () {
-    const msg = { length: this.core.tree.length, fork: this.core.tree.fork }
-    for (const peer of this.peers) peer.info(msg)
-    this.updateAll()
-  }
+    const len = s.seeker.end - s.seeker.start
+    const off = s.seeker.start + Math.floor(Math.random() * len)
 
-  requestUpgrade () {
-    const promise = this.requests.upgrade()
-    this.updateAll()
-    return promise
-  }
+    for (let i = 0; i < len; i++) {
+      let index = off + i
+      if (index > s.seeker.end) index -= len
 
-  requestSeek (seeker) {
-    if (typeof seeker === 'number') seeker = this.core.tree.seek(seeker)
-    const promise = this.requests.seek(seeker)
-    this.updateAll()
-    return promise
-  }
+      if (this.remoteBitfield.get(index) === false) continue
+      if (this.core.bitfield.get(index) === true) continue
 
-  requestBlock (index) {
-    const promise = this.requests.block(index)
-    this.updateAll()
-    return promise
-  }
+      // Check if this block is currently inflight - if so pick another
+      const b = this.replicator._blocks.get(index)
+      if (b !== null && b.inflight.length > 0) continue
 
-  static createRange (start, end, linear) {
-    return new Range(start, end, linear)
-  }
+      // Block is not inflight, but we only want the hash, check if that is inflight
+      const h = this.replicator._hashes.add(index)
+      if (h.inflight.length > 0) continue
 
-  addRange (range) {
-    if (!range.done) {
-      range._ranges = this.requests.ranges
-      this.requests.ranges.push(range)
-      if (range.update(this.core.bitfield)) range.resolve(true)
-    }
-    this.updateAll()
-  }
+      const req = this._makeRequest(false)
 
-  async _resolveBlock (index) {
-    const value = await this.core.blocks.get(index)
-    this.requests.resolveBlock(index, value)
-  }
+      req.hash = { index: 2 * index, nodes: 0 }
+      req.seek = { bytes: s.seeker.bytes }
 
-  updateAll () {
-    const peers = new RandomIterator(this.peers)
-    for (const peer of peers) {
-      if (paused(peer, this.core.tree.fork)) continue
-      if (this.requests.update(peer)) peers.requeue()
+      s.inflight.push(req)
+      h.inflight.push(req)
+      this._send(req)
+
+      return true
     }
-    // TODO: this is a silly way of doing it
-    if (this.pendingPeers.size === 0) this.requests.checkTimeouts(this.peers)
+
+    return false
   }
 
-  onunregister (peer, err) {
-    const idx = this.peers.indexOf(peer)
-    if (idx === -1) return
+  // mb turn this into a YES/NO/MAYBE enum, could simplify ifavail logic
+  _blockAvailable (b) { // TODO: fork also
+    return this.remoteBitfield.get(b.index)
+  }
 
-    this.pendingPeers.delete(peer)
-    this.peers.splice(idx, 1)
-    this.requests.clear(peer)
-    this.onupdate(false, peer)
+  _requestBlock (b) {
+    const { length, fork } = this.core.tree
 
-    this.updateAll()
-  }
+    if (this.remoteBitfield.get(b.index) === false || fork !== this.remoteFork) return false
 
-  oncore (_, peer) {
-    this.pendingPeers.delete(peer)
-    this.peers.push(peer)
-    this.onupdate(true, peer)
+    const req = this._makeRequest(b.index >= length)
+    if (req === null) return false
 
-    peer.info({ length: this.core.tree.length, fork: this.core.tree.fork })
+    req.block = { index: b.index, nodes: 0 }
 
-    // YOLO send over all the pages for now
-    const p = pages(this.core)
-    for (let index = 0; index < p.length; index++) {
-      peer.bitfield({ start: index, bitfield: p[index] })
-    }
-  }
+    b.inflight.push(req)
+    this._send(req)
 
-  onunknowncore (_, peer) {
-    this.pendingPeers.delete(peer)
-    this.updateAll()
-    // This is a no-op because there isn't any state to dealloc currently
+    return true
   }
 
-  oninfo ({ length, fork }, peer) {
-    const len = peer.state.length
-    const forked = peer.state.fork !== fork
+  _requestRange (r) {
+    const { length, fork } = this.core.tree
 
-    peer.state.length = length
-    peer.state.fork = fork
+    const end = Math.min(r.end === -1 ? this.remoteLength : r.end, this.remoteLength)
+    if (end < r.start || fork !== this.remoteFork) return false
 
-    if (forked) {
-      for (let i = peer.state.length; i < len; i++) {
-        peer.state.bitfield.set(i, false)
-      }
+    const len = end - r.start
+    const off = r.start + (r.linear ? 0 : Math.floor(Math.random() * len))
 
-      if (fork > this.core.tree.fork && length > 0) {
-        this.requests.clear(peer) // clear all pending requests from this peer as they forked, this can be removed if we add remote cancel
-        peer.request({ fork, upgrade: { start: 0, length } })
-      }
-    }
+    // TODO: we should weight this to request blocks < .length first
+    // as they are "cheaper" and will trigger an auto upgrade if possible
+    // If no blocks < .length is avaible then try the "needs upgrade" range
 
-    if (!peer.state.receivedInfo) {
-      peer.state.receivedInfo = true
-    }
+    for (let i = 0; i < len; i++) {
+      let index = off + i
+      if (index >= end) index -= len
 
-    // TODO: do we need to update ALL peers here? prob not
-    this.updateAll()
-  }
+      if (r.blocks !== null) index = r.blocks[index]
 
-  onbitfield ({ start, bitfield }, peer) {
-    if (bitfield.length < 1024) {
-      const buf = Buffer.from(bitfield.buffer, bitfield.byteOffset, bitfield.byteLength)
-      const bigger = Buffer.concat([buf, Buffer.alloc(4096 - buf.length)])
-      bitfield = new Uint32Array(bigger.buffer, bigger.byteOffset, 1024)
-    }
-    peer.state.bitfield.pages.set(start, bitfield)
+      if (this.remoteBitfield.get(index) === false) continue
+      if (this.core.bitfield.get(index) === true) continue
 
-    // TODO: do we need to update ALL peers here? prob not
-    this.updateAll()
-  }
+      const b = this.replicator._blocks.add(index)
+      if (b.inflight.length > 0) continue
 
-  onhave ({ start, length }, peer) {
-    const end = start + length
-    for (; start < end; start++) {
-      peer.state.bitfield.set(start, true)
-    }
+      const req = this._makeRequest(index >= length)
 
-    // TODO: do we need to update ALL peers here? prob not
-    this.updateAll()
-  }
+      // If the request cannot be satisfied, dealloc the block request if no one is subscribed to it
+      if (req === null) {
+        b.gc()
+        return false
+      }
 
-  async ondata (proof, peer) {
-    try {
-      await this.requests.ondata(proof, peer)
-    } catch (err) {
-      // TODO: the request pool should have this cap, so we can just bubble it up
-      this.updateAll()
-      throw err
-    }
-  }
+      req.block = { index, nodes: 0 }
 
-  async onrequest (req, peer) {
-    const fork = req.fork || peer.state.fork
-    if (fork !== this.core.tree.fork) return
+      b.inflight.push(req)
+      this._send(req)
 
-    const proof = await this.core.tree.proof(req)
+      // Don't think this will ever happen, as the pending queue is drained before the range queue
+      // but doesn't hurt to check this explicitly here also.
+      if (b.queued) b.queued = false
 
-    if (req.block && req.block.value) {
-      proof.block.value = await this.core.blocks.get(req.block.index)
+      return true
     }
 
-    peer.data(proof)
+    return false
   }
-}
 
-function paused (peer, fork) {
-  return peer.state.fork !== fork
-}
+  _requestForkProof (f) {
+    const req = this._makeRequest(false)
 
-function pages (core) {
-  const res = []
+    req.upgrade = { start: 0, length: this.remoteLength }
 
-  for (let i = 0; i < core.tree.length; i += core.bitfield.pageSize) {
-    const p = core.bitfield.page(i / core.bitfield.pageSize)
-    res.push(p)
+    f.inflight.push(req)
+    this._send(req)
   }
 
-  return res
-}
+  _requestForkRange (f) {
+    if (f.fork !== this.remoteFork || f.batch.want === null) return false
 
-function noop () {}
+    const end = Math.min(f.batch.want.end, this.remoteLength)
+    if (end < f.batch.want.start) return false
 
-function log2 (n) {
-  let res = 1
+    const len = end - f.batch.want.start
+    const off = f.batch.want.start + Math.floor(Math.random() * len)
 
-  while (n > 2) {
-    n /= 2
-    res++
-  }
+    for (let i = 0; i < len; i++) {
+      let index = off + i
+      if (index >= end) index -= len
 
-  return res
+      if (this.remoteBitfield.get(index) === false) continue
+
+      const req = this._makeRequest(false)
+
+      req.hash = { index: 2 * index, nodes: f.batch.want.nodes }
+
+      f.inflight.push(req)
+      this._send(req)
+
+      return true
+    }
+
+    return false
+  }
+
+  async _send (req) {
+    const fork = this.core.tree.fork
+
+    this.inflight++
+    this.replicator._inflight.add(req)
+
+    if (req.upgrade !== null && req.fork === fork) {
+      const u = this.replicator._addUpgrade()
+      u.inflight.push(req)
+    }
+
+    try {
+      if (req.block !== null && req.fork === fork) {
+        req.block.nodes = await this.core.tree.missingNodes(2 * req.block.index)
+      }
+      if (req.hash !== null && req.fork === fork) {
+        req.hash.nodes = await this.core.tree.missingNodes(req.hash.index)
+      }
+    } catch (err) {
+      this.stream.destroy(err)
+      return
+    }
+
+    this.wireRequest.send(req)
+  }
+}
+
+module.exports = class Replicator {
+  constructor (core, key, { eagerUpgrade = true, allowFork = true, onpeerupdate = noop, onupload = noop } = {}) {
+    this.key = key
+    this.discoveryKey = core.crypto.discoveryKey(key)
+    this.core = core
+    this.eagerUpgrade = eagerUpgrade
+    this.allowFork = allowFork
+    this.onpeerupdate = onpeerupdate
+    this.onupload = onupload
+    this.peers = []
+    this.findingPeers = 0 // updateable from the outside
+
+    this._inflight = new InflightTracker()
+    this._blocks = new BlockTracker()
+    this._hashes = new BlockTracker()
+
+    this._queued = []
+
+    this._seeks = []
+    this._upgrade = null
+    this._reorgs = []
+    this._ranges = []
+
+    this._hadPeers = false
+    this._ifAvailable = 0
+    this._updatesPending = 0
+    this._applyingReorg = false
+  }
+
+  cork () {
+    for (const peer of this.peers) peer.protomux.cork()
+  }
+
+  uncork () {
+    for (const peer of this.peers) peer.protomux.uncork()
+  }
+
+  // Called externally when a range of new blocks has been processed/removed
+  onhave (start, length, drop = false) {
+    for (const peer of this.peers) peer.broadcastRange(start, length, drop)
+  }
+
+  // Called externally when a truncation upgrade has been processed
+  ontruncate (newLength) {
+    const notify = []
+
+    for (const blk of this._blocks) {
+      if (blk.index < newLength) continue
+      notify.push(blk)
+    }
+
+    for (const blk of notify) {
+      for (const r of blk.refs) {
+        if (r.snapshot === false) continue
+        blk.detach(r, SNAPSHOT_NOT_AVAILABLE())
+      }
+    }
+  }
+
+  // Called externally when a upgrade has been processed
+  onupgrade () {
+    for (const peer of this.peers) peer.signalUpgrade()
+    if (this._blocks.isEmpty() === false) this._resolveBlocksLocally()
+    if (this._upgrade !== null) this._resolveUpgradeRequest(null)
+    if (this._ranges.length !== 0 || this._seeks.length !== 0) this._updateNonPrimary()
+  }
+
+  addUpgrade (session) {
+    if (this._upgrade !== null) {
+      const ref = this._upgrade.attach(session)
+      this._checkUpgradeIfAvailable()
+      return ref
+    }
+
+    const ref = this._addUpgrade().attach(session)
+
+    for (let i = this._reorgs.length - 1; i >= 0 && this._applyingReorg === false; i--) {
+      const f = this._reorgs[i]
+      if (f.batch !== null && f.batch.finished) {
+        this._applyReorg(f)
+        break
+      }
+    }
+
+    this.updateAll()
+
+    return ref
+  }
+
+  addBlock (session, index) {
+    const b = this._blocks.add(index)
+    const ref = b.attach(session)
+
+    this._queueBlock(b)
+    this.updateAll()
+
+    return ref
+  }
+
+  addSeek (session, seeker) {
+    const s = new SeekRequest(this._seeks, seeker)
+    const ref = s.attach(session)
+
+    this._seeks.push(s)
+    this.updateAll()
+
+    return ref
+  }
+
+  addRange (session, { start = 0, end = -1, length = toLength(start, end), blocks = null, linear = false, ifAvailable = false } = {}) {
+    if (blocks !== null) { // if using blocks, start, end just acts as frames around the blocks array
+      start = 0
+      end = length = blocks.length
+    }
+
+    const r = new RangeRequest(
+      this._ranges,
+      start,
+      length === -1 ? -1 : start + length,
+      linear,
+      ifAvailable,
+      blocks
+    )
+
+    const ref = r.attach(session)
+
+    this._ranges.push(r)
+
+    // Trigger this to see if this is already resolved...
+    // Also auto compresses the range based on local bitfield
+    this._updateNonPrimary()
+
+    return ref
+  }
+
+  cancel (ref) {
+    ref.context.detach(ref, null)
+  }
+
+  clearRequests (session) {
+    while (session.length > 0) {
+      const ref = session[session.length - 1]
+      ref.context.detach(ref, null)
+    }
+
+    this.updateAll()
+  }
+
+  _addUpgradeMaybe () {
+    return this.eagerUpgrade === true ? this._addUpgrade() : this._upgrade
+  }
+
+  // TODO: this function is OVER called atm, at each updatePeer/updateAll
+  // instead its more efficient to only call it when the conditions in here change - ie on sync/add/remove peer
+  // Do this when we have more tests.
+  _checkUpgradeIfAvailable () {
+    if (this._ifAvailable > 0 || this._upgrade === null || this._upgrade.refs.length === 0) return
+    if (this._hadPeers === false && this.findingPeers > 0) return
+
+    // check if a peer can upgrade us
+
+    for (let i = 0; i < this.peers.length; i++) {
+      const peer = this.peers[i]
+
+      if (peer.remoteSynced === false) return
+
+      if (this.core.tree.length === 0 && peer.remoteLength > 0) return
+
+      if (peer.remoteLength <= this._upgrade.length || peer.remoteFork !== this._upgrade.fork) continue
+
+      if (peer.syncsProcessing > 0) return
+
+      if (peer.lengthAcked !== this.core.tree.length && peer.remoteFork === this.core.tree.fork) return
+      if (peer.remoteCanUpgrade === true) return
+    }
+
+    // check if reorgs in progress...
+
+    if (this._applyingReorg === true) return
+
+    // TODO: we prob should NOT wait for inflight reorgs here, seems better to just resolve the upgrade
+    // and then apply the reorg on the next call in case it's slow - needs some testing in practice
+
+    for (let i = 0; i < this._reorgs.length; i++) {
+      const r = this._reorgs[i]
+      if (r.inflight.length > 0) return
+    }
+
+    // nothing to do, indicate no update avail
+
+    const u = this._upgrade
+    this._upgrade = null
+    u.resolve(false)
+  }
+
+  _addUpgrade () {
+    if (this._upgrade !== null) return this._upgrade
+
+    // TODO: needs a reorg: true/false flag to indicate if the user requested a reorg
+    this._upgrade = new UpgradeRequest(this, this.core.tree.fork, this.core.tree.length)
+
+    return this._upgrade
+  }
+
+  _addReorg (fork, peer) {
+    if (this.allowFork === false) return null
+
+    // TODO: eager gc old reorgs from the same peer
+    // not super important because they'll get gc'ed when the request finishes
+    // but just spam the remote can do ...
+
+    for (const f of this._reorgs) {
+      if (f.fork > fork && f.batch !== null) return null
+      if (f.fork === fork) return f
+    }
+
+    const f = {
+      fork,
+      inflight: [],
+      batch: null
+    }
+
+    this._reorgs.push(f)
+
+    // maintain sorted by fork
+    let i = this._reorgs.length - 1
+    while (i > 0 && this._reorgs[i - 1].fork > fork) {
+      this._reorgs[i] = this._reorgs[i - 1]
+      this._reorgs[--i] = f
+    }
+
+    return f
+  }
+
+  _shouldUpgrade (peer) {
+    if (this._upgrade !== null && this._upgrade.inflight.length > 0) return false
+    return peer.remoteCanUpgrade === true &&
+      peer.remoteLength > this.core.tree.length &&
+      peer.lengthAcked === this.core.tree.length
+  }
+
+  _autoUpgrade (peer) {
+    return this._upgrade !== null && peer.remoteFork === this.core.tree.fork && this._shouldUpgrade(peer)
+  }
+
+  _addPeer (peer) {
+    this._hadPeers = true
+    this.peers.push(peer)
+    this.updatePeer(peer)
+    this.onpeerupdate(true, peer)
+  }
+
+  _removePeer (peer) {
+    this.peers.splice(this.peers.indexOf(peer), 1)
+
+    for (const req of this._inflight) {
+      if (req.peer !== peer) continue
+      this._inflight.remove(req.id)
+      this._clearRequest(peer, req)
+    }
+
+    this.onpeerupdate(false, peer)
+    this.updateAll()
+  }
+
+  _queueBlock (b) {
+    if (b.inflight.length > 0 || b.queued === true) return
+    b.queued = true
+    this._queued.push(b)
+  }
+
+  // Runs in the background - not allowed to throw
+  async _resolveBlocksLocally () {
+    // TODO: check if fork compat etc. Requires that we pass down truncation info
+
+    let clear = null
+
+    for (const b of this._blocks) {
+      if (this.core.bitfield.get(b.index) === false) continue
+
+      try {
+        b.resolve(await this.core.blocks.get(b.index))
+      } catch (err) {
+        b.reject(err)
+      }
+
+      if (clear === null) clear = []
+      clear.push(b)
+    }
+
+    if (clear === null) return
+
+    // Currently the block tracker does not support deletes during iteration, so we make
+    // sure to clear them afterwards.
+    for (const b of clear) {
+      this._blocks.remove(b.index)
+    }
+  }
+
+  _resolveBlockRequest (tracker, index, value, req) {
+    const b = tracker.remove(index)
+    if (b === null) return false
+
+    removeInflight(b.inflight, req)
+    b.queued = false
+
+    b.resolve(value)
+
+    return true
+  }
+
+  _resolveUpgradeRequest (req) {
+    if (req !== null) removeInflight(this._upgrade.inflight, req)
+
+    if (this.core.tree.length === this._upgrade.length && this.core.tree.fork === this._upgrade.fork) return false
+
+    const u = this._upgrade
+    this._upgrade = null
+    u.resolve(true)
+
+    return true
+  }
+
+  _resolveRangeRequest (req, index) {
+    const head = this._ranges.pop()
+
+    if (index < this._ranges.length) this._ranges[index] = head
+
+    req.resolve(true)
+  }
+
+  _clearInflightBlock (tracker, req) {
+    const isBlock = tracker === this._blocks
+    const index = isBlock === true ? req.block.index : req.hash.index / 2
+    const b = tracker.get(index)
+
+    if (b === null || removeInflight(b.inflight, req) === false) return
+
+    if (b.refs.length > 0 && isBlock === true) {
+      this._queueBlock(b)
+      return
+    }
+
+    b.gc()
+  }
+
+  _clearInflightUpgrade (req) {
+    if (removeInflight(this._upgrade.inflight, req) === false) return
+    this._upgrade.gc()
+  }
+
+  _clearInflightSeeks (req) {
+    for (const s of this._seeks) {
+      if (removeInflight(s.inflight, req) === false) continue
+      s.gc()
+    }
+  }
+
+  _clearInflightReorgs (req) {
+    for (const r of this._reorgs) {
+      removeInflight(r.inflight, req)
+    }
+  }
+
+  _clearOldReorgs (fork) {
+    for (let i = 0; i < this._reorgs.length; i++) {
+      const f = this._reorgs[i]
+      if (f.fork >= fork) continue
+      if (i === this._reorgs.length - 1) this._reorgs.pop()
+      else this._reorgs[i] = this._reorgs.pop()
+      i--
+    }
+  }
+
+  // "slow" updates here - async but not allowed to ever throw
+  async _updateNonPrimary () {
+    // Check if running, if so skip it and the running one will issue another update for us (debounce)
+    while (++this._updatesPending === 1) {
+      for (let i = 0; i < this._ranges.length; i++) {
+        const r = this._ranges[i]
+
+        while (r.start < r.end && this.core.bitfield.get(mapIndex(r.blocks, r.start)) === true) r.start++
+        while (r.start < r.end && this.core.bitfield.get(mapIndex(r.blocks, r.end - 1)) === true) r.end--
+
+        if (r.end !== -1 && r.start >= r.end) {
+          this._resolveRangeRequest(r, i--)
+        }
+      }
+
+      for (let i = 0; i < this._seeks.length; i++) {
+        const s = this._seeks[i]
+
+        let err = null
+        let res = null
+
+        try {
+          res = await s.seeker.update()
+        } catch (error) {
+          err = error
+        }
+
+        if (!res && !err) continue
+
+        if (i < this._seeks.length - 1) this._seeks[i] = this._seeks.pop()
+        else this._seeks.pop()
+
+        i--
+
+        if (err) s.reject(err)
+        else s.resolve(res)
+      }
+
+      if (this._inflight.idle) this.updateAll()
+
+      // No additional updates scheduled - return
+      if (--this._updatesPending === 0) return
+      // Debounce the additional updates - continue
+      this._updatesPending = 0
+    }
+  }
+
+  _maybeResolveIfAvailableRanges () {
+    if (this._ifAvailable > 0 || !this._inflight.idle || !this._ranges.length) return
+
+    for (let i = 0; i < this.peers.length; i++) {
+      if (this.peers[i].dataProcessing > 0) return
+    }
+
+    for (let i = 0; i < this._ranges.length; i++) {
+      const r = this._ranges[i]
+
+      if (r.ifAvailable) {
+        this._resolveRangeRequest(r, i--)
+      }
+    }
+  }
+
+  _clearRequest (peer, req) {
+    if (req.block !== null) {
+      this._clearInflightBlock(this._blocks, req)
+    }
+
+    if (req.hash !== null) {
+      this._clearInflightBlock(this._hashes, req)
+    }
+
+    if (req.upgrade !== null && this._upgrade !== null) {
+      this._clearInflightUpgrade(req)
+    }
+
+    if (this._seeks.length > 0) {
+      this._clearInflightSeeks(req)
+    }
+
+    if (this._reorgs.length > 0) {
+      this._clearInflightReorgs(req)
+    }
+  }
+
+  _onnodata (peer, req) {
+    this._clearRequest(peer, req)
+    this.updateAll()
+  }
+
+  _ondata (peer, req, data) {
+    if (data.block !== null) {
+      this._resolveBlockRequest(this._blocks, data.block.index, data.block.value, req)
+    }
+
+    if (data.hash !== null && (data.hash.index & 1) === 0) {
+      this._resolveBlockRequest(this._hashes, data.hash.index / 2, null, req)
+    }
+
+    if (this._upgrade !== null) {
+      this._resolveUpgradeRequest(req)
+    }
+
+    if (this._seeks.length > 0) {
+      this._clearInflightSeeks(req)
+    }
+
+    if (this._reorgs.length > 0) {
+      this._clearInflightReorgs(req)
+    }
+
+    if (this._seeks.length > 0 || this._ranges.length > 0) this._updateNonPrimary()
+    else this.updatePeer(peer)
+  }
+
+  async _onreorgdata (peer, req, data) {
+    const f = this._addReorg(data.fork, peer)
+
+    if (f === null) {
+      this.updateAll()
+      return
+    }
+
+    removeInflight(f.inflight, req)
+
+    if (f.batch) {
+      await f.batch.update(data)
+    } else if (data.upgrade) {
+      f.batch = await this.core.tree.reorg(data)
+
+      // Remove "older" reorgs in progress as we just verified this one.
+      this._clearOldReorgs(f.fork)
+    }
+
+    if (f.batch && f.batch.finished) {
+      if (this._addUpgradeMaybe() !== null) {
+        await this._applyReorg(f)
+      }
+    }
+
+    this.updateAll()
+  }
+
+  // Never throws, allowed to run in the background
+  async _applyReorg (f) {
+    // TODO: more optimal here to check if potentially a better reorg
+    // is available, ie higher fork, and request that one first.
+    // This will request that one after this finishes, which is fine, but we
+    // should investigate the complexity in going the other way
+
+    const u = this._upgrade
+
+    this._applyingReorg = true
+    this._reorgs = [] // clear all as the nodes are against the old tree - easier
+
+    try {
+      await this.core.reorg(f.batch, null) // TODO: null should be the first/last peer?
+    } catch (err) {
+      this._upgrade = null
+      u.reject(err)
+    }
+
+    this._applyingReorg = false
+
+    if (this._upgrade !== null) {
+      this._resolveUpgradeRequest(null)
+    }
+
+    for (const peer of this.peers) this._updateFork(peer)
+
+    // TODO: all the remaining is a tmp workaround until we have a flag/way for ANY_FORK
+    for (const r of this._ranges) {
+      r.start = r.userStart
+      r.end = r.userEnd
+    }
+
+    this.updateAll()
+  }
+
+  _maybeUpdate () {
+    return this._upgrade !== null && this._upgrade.inflight.length === 0
+  }
+
+  _updateFork (peer) {
+    if (this._applyingReorg === true || this.allowFork === false || peer.remoteFork <= this.core.tree.fork) {
+      return false
+    }
+
+    const f = this._addReorg(peer.remoteFork, peer)
+
+    // TODO: one per peer is better
+    if (f !== null && f.batch === null && f.inflight.length === 0) {
+      return peer._requestForkProof(f)
+    }
+
+    return false
+  }
+
+  _updatePeer (peer) {
+    if (peer.inflight >= peer.maxInflight) {
+      return false
+    }
+
+    for (const s of this._seeks) {
+      if (s.inflight.length > 0) continue // TODO: one per peer is better
+      if (peer._requestSeek(s) === true) {
+        return true
+      }
+    }
+
+    // Implied that any block in the queue should be requested, no matter how many inflights
+    const blks = new RandomIterator(this._queued)
+
+    for (const b of blks) {
+      if (b.queued === false || peer._requestBlock(b) === true) {
+        b.queued = false
+        blks.dequeue()
+        return true
+      }
+    }
+
+    return false
+  }
+
+  _updatePeerNonPrimary (peer) {
+    if (peer.inflight >= peer.maxInflight) {
+      return false
+    }
+
+    const ranges = new RandomIterator(this._ranges)
+
+    for (const r of ranges) {
+      if (peer._requestRange(r) === true) {
+        return true
+      }
+    }
+
+    // Iterate from newest fork to oldest fork...
+    for (let i = this._reorgs.length - 1; i >= 0; i--) {
+      const f = this._reorgs[i]
+      if (f.batch !== null && f.inflight.length === 0 && peer._requestForkRange(f) === true) {
+        return true
+      }
+    }
+
+    if (this._maybeUpdate() === true && peer._requestUpgrade(this._upgrade) === true) {
+      return true
+    }
+
+    return false
+  }
+
+  updatePeer (peer) {
+    // Quick shortcut to wait for flushing reorgs - not needed but less waisted requests
+    if (this._applyingReorg === true) return
+
+    while (this._updatePeer(peer) === true);
+    while (this._updatePeerNonPrimary(peer) === true);
+
+    this._checkUpgradeIfAvailable()
+    this._maybeResolveIfAvailableRanges()
+  }
+
+  updateAll () {
+    // Quick shortcut to wait for flushing reorgs - not needed but less waisted requests
+    if (this._applyingReorg === true) return
+
+    const peers = new RandomIterator(this.peers)
+
+    for (const peer of peers) {
+      if (this._updatePeer(peer) === true) {
+        peers.requeue()
+      }
+    }
+
+    // Check if we can skip the non primary check fully
+    if (this._maybeUpdate() === false && this._ranges.length === 0 && this._reorgs.length === 0) {
+      this._checkUpgradeIfAvailable()
+      return
+    }
+
+    for (const peer of peers.restart()) {
+      if (this._updatePeerNonPrimary(peer) === true) {
+        peers.requeue()
+      }
+    }
+
+    this._checkUpgradeIfAvailable()
+    this._maybeResolveIfAvailableRanges()
+  }
+
+  attachTo (protomux) {
+    const makePeer = this._makePeer.bind(this, protomux)
+
+    protomux.pair({ protocol: 'hypercore/alpha', id: this.discoveryKey }, makePeer)
+
+    this._ifAvailable++
+    protomux.stream.opened.then((opened) => {
+      this._ifAvailable--
+      if (opened) makePeer()
+      this._checkUpgradeIfAvailable()
+    })
+  }
+
+  _makePeer (protomux) {
+    if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) return false
+
+    const channel = protomux.createChannel({
+      userData: null,
+      protocol: 'hypercore/alpha',
+      id: this.discoveryKey,
+      handshake: m.wire.handshake,
+      messages: [
+        { encoding: m.wire.sync, onmessage: onwiresync },
+        { encoding: m.wire.request, onmessage: onwirerequest },
+        null, // oncancel
+        { encoding: m.wire.data, onmessage: onwiredata },
+        { encoding: m.wire.noData, onmessage: onwirenodata },
+        { encoding: m.wire.want, onmessage: onwirewant },
+        { encoding: m.wire.unwant, onmessage: onwireunwant },
+        { encoding: m.wire.bitfield, onmessage: onwirebitfield },
+        { encoding: m.wire.range, onmessage: onwirerange },
+        { encoding: m.wire.extension, onmessage: onwireextension }
+      ],
+      onopen: onwireopen,
+      onclose: onwireclose
+    })
+
+    if (channel === null) return false
+
+    const peer = new Peer(this, protomux, channel)
+    const stream = protomux.stream
+
+    peer.channel.open({
+      capability: caps.replicate(stream.isInitiator, this.key, stream.handshakeHash)
+    })
+
+    return true
+  }
+}
+
+function pages (core) {
+  const res = []
+
+  for (let i = 0; i < core.tree.length; i += core.bitfield.pageSize) {
+    const p = core.bitfield.page(i / core.bitfield.pageSize)
+    res.push(p)
+  }
+
+  return res
+}
+
+function matchingRequest (req, data) {
+  if (data.block !== null && (req.block === null || req.block.index !== data.block.index)) return false
+  if (data.hash !== null && (req.hash === null || req.hash.index !== data.hash.index)) return false
+  if (data.seek !== null && (req.seek === null || req.seek.bytes !== data.seek.bytes)) return false
+  if (data.upgrade !== null && req.upgrade === null) return false
+  return req.fork === data.fork
+}
+
+function removeInflight (inf, req) {
+  const i = inf.indexOf(req)
+  if (i === -1) return false
+  if (i < inf.length - 1) inf[i] = inf.pop()
+  else inf.pop()
+  return true
+}
+
+function mapIndex (blocks, index) {
+  return blocks === null ? index : blocks[index]
+}
+
+function noop () {}
+
+function toLength (start, end) {
+  return end === -1 ? -1 : (end < start ? 0 : end - start)
+}
+
+function onwireopen (m, c) {
+  return c.userData.onopen(m)
+}
+
+function onwireclose (isRemote, c) {
+  return c.userData.onclose(isRemote)
+}
+
+function onwiresync (m, c) {
+  return c.userData.onsync(m)
+}
+
+function onwirerequest (m, c) {
+  return c.userData.onrequest(m)
+}
+
+function onwiredata (m, c) {
+  return c.userData.ondata(m)
+}
+
+function onwirenodata (m, c) {
+  return c.userData.onnodata(m)
+}
+
+function onwirewant (m, c) {
+  return c.userData.onwant(m)
+}
+
+function onwireunwant (m, c) {
+  return c.userData.onunwant(m)
+}
+
+function onwirebitfield (m, c) {
+  return c.userData.onbitfield(m)
+}
+
+function onwirerange (m, c) {
+  return c.userData.onrange(m)
+}
+
+function onwireextension (m, c) {
+  return c.userData.onextension(m)
 }