hypercore 10.38.2 → 11.0.0

package/lib/replicator.js

@@ -51,6 +51,7 @@ const PRIORITY = {
 class Attachable {
   constructor () {
     this.resolved = false
+    this.processing = false
     this.refs = []
   }
 
@@ -101,7 +102,12 @@ class Attachable {
   }
 
   gc () {
-    if (this.refs.length === 0) this._unref()
+    if (this.refs.length === 0 && !this.processing) this._unref()
+  }
+
+  processed () {
+    this.processing = false
+    this.gc()
   }
 
   _cancel (r, err) {
@@ -333,8 +339,28 @@ class RoundtripQueue {
   }
 }
 
+class ProofRequest {
+  constructor (msg, proof, block, manifest) {
+    this.msg = msg
+    this.proof = proof
+    this.block = block
+    this.manifest = manifest
+  }
+
+  async fulfill () {
+    if (this.proof === null) return null
+
+    const proof = await this.proof.settle()
+
+    if (this.manifest) proof.manifest = this.manifest
+    if (this.block) proof.block.value = await this.block
+
+    return proof
+  }
+}
+
 class Peer {
-  constructor (replicator, protomux, channel, useSession, inflightRange) {
+  constructor (replicator, protomux, channel, inflightRange) {
     this.core = replicator.core
     this.replicator = replicator
     this.stream = protomux.stream
@@ -346,8 +372,6 @@ class Peer {
     this.paused = false
     this.removed = false
 
-    this.useSession = useSession
-
     this.channel = channel
     this.channel.userData = this
 
@@ -418,6 +442,7 @@ class Peer {
     this.lastExtensionRecv = ''
 
     replicator._ifAvailable++
+    replicator._active++
   }
 
   get remoteContiguousLength () {
@@ -490,16 +515,16 @@ class Peer {
       return
     }
 
-    if (this.core.tree.fork !== this.remoteFork) {
+    if (this.core.state.fork !== this.remoteFork) {
       this.canUpgrade = false
     }
 
     this.needsSync = false
 
     this.wireSync.send({
-      fork: this.core.tree.fork,
-      length: this.core.tree.length,
-      remoteLength: this.core.tree.fork === this.remoteFork ? this.remoteLength : 0,
+      fork: this.core.state.fork,
+      length: this.core.state.length,
+      remoteLength: this.core.state.fork === this.remoteFork ? this.remoteLength : 0,
       canUpgrade: this.canUpgrade,
       uploading: true,
       downloading: this.replicator.isDownloading(),
@@ -509,7 +534,7 @@ class Peer {
   }
 
   onopen ({ seeks, capability }) {
-    const expected = caps.replicate(this.stream.isInitiator === false, this.replicator.key, this.stream.handshakeHash)
+    const expected = caps.replicate(this.stream.isInitiator === false, this.core.key, this.stream.handshakeHash)
 
     if (b4a.equals(capability, expected) !== true) { // TODO: change this to a rejection instead, less leakage
       throw INVALID_CAPABILITY('Remote sent an invalid replication capability')
@@ -523,11 +548,11 @@ class Peer {
 
     this.sendSync()
 
-    const contig = Math.min(this.core.tree.length, this.core.header.hints.contiguousLength)
+    const contig = Math.min(this.core.state.length, this.core.header.hints.contiguousLength)
     if (contig > 0) {
       this.broadcastRange(0, contig, false)
 
-      if (contig === this.core.tree.length) {
+      if (contig === this.core.state.length) {
         this.broadcastedNonSparse = true
       }
     }
@@ -536,6 +561,8 @@ class Peer {
     this.replicator._addPeer(this)
 
     this.protomux.uncork()
+
+    this.core.checkIfIdle()
   }
 
   onclose (isRemote) {
@@ -546,10 +573,6 @@ class Peer {
        this.remoteUploading === true && this.replicator.downloading === true
 
     if (this.remoteOpened === false) {
-      if (this.useSession) {
-        this.replicator._peerSessions--
-        this.replicator._closeSessionMaybe()
-      }
       this.replicator._ifAvailable--
       this.replicator.updateAll()
       return
@@ -567,12 +590,7 @@ class Peer {
     this.replicator._removePeer(this)
 
     if (reopen) {
-      this.replicator._makePeer(this.protomux, this.useSession)
-    }
-
-    if (this.useSession) {
-      this.replicator._peerSessions--
-      this.replicator._closeSessionMaybe()
+      this.replicator._makePeer(this.protomux)
     }
   }
 
@@ -588,7 +606,7 @@ class Peer {
 
   async onsync ({ fork, length, remoteLength, canUpgrade, uploading, downloading, hasManifest }) {
     const lengthChanged = length !== this.remoteLength
-    const sameFork = fork === this.core.tree.fork
+    const sameFork = fork === this.core.state.fork
 
     this.remoteSynced = true
     this.remoteFork = fork
@@ -605,7 +623,7 @@ class Peer {
 
     this.replicator._updateFork(this)
 
-    if (this.remoteLength > this.core.tree.length && this.lengthAcked === this.core.tree.length) {
+    if (this.remoteLength > this.core.state.length && this.lengthAcked === this.core.state.length) {
       if (this.replicator._addUpgradeMaybe() !== null) this._update()
     }
 
@@ -613,13 +631,13 @@ class Peer {
       ? this.canUpgrade && sameFork
       : await this._canUpgrade(length, fork)
 
-    if (length === this.remoteLength && fork === this.core.tree.fork) {
+    if (length === this.remoteLength && fork === this.core.state.fork) {
       this.canUpgrade = upgrade
     }
 
     if (--this.syncsProcessing !== 0) return // ie not latest
 
-    if (this.needsSync === true || (this.core.tree.fork === this.remoteFork && this.core.tree.length > this.remoteLength)) {
+    if (this.needsSync === true || (this.core.state.fork === this.remoteFork && this.core.state.length > this.remoteLength)) {
       this.signalUpgrade()
     }
 
@@ -627,18 +645,18 @@ class Peer {
   }
 
   _shouldUpdateCanUpgrade () {
-    return this.core.tree.fork === this.remoteFork &&
-      this.core.tree.length > this.remoteLength &&
+    return this.core.state.fork === this.remoteFork &&
+      this.core.state.length > this.remoteLength &&
       this.canUpgrade === false &&
       this.syncsProcessing === 0
   }
 
   async _updateCanUpgradeAndSync () {
-    const { length, fork } = this.core.tree
+    const { length, fork } = this.core.state
 
     const canUpgrade = await this._canUpgrade(this.remoteLength, this.remoteFork)
 
-    if (this.syncsProcessing > 0 || length !== this.core.tree.length || fork !== this.core.tree.fork) {
+    if (this.syncsProcessing > 0 || length !== this.core.state.length || fork !== this.core.state.fork) {
       return
     }
     if (canUpgrade === this.canUpgrade) {
@@ -651,16 +669,16 @@ class Peer {
 
   // Safe to call in the background - never fails
   async _canUpgrade (remoteLength, remoteFork) {
-    if (remoteFork !== this.core.tree.fork) return false
+    if (remoteFork !== this.core.state.fork) return false
 
     if (remoteLength === 0) return true
-    if (remoteLength >= this.core.tree.length) return false
+    if (remoteLength >= this.core.state.length) return false
 
     try {
       // Rely on caching to make sure this is cheap...
       const canUpgrade = await this.core.tree.upgradeable(remoteLength)
 
-      if (remoteFork !== this.core.tree.fork) return false
+      if (remoteFork !== this.core.state.fork) return false
 
       return canUpgrade
     } catch {
@@ -668,24 +686,24 @@ class Peer {
     }
   }
 
-  async _getProof (msg) {
-    const proof = await this.core.tree.proof(msg)
+  async _getProof (batch, msg) {
+    let block = null
 
-    if (proof.block) {
+    if (msg.block) {
       const index = msg.block.index
 
-      if (msg.fork !== this.core.tree.fork || !this.core.bitfield.get(index)) {
-        return null
+      if (msg.fork !== this.core.state.fork || !this.core.bitfield.get(index)) {
+        return new ProofRequest(msg, null, null, null)
       }
 
-      proof.block.value = await this.core.blocks.get(index)
+      block = this.core.blocks.get(batch, index)
     }
 
-    if (msg.manifest && !this.core.compat) {
-      proof.manifest = this.core.header.manifest
-    }
+    const manifest = (msg.manifest && !this.core.compat) ? this.core.header.manifest : null
+    const treeBatch = this.core.state.createTreeBatch()
+    const proof = await this.core.tree.proof(batch, treeBatch, msg)
 
-    return proof
+    return new ProofRequest(msg, proof, block, manifest)
   }
 
   async onrequest (msg) {
@@ -703,6 +721,8 @@ class Peer {
       return
     }
 
+    if (this.replicator.destroyed) return
+
     await this._handleRequest(msg)
   }
 
@@ -720,7 +740,7 @@ class Peer {
   }
 
   async _handleRequests () {
-    if (this.receiverBusy) return
+    if (this.receiverBusy || this.replicator.destroyed) return
     this.receiverBusy = true
     this.protomux.cork()
 
@@ -734,45 +754,48 @@ class Peer {
   }
 
   async _handleRequest (msg) {
-    let proof = null
+    const batch = this.core.storage.read()
 
     // TODO: could still be answerable if (index, fork) is an ancestor of the current fork
-    if (msg.fork === this.core.tree.fork) {
-      try {
-        proof = await this._getProof(msg)
-      } catch (err) {
-        safetyCatch(err)
-        if (msg.fork === this.core.tree.fork && isCriticalError(err)) throw err
-      }
-    }
+    const req = msg.fork === this.core.state.fork
+      ? await this._getProof(batch, msg)
+      : new ProofRequest(msg, null, null, null)
+
+    batch.tryFlush()
+
+    await this._fulfillRequest(req)
+  }
+
+  async _fulfillRequest (req) {
+    const proof = await req.fulfill()
 
     // if cancelled do not reply
-    if (this.remoteRequests.get(msg.id) !== msg) {
+    if (this.remoteRequests.get(req.msg.id) !== req.msg) {
       return
     }
 
     // sync from now on, so safe to delete from the map
-    this.remoteRequests.delete(msg.id)
+    this.remoteRequests.delete(req.msg.id)
 
     if (proof === null) {
-      if (msg.manifest && this.core.header.manifest) {
+      if (req.msg.manifest && this.core.header.manifest) {
         const manifest = this.core.header.manifest
-        this.wireData.send({ request: msg.id, fork: this.core.tree.fork, block: null, hash: null, seek: null, upgrade: null, manifest })
+        this.wireData.send({ request: req.msg.id, fork: this.core.state.fork, block: null, hash: null, seek: null, upgrade: null, manifest })
         incrementTx(this.stats.wireData, this.replicator.stats.wireData)
         return
       }
 
-      this.wireNoData.send({ request: msg.id })
+      this.wireNoData.send({ request: req.msg.id })
       return
     }
 
     if (proof.block !== null) {
-      this.replicator.onupload(proof.block.index, proof.block.value, this)
+      this.replicator._onupload(proof.block.index, proof.block.value.byteLength, this)
     }
 
     this.wireData.send({
-      request: msg.id,
-      fork: msg.fork,
+      request: req.msg.id,
+      fork: req.msg.fork,
       block: proof.block,
       hash: proof.hash,
       seek: proof.seek,
@@ -803,7 +826,7 @@ class Peer {
   _checkIfConflict () {
     this.paused = true
 
-    const length = Math.min(this.core.tree.length, this.remoteLength)
+    const length = Math.min(this.core.state.length, this.remoteLength)
     if (length === 0) return // pause and ignore
 
     this.wireRequest.send({
@@ -829,7 +852,7 @@ class Peer {
     }
 
     const req = data.request > 0 ? this.replicator._inflight.get(data.request) : null
-    const reorg = data.fork > this.core.tree.fork
+    const reorg = data.fork > this.core.state.fork
 
     // no push atm, TODO: check if this satisfies another pending request
     // allow reorg pushes tho as those are not written to storage so we'll take all the help we can get
@@ -847,11 +870,12 @@ class Peer {
       if (isBlockRequest(req)) this.replicator._unmarkInflight(req.block.index)
 
       this.paused = true
-      this.replicator.oninvalid(err, req, data, this)
+      this.replicator._oninvalid(err, req, data, this)
       return
     }
 
     this.dataProcessing++
+    if (isBlockRequest(req)) this.replicator._markProcessing(req.block.index)
 
     try {
       if (!matchingRequest(req, data) || !(await this.core.verify(data, this))) {
@@ -877,9 +901,10 @@ class Peer {
       }
 
       this.replicator._onnodata(this, req)
-      this.replicator.oninvalid(err, req, data, this)
+      this.replicator._oninvalid(err, req, data, this)
       return
     } finally {
+      if (isBlockRequest(req)) this.replicator._markProcessed(req.block.index)
       this.dataProcessing--
     }
 
@@ -936,11 +961,10 @@ class Peer {
       return
     }
 
-    const contig = Math.min(this.core.tree.length, this.core.header.hints.contiguousLength)
+    const contig = Math.min(this.core.state.length, this.core.header.hints.contiguousLength)
 
     if (start + length < contig) {
-      const delta = contig - start
-      this.missingBlocks.setRange(start, delta, false)
+      this.missingBlocks.setRange(start, contig, false)
       return
     }
 
@@ -950,7 +974,7 @@ class Peer {
       length += rem
     }
 
-    const end = start + Math.min(length, this.core.tree.length)
+    const end = start + Math.min(length, this.core.state.length)
     while (start < end) {
       const local = bitfield.getBitfield(start)
 
@@ -1009,12 +1033,12 @@ class Peer {
       this.missingBlocks.set(start, has && !bitfield.get(start))
     } else {
       const rangeStart = this.remoteBitfield.findFirst(!has, start)
-      const rangeLength = length - (rangeStart - start)
+      const rangeEnd = length + start
 
-      if (rangeLength > 0) {
-        this.remoteBitfield.setRange(rangeStart, rangeLength, has)
-        this.missingBlocks.setRange(rangeStart, rangeLength, has)
-        if (has) this._clearLocalRange(rangeStart, rangeLength)
+      if (rangeStart !== -1 && rangeStart < rangeEnd) {
+        this.remoteBitfield.setRange(rangeStart, rangeEnd, has)
+        this.missingBlocks.setRange(rangeStart, rangeEnd, has)
+        if (has) this._clearLocalRange(rangeStart, rangeEnd - rangeStart)
       }
     }
 
@@ -1033,16 +1057,16 @@ class Peer {
 
   async _onconflict () {
     this.protomux.cork()
-    if (this.remoteLength > 0 && this.core.tree.fork === this.remoteFork) {
+    if (this.remoteLength > 0 && this.core.state.fork === this.remoteFork) {
       await this.onrequest({
         id: 0,
-        fork: this.core.tree.fork,
+        fork: this.core.state.fork,
         block: null,
         hash: null,
         seek: null,
         upgrade: {
           start: 0,
-          length: Math.min(this.core.tree.length, this.remoteLength)
+          length: Math.min(this.core.state.length, this.remoteLength)
         }
       })
     }
@@ -1074,7 +1098,7 @@ class Peer {
       seek: null,
       upgrade: needsUpgrade === false
         ? null
-        : { start: this.core.tree.length, length: this.remoteLength - this.core.tree.length },
+        : { start: this.core.state.length, length: this.remoteLength - this.core.state.length },
       // remote manifest check can be removed eventually...
       manifest: this.core.header.manifest === null && this.remoteHasManifest === true,
       priority
@@ -1099,7 +1123,7 @@ class Peer {
     // if replicator is updating the seeks etc, bail and wait for it to drain
     if (this.replicator._updatesPending > 0) return false
 
-    const { length, fork } = this.core.tree
+    const { length, fork } = this.core.state
 
     if (fork !== this.remoteFork) return false
 
@@ -1156,7 +1180,7 @@ class Peer {
   }
 
   _hasTreeParent (index) {
-    if (this.remoteLength >= this.core.tree.length) return true
+    if (this.remoteLength >= this.core.state.length) return true
 
     const ite = flatTree.iterator(index * 2)
 
@@ -1170,7 +1194,7 @@ class Peer {
       length = left + span
 
       // if larger than local AND larger than remote - they share the root so its ok
-      if (length > this.core.tree.length) {
+      if (length > this.core.state.length) {
         if (length > this.remoteLength) return true
         break
       }
@@ -1201,7 +1225,7 @@ class Peer {
   }
 
   _requestBlock (b) {
-    const { length, fork } = this.core.tree
+    const { length, fork } = this.core.state
 
     if (this._remoteHasBlock(b.index) === false || fork !== this.remoteFork) {
       this._maybeWant(b.index)
@@ -1257,7 +1281,7 @@ class Peer {
   }
 
   _requestRange (r) {
-    const { length, fork } = this.core.tree
+    const { length, fork } = this.core.state
 
     if (r.blocks) {
       let min = -1
@@ -1275,7 +1299,7 @@ class Peer {
       return false
     }
 
-    const end = Math.min(this.core.tree.length, Math.min(r.end === -1 ? this.remoteLength : r.end, this.remoteLength))
+    const end = Math.min(this.core.state.length, Math.min(r.end === -1 ? this.remoteLength : r.end, this.remoteLength))
     if (end <= r.start || fork !== this.remoteFork) return false
 
     const len = end - r.start
@@ -1368,7 +1392,7 @@ class Peer {
   }
 
   async _send (req) {
-    const fork = this.core.tree.fork
+    const fork = this.core.state.fork
 
     this.inflight++
     this.replicator._inflight.add(req)
@@ -1380,11 +1404,11 @@ class Peer {
 
     try {
       if (req.block !== null && req.fork === fork) {
-        req.block.nodes = await this.core.tree.missingNodes(2 * req.block.index)
+        req.block.nodes = await this.core.tree.missingNodes(2 * req.block.index, this.core.state.length)
         if (req.priority === PRIORITY.CANCELLED) return
       }
       if (req.hash !== null && req.fork === fork && req.hash.nodes === 0) {
-        req.hash.nodes = await this.core.tree.missingNodes(req.hash.index)
+        req.hash.nodes = await this.core.tree.missingNodes(req.hash.index, this.core.state.length)
         if (req.priority === PRIORITY.CANCELLED) return
 
         // nodes === 0, we already have it, bail
@@ -1407,23 +1431,15 @@ class Peer {
 module.exports = class Replicator {
   static Peer = Peer // hack to be able to access Peer from outside this module
 
-  constructor (core, key, {
+  constructor (core, {
     notDownloadingLinger = NOT_DOWNLOADING_SLACK,
     eagerUpgrade = true,
     allowFork = true,
-    inflightRange = null,
-    onpeerupdate = noop,
-    onupload = noop,
-    oninvalid = noop
+    inflightRange = null
   } = {}) {
-    this.key = key
-    this.discoveryKey = core.crypto.discoveryKey(key)
     this.core = core
     this.eagerUpgrade = eagerUpgrade
     this.allowFork = allowFork
-    this.onpeerupdate = onpeerupdate
-    this.onupload = onupload
-    this.oninvalid = oninvalid
     this.ondownloading = null // optional external hook for monitoring downloading status
     this.peers = []
     this.findingPeers = 0 // updateable from the outside
@@ -1461,12 +1477,11 @@ module.exports = class Replicator {
     this._ranges = []
 
     this._hadPeers = false
+    this._active = 0
     this._ifAvailable = 0
     this._updatesPending = 0
     this._applyingReorg = null
     this._manifestPeer = null
-    this._hasSession = false
-    this._peerSessions = 0
     this._notDownloadingLinger = notDownloadingLinger
     this._downloadingTimer = null
 
@@ -1510,7 +1525,7 @@ module.exports = class Replicator {
     if (downloading) { // restart channel if needed...
       for (const protomux of this._attached) {
         if (!protomux.stream.handshakeHash) continue
-        if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) continue
+        if (protomux.opened({ protocol: 'hypercore/alpha', id: this.core.discoveryKey })) continue
         this._makePeer(protomux, true)
       }
     } else {
@@ -1557,7 +1572,9 @@ module.exports = class Replicator {
     for (const peer of this.peers) peer.signalUpgrade()
     if (this._blocks.isEmpty() === false) this._resolveBlocksLocally()
     if (this._upgrade !== null) this._resolveUpgradeRequest(null)
-    if (this._ranges.length !== 0 || this._seeks.length !== 0) this._updateNonPrimary(true)
+    if (!this._blocks.isEmpty() || this._ranges.length !== 0 || this._seeks.length !== 0) {
+      this._updateNonPrimary(true)
+    }
   }
 
   // Called externally when a conflict has been detected and verified
@@ -1689,13 +1706,13 @@ module.exports = class Replicator {
 
       if (peer.remoteSynced === false) return
 
-      if (this.core.tree.length === 0 && peer.remoteLength > 0) return
+      if (this.core.state.length === 0 && peer.remoteLength > 0) return
 
       if (peer.remoteLength <= this._upgrade.length || peer.remoteFork !== this._upgrade.fork) continue
 
       if (peer.syncsProcessing > 0) return
 
-      if (peer.lengthAcked !== this.core.tree.length && peer.remoteFork === this.core.tree.fork) return
+      if (peer.lengthAcked !== this.core.state.length && peer.remoteFork === this.core.state.fork) return
       if (peer.remoteCanUpgrade === true) return
     }
 
@@ -1725,7 +1742,7 @@ module.exports = class Replicator {
     if (this._upgrade !== null) return this._upgrade
 
     // TODO: needs a reorg: true/false flag to indicate if the user requested a reorg
-    this._upgrade = new UpgradeRequest(this, this.core.tree.fork, this.core.tree.length)
+    this._upgrade = new UpgradeRequest(this, this.core.state.fork, this.core.state.length)
 
     return this._upgrade
   }
@@ -1763,19 +1780,19 @@ module.exports = class Replicator {
   _shouldUpgrade (peer) {
     if (this._upgrade !== null && this._upgrade.inflight.length > 0) return false
     return peer.remoteCanUpgrade === true &&
-      peer.remoteLength > this.core.tree.length &&
-      peer.lengthAcked === this.core.tree.length
+      peer.remoteLength > this.core.state.length &&
+      peer.lengthAcked === this.core.state.length
   }
 
   _autoUpgrade (peer) {
-    return this._upgrade !== null && peer.remoteFork === this.core.tree.fork && this._shouldUpgrade(peer)
+    return this._upgrade !== null && peer.remoteFork === this.core.state.fork && this._shouldUpgrade(peer)
   }
 
   _addPeer (peer) {
     this._hadPeers = true
     this.peers.push(peer)
     this.updatePeer(peer)
-    this.onpeerupdate(true, peer)
+    this._onpeerupdate(true, peer)
   }
 
   _requestDone (id, roundtrip) {
@@ -1795,9 +1812,7 @@ module.exports = class Replicator {
       this._clearRequest(peer, req)
     }
 
-    if (peer.useSession) this._closeSessionMaybe()
-
-    this.onpeerupdate(false, peer)
+    this._onpeerupdate(false, peer)
     this.updateAll()
   }
 
@@ -1817,22 +1832,19 @@ module.exports = class Replicator {
   async _resolveBlocksLocally () {
     // TODO: check if fork compat etc. Requires that we pass down truncation info
 
-    let clear = null
+    const clear = []
+    const blocks = []
 
+    const reader = this.core.storage.read()
     for (const b of this._blocks) {
       if (this.core.bitfield.get(b.index) === false) continue
-
-      try {
-        b.resolve(await this.core.blocks.get(b.index))
-      } catch (err) {
-        b.reject(err)
-      }
-
-      if (clear === null) clear = []
-      clear.push(b)
+      blocks.push(this._resolveLocalBlock(b, reader, clear))
     }
+    reader.tryFlush()
 
-    if (clear === null) return
+    await Promise.all(blocks)
+
+    if (!clear.length) return
 
     // Currently the block tracker does not support deletes during iteration, so we make
     // sure to clear them afterwards.
@@ -1842,6 +1854,17 @@ module.exports = class Replicator {
     }
   }
 
+  async _resolveLocalBlock (b, reader, resolved) {
+    try {
+      b.resolve(await this.core.blocks.get(reader, b.index))
+    } catch (err) {
+      b.reject(err)
+      return
+    }
+
+    resolved.push(b)
+  }
+
   _resolveBlockRequest (tracker, index, value, req) {
     const b = tracker.remove(index)
     if (b === null) return false
@@ -1865,7 +1888,7 @@ module.exports = class Replicator {
   _resolveUpgradeRequest (req) {
     if (req !== null) removeInflight(this._upgrade.inflight, req)
 
-    if (this.core.tree.length === this._upgrade.length && this.core.tree.fork === this._upgrade.fork) return false
+    if (this.core.state.length === this._upgrade.length && this.core.state.fork === this._upgrade.fork) return false
 
     const u = this._upgrade
     this._upgrade = null
@@ -2035,6 +2058,25 @@ module.exports = class Replicator {
     }
   }
 
+  _markProcessing (index) {
+    const b = this._blocks.get(index)
+    if (b) {
+      b.processing = true
+      return
+    }
+
+    const h = this._hashes.get(index)
+    if (h) h.processing = true
+  }
+
+  _markProcessed (index) {
+    const b = this._blocks.get(index)
+    if (b) return b.processed()
+
+    const h = this._hashes.get(index)
+    if (h) h.processed()
+  }
+
   _markInflight (index) {
     if (this.core.skipBitfield !== null) this.core.skipBitfield.set(index, true)
     for (const peer of this.peers) peer._markInflight(index)
@@ -2048,6 +2090,7 @@ module.exports = class Replicator {
   _ondata (peer, req, data) {
     if (data.block !== null) {
       this._resolveBlockRequest(this._blocks, data.block.index, data.block.value, req)
+      this._ondownload(data.block.index, data.block.value.byteLength, peer)
     }
 
     if (data.hash !== null && (data.hash.index & 1) === 0) {
@@ -2075,9 +2118,9 @@ module.exports = class Replicator {
   }
 
   _onwant (peer, start, length) {
-    const contig = Math.min(this.core.tree.length, this.core.header.hints.contiguousLength)
+    const contig = Math.min(this.core.state.length, this.core.header.hints.contiguousLength)
 
-    if (start + length < contig || (this.core.tree.length === contig)) {
+    if (start + length < contig || (this.core.state.length === contig)) {
       peer.wireRange.send({
         drop: false,
         start: 0,
@@ -2087,7 +2130,7 @@ module.exports = class Replicator {
       return
     }
 
-    length = Math.min(length, this.core.tree.length - start)
+    length = Math.min(length, this.core.state.length - start)
 
     peer.protomux.cork()
 
@@ -2173,7 +2216,7 @@ module.exports = class Replicator {
   }
 
   _updateFork (peer) {
-    if (this._applyingReorg !== null || this.allowFork === false || peer.remoteFork <= this.core.tree.fork) {
+    if (this._applyingReorg !== null || this.allowFork === false || peer.remoteFork <= this.core.state.fork) {
       return false
     }
 
@@ -2304,67 +2347,62 @@ module.exports = class Replicator {
     this._maybeResolveIfAvailableRanges()
   }
 
-  _closeSessionMaybe () {
-    if (this._hasSession && this._peerSessions === 0) {
-      this._hasSession = false
-      this.core.active--
-    }
-
-    // we were the last active ref, so lets shut things down
-    if (this.core.active === 0 && this.core.sessions.length === 0) {
-      this.destroy()
-      this.core.close().catch(safetyCatch)
-      return
-    }
-
-    // in case one session is still alive but its been marked for auto close also kill it
-    if (this.core.sessions.length === 1 && this.core.active === 1 && this.core.sessions[0].autoClose) {
-      this.core.sessions[0].close().catch(safetyCatch)
-    }
+  onpeerdestroy () {
+    if (--this._active === 0) this.core.checkIfIdle()
   }
 
   attached (protomux) {
     return this._attached.has(protomux)
   }
 
-  ensureSession () {
-    if (this._hasSession) return
-    this._hasSession = true
-    this.core.active++
-  }
-
-  attachTo (protomux, useSession) {
+  attachTo (protomux) {
     if (this.core.closed) return
-    if (useSession) this.ensureSession()
 
-    const makePeer = this._makePeer.bind(this, protomux, useSession)
+    const makePeer = this._makePeer.bind(this, protomux)
 
     this._attached.add(protomux)
-    protomux.pair({ protocol: 'hypercore/alpha', id: this.discoveryKey }, makePeer)
+    protomux.pair({ protocol: 'hypercore/alpha', id: this.core.discoveryKey }, makePeer)
     protomux.stream.setMaxListeners(0)
     protomux.stream.on('close', this._onstreamclose)
 
-    if (useSession) this._peerSessions++
     this._ifAvailable++
+    this._active++
 
     protomux.stream.opened.then((opened) => {
-      if (useSession) this._peerSessions--
       this._ifAvailable--
+      this._active--
 
       if (opened && !this.destroyed) makePeer()
-      else if (useSession) this._closeSessionMaybe()
       this._checkUpgradeIfAvailable()
+
+      this.core.checkIfIdle()
     })
   }
 
   detachFrom (protomux) {
     if (this._attached.delete(protomux)) {
       protomux.stream.removeListener('close', this._onstreamclose)
-      protomux.unpair({ protocol: 'hypercore/alpha', id: this.discoveryKey })
+      protomux.unpair({ protocol: 'hypercore/alpha', id: this.core.discoveryKey })
+    }
+  }
+
+  idle () {
+    return this.peers.length === 0 && this._active === 0
+  }
+
+  close () {
+    const waiting = []
+
+    for (const peer of this.peers) {
+      waiting.push(peer.channel.fullyClosed())
     }
+
+    this.destroy()
+    return Promise.all(waiting)
   }
 
   destroy () {
+    if (this.destroyed) return
     this.destroyed = true
 
     if (this._downloadingTimer) {
@@ -2372,31 +2410,26 @@ module.exports = class Replicator {
       this._downloadingTimer = null
     }
 
-    const waiting = []
-
     while (this.peers.length) {
       const peer = this.peers[this.peers.length - 1]
       this.detachFrom(peer.protomux)
       peer.channel.close() // peer is removed from array in onclose
-      waiting.push(peer.channel.fullyClosed())
     }
 
     for (const protomux of this._attached) {
       this.detachFrom(protomux)
     }
-
-    return Promise.all(waiting)
   }
 
-  _makePeer (protomux, useSession) {
+  _makePeer (protomux) {
     const replicator = this
-    if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) return onnochannel()
+    if (protomux.opened({ protocol: 'hypercore/alpha', id: this.core.discoveryKey })) return onnochannel()
 
     const channel = protomux.createChannel({
       userData: null,
       protocol: 'hypercore/alpha',
       aliases: ['hypercore'],
-      id: this.discoveryKey,
+      id: this.core.discoveryKey,
       handshake: m.wire.handshake,
       messages: [
         { encoding: m.wire.sync, onmessage: onwiresync },
@@ -2412,32 +2445,67 @@ module.exports = class Replicator {
       ],
       onopen: onwireopen,
       onclose: onwireclose,
-      ondrain: onwiredrain
+      ondrain: onwiredrain,
+      ondestroy: onwiredestroy
     })
 
     if (channel === null) return onnochannel()
 
-    const peer = new Peer(replicator, protomux, channel, useSession, this.inflightRange)
+    const peer = new Peer(replicator, protomux, channel, this.inflightRange)
     const stream = protomux.stream
 
-    if (useSession) {
-      // session may have been unref'd underneath us
-      replicator.ensureSession()
-      replicator._peerSessions++
-    }
-
     peer.channel.open({
       seeks: true,
-      capability: caps.replicate(stream.isInitiator, this.key, stream.handshakeHash)
+      capability: caps.replicate(stream.isInitiator, this.core.key, stream.handshakeHash)
     })
 
     return true
 
     function onnochannel () {
-      if (useSession) replicator._closeSessionMaybe()
       return false
     }
   }
+
+  _onpeerupdate (added, peer) {
+    const name = added ? 'peer-add' : 'peer-remove'
+    const sessions = this.core.monitors
+
+    for (let i = sessions.length - 1; i >= 0; i--) {
+      sessions[i].emit(name, peer)
+
+      if (added) {
+        for (const ext of sessions[i].extensions.values()) {
+          peer.extensions.set(ext.name, ext)
+        }
+      }
+    }
+  }
+
+  _ondownload (index, byteLength, from) {
+    const sessions = this.core.monitors
+
+    for (let i = sessions.length - 1; i >= 0; i--) {
+      const s = sessions[i]
+      s.emit('download', index, byteLength - s.padding, from)
+    }
+  }
+
+  _onupload (index, byteLength, from) {
+    const sessions = this.core.monitors
+
+    for (let i = sessions.length - 1; i >= 0; i--) {
+      const s = sessions[i]
+      s.emit('upload', index, byteLength - s.padding, from)
+    }
+  }
+
+  _oninvalid (err, req, res, from) {
+    const sessions = this.core.monitors
+
+    for (let i = 0; i < sessions.length; i++) {
+      sessions[i].emit('verification-error', err, req, res, from)
+    }
+  }
 }
 
 function matchingRequest (req, data) {
@@ -2462,8 +2530,6 @@ function removeInflight (inf, req) {
   return true
 }
 
-function noop () {}
-
 function toLength (start, end) {
   return end === -1 ? -1 : (end < start ? 0 : end - start)
 }
@@ -2512,6 +2578,10 @@ function onwireclose (isRemote, c) {
   return c.userData.onclose(isRemote)
 }
 
+function onwiredestroy (c) {
+  c.userData.replicator.onpeerdestroy()
+}
+
 function onwiredrain (c) {
   return c.userData.ondrain()
 }