hypercore 10.37.1 → 10.37.3

package/index.js

@@ -249,12 +249,16 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     if (this.opened) ensureEncryption(s, opts)
-
-    this.sessions.push(s)
+    this._addSession(s)
 
     return s
   }
 
+  _addSession (s) {
+    this.sessions.push(s)
+    if (this.core) this.core.active++
+  }
+
   async setEncryptionKey (encryptionKey, opts) {
     if (!this.opened) await this.opening
     this.encryption = encryptionKey ? new BlockEncryption(encryptionKey, this.key, { compat: this.core.compat, ...opts }) : null
@@ -289,8 +293,8 @@ module.exports = class Hypercore extends EventEmitter {
 
     for (const s of sessions) {
       s.sessions = from.sessions
-      s.sessions.push(s)
       s._passCapabilities(from)
+      s._addSession(s)
     }
 
     this.storage = from.storage
@@ -307,7 +311,9 @@ module.exports = class Hypercore extends EventEmitter {
   async _openSession (key, storage, opts) {
     const isFirst = !opts._opening
 
-    if (!isFirst) await opts._opening
+    if (!isFirst) {
+      await opts._opening
+    }
     if (opts.preload) opts = { ...opts, ...(await this._retryPreload(opts.preload)) }
     if (this.cache === null && opts.cache) this.cache = createCache(opts.cache)
 
@@ -346,7 +352,7 @@ module.exports = class Hypercore extends EventEmitter {
     // It's required so that corestore can load a name from userData before 'ready' is emitted.
     if (opts._preready) await opts._preready(this)
 
-    this.replicator.updateActivity(this._active ? 1 : 0, this)
+    this.replicator.updateActivity(this._active ? 1 : 0)
 
     this.opened = true
     this.emit('ready')
@@ -373,6 +379,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.core = await Core.open(this.storage, {
       compat: opts.compat,
       force: opts.force,
+      sessions: this.sessions,
       createIfMissing: opts.createIfMissing,
       readonly: unlocked,
       overwrite: opts.overwrite,
@@ -456,6 +463,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (i === -1) return
 
     this.sessions.splice(i, 1)
+    this.core.active--
     this.readable = false
     this.writable = false
     this.closed = true
@@ -470,14 +478,14 @@ module.exports = class Hypercore extends EventEmitter {
     if (this.replicator !== null) {
       this.replicator.findingPeers -= this._findingPeers
       this.replicator.clearRequests(this.activeRequests, err)
-      this.replicator.updateActivity(this._active ? -1 : 0, this)
+      this.replicator.updateActivity(this._active ? -1 : 0)
     }
 
     this._findingPeers = 0
 
-    if (this.sessions.length) {
+    if (this.sessions.length || this.core.active > 0) {
       // if this is the last session and we are auto closing, trigger that first to enforce error handling
-      if (this.sessions.length === 1 && this.autoClose) await this.sessions[0].close(err)
+      if (this.sessions.length === 1 && this.core.active === 1 && this.autoClose) await this.sessions[0].close(err)
       // emit "fake" close as this is a session
       this.emit('close', false)
       return
@@ -527,8 +535,7 @@ module.exports = class Hypercore extends EventEmitter {
   _attachToMuxerOpened (mux, useSession) {
     // If the user wants to, we can make this replication run in a session
     // that way the core wont close "under them" during replication
-    const session = useSession ? this.session({ active: false }) : null
-    this.replicator.attachTo(mux, session)
+    this.replicator.attachTo(mux, useSession)
   }
 
   get discoveryKey () {

package/lib/core.js

@@ -15,7 +15,7 @@ const audit = require('./audit')
 const { createTracer } = require('hypertrace')
 
 module.exports = class Core {
-  constructor (header, compat, crypto, oplog, bigHeader, tree, blocks, bitfield, verifier, legacy, onupdate, onconflict) {
+  constructor (header, compat, crypto, oplog, bigHeader, tree, blocks, bitfield, verifier, sessions, legacy, onupdate, onconflict) {
     this.tracer = createTracer(this)
     this.onupdate = onupdate
     this.onconflict = onconflict
@@ -33,6 +33,8 @@ module.exports = class Core {
     this.updating = false
     this.closed = false
     this.skipBitfield = null
+    this.active = sessions.length
+    this.sessions = sessions
 
     this._manifestFlushed = !!header.manifest
     this._maxOplogSize = 65536
@@ -192,7 +194,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, compat, crypto, oplog, bigHeader, tree, blocks, bitfield, verifier, legacy, opts.onupdate || noop, opts.onconflict || noop)
+    return new this(header, compat, crypto, oplog, bigHeader, tree, blocks, bitfield, verifier, opts.sessions || [], legacy, opts.onupdate || noop, opts.onconflict || noop)
   }
 
   async audit () {

package/lib/receiver-queue.js

@@ -43,6 +43,13 @@ module.exports = class ReceiverQueue {
     return null
   }
 
+  clear () {
+    this.queue.clear()
+    this.priority = []
+    this.length = 0
+    this.requests.clear()
+  }
+
   delete (id) {
     const req = this.requests.get(id)
     if (!req) return

package/lib/remote-bitfield.js

@@ -146,6 +146,8 @@ class RemoteBitfieldSegment {
 }
 
 module.exports = class RemoteBitfield {
+  static BITS_PER_PAGE = BITS_PER_PAGE
+
   constructor () {
     this._pages = new BigSparseArray()
     this._segments = new BigSparseArray()

package/lib/replicator.js

@@ -232,7 +232,6 @@ class InflightTracker {
 
   add (req) {
     const id = this._free.length ? this._free.pop() : this._requests.push(null)
-
     req.id = id
     this._requests[id - 1] = req
     return req
@@ -242,14 +241,14 @@ class InflightTracker {
     return id <= this._requests.length ? this._requests[id - 1] : null
   }
 
-  remove (id) {
-    if (id <= this._requests.length) {
-      const req = this._requests[id - 1]
-      clearTimeout(req.timeout)
-      req.timeout = null
-      this._requests[id - 1] = null
-      this._free.push(id)
-    }
+  remove (id, roundtrip) {
+    if (id > this._requests.length) return
+    this._requests[id - 1] = null
+    if (roundtrip === true) this._free.push(id)
+  }
+
+  reusable (id) {
+    this._free.push(id)
   }
 }
 
@@ -291,8 +290,38 @@ class BlockTracker {
   }
 }
 
+class RoundtripQueue {
+  constructor () {
+    this.queue = []
+    this.tick = 0
+  }
+
+  clear () {
+    const q = this.queue
+    this.queue = []
+    return q
+  }
+
+  add (id) {
+    this.queue.push([++this.tick, id])
+  }
+
+  flush (tick) {
+    let flushed = null
+
+    for (let i = 0; i < this.queue.length; i++) {
+      if (this.queue[i][0] > tick) break
+      if (flushed === null) flushed = []
+      flushed.push(this.queue[i][1])
+    }
+
+    if (flushed !== null) this.queue.splice(0, flushed.length)
+    return flushed
+  }
+}
+
 class Peer {
-  constructor (replicator, protomux, channel, session, inflightRange) {
+  constructor (replicator, protomux, channel, useSession, inflightRange) {
     this.tracer = createTracer(this, { parent: replicator.core.tracer })
     this.core = replicator.core
     this.replicator = replicator
@@ -303,8 +332,9 @@ class Peer {
     this.inflightRange = inflightRange
 
     this.paused = false
+    this.removed = false
 
-    this.session = session
+    this.useSession = useSession
 
     this.channel = channel
     this.channel.userData = this
@@ -323,6 +353,9 @@ class Peer {
     this.receiverQueue = new ReceiverQueue()
     this.receiverBusy = false
 
+    // most often not used, so made on demand
+    this.roundtripQueue = null
+
     this.inflight = 0
     this.dataProcessing = 0
 
@@ -348,6 +381,7 @@ class Peer {
     this.remoteDownloading = true
     this.remoteSynced = false
     this.remoteHasManifest = false
+    this.remoteRequests = new Map()
 
     this.segmentsWanted = new Set()
     this.broadcastedNonSparse = false
@@ -386,6 +420,19 @@ class Peer {
     if (drop) this._unclearLocalRange(start, length)
     else this._clearLocalRange(start, length)
 
+    // TODO: consider also adding early-returns on the drop===true case
+    if (!drop) {
+      // No need to broadcast if the remote already has this range
+
+      if (this._remoteContiguousLength >= start + length) return
+
+      if (length === 1) {
+        if (this.remoteBitfield.get(start)) return
+      } else {
+        if (this.remoteBitfield.firstUnset(start) >= start + length) return
+      }
+    }
+
     this.wireRange.send({
       drop,
       start,
@@ -467,7 +514,7 @@ class Peer {
     const reopen = isRemote === true && this.remoteOpened === true && this.remoteDownloading === false &&
        this.remoteUploading === true && this.replicator.downloading === true
 
-    if (this.session && !reopen) this.replicator._closeSession(this.session)
+    if (this.useSession && !reopen) this.replicator._closeSession()
 
     if (this.remoteOpened === false) {
       this.replicator._ifAvailable--
@@ -476,10 +523,18 @@ class Peer {
     }
 
     this.remoteOpened = false
+    this.removed = true
+    this.remoteRequests.clear() // cancel all
+    this.receiverQueue.clear()
+
+    if (this.roundtripQueue !== null) {
+      for (const id of this.roundtripQueue.clear()) this.replicator._inflight.reusable(id)
+    }
+
     this.replicator._removePeer(this)
 
     if (reopen) {
-      this.replicator._makePeer(this.protomux, this.session)
+      this.replicator._makePeer(this.protomux, this.useSession)
     }
   }
 
@@ -598,6 +653,15 @@ class Peer {
   async onrequest (msg) {
     this.tracer.trace('onrequest', msg)
 
+    const size = this.remoteRequests.size
+    this.remoteRequests.set(msg.id, msg)
+
+    // if size didnt change -> id overwrite -> old one is deleted, cancel current and re-add
+    if (size === this.remoteRequests.size) {
+      this._cancel(msg.id)
+      this.remoteRequests.set(msg.id, msg)
+    }
+
     if (!this.protomux.drained || this.receiverQueue.length) {
       this.receiverQueue.push(msg)
       return
@@ -607,7 +671,12 @@ class Peer {
   }
 
   oncancel (msg) {
-    this.receiverQueue.delete(msg.request)
+    this._cancel(msg.request)
+  }
+
+  _cancel (id) {
+    this.remoteRequests.delete(id)
+    this.receiverQueue.delete(id)
   }
 
   ondrain () {
@@ -617,12 +686,14 @@ class Peer {
   async _handleRequests () {
     if (this.receiverBusy) return
     this.receiverBusy = true
+    this.protomux.cork()
 
-    while (this.remoteOpened && this.protomux.drained && this.receiverQueue.length > 0) {
+    while (this.remoteOpened && this.protomux.drained && this.receiverQueue.length > 0 && !this.removed) {
       const msg = this.receiverQueue.shift()
       await this._handleRequest(msg)
     }
 
+    this.protomux.uncork()
     this.receiverBusy = false
   }
 
@@ -639,6 +710,14 @@ class Peer {
       }
     }
 
+    // if cancelled do not reply
+    if (this.remoteRequests.get(msg.id) !== msg) {
+      return
+    }
+
+    // sync from now on, so safe to delete from the map
+    this.remoteRequests.delete(msg.id)
+
     if (proof === null) {
       if (msg.manifest && this.core.header.manifest) {
         const manifest = this.core.header.manifest
@@ -670,9 +749,11 @@ class Peer {
     if (!req) return
 
     this.inflight--
-    this.replicator._removeInflight(id)
+    this.replicator._removeInflight(id, false)
     if (isBlockRequest(req)) this.replicator._unmarkInflight(req.block.index)
 
+    if (this.roundtripQueue === null) this.roundtripQueue = new RoundtripQueue()
+    this.roundtripQueue.add(id)
     this.wireCancel.send({ request: id })
   }
 
@@ -713,8 +794,7 @@ class Peer {
 
     if (req !== null) {
       if (req.peer !== this) return
-      this.inflight--
-      this.replicator._removeInflight(req.id)
+      this._onrequestroundtrip(req)
     }
 
     try {
@@ -774,11 +854,19 @@ class Peer {
 
     if (req === null || req.peer !== this) return
 
-    this.inflight--
-    this.replicator._removeInflight(req.id)
+    this._onrequestroundtrip(req)
     this.replicator._onnodata(this, req)
   }
 
+  _onrequestroundtrip (req) {
+    this.inflight--
+    this.replicator._removeInflight(req.id, true)
+    if (this.roundtripQueue === null) return
+    const flushed = this.roundtripQueue.flush(req.rt)
+    if (flushed === null) return
+    for (const id of flushed) this.replicator._inflight.reusable(id)
+  }
+
   onwant ({ start, length }) {
     this.replicator._onwant(this, start, length)
   }
@@ -928,6 +1016,7 @@ class Peer {
 
     return {
       peer: this,
+      rt: this.roundtripQueue === null ? 0 : this.roundtripQueue.tick,
       id: 0,
       fork: this.remoteFork,
       block: null,
@@ -1327,19 +1416,19 @@ module.exports = class Replicator {
     return this.downloading || !this._inflight.idle
   }
 
-  setDownloading (downloading, session) {
+  setDownloading (downloading) {
     clearTimeout(this._downloadingTimer)
 
     if (this.destroyed) return
     if (downloading || this._notDownloadingLinger === 0) {
-      this.setDownloadingNow(downloading, session)
+      this.setDownloadingNow(downloading)
       return
     }
 
-    this._downloadingTimer = setTimeout(setDownloadingLater, this._notDownloadingLinger, this, downloading, session)
+    this._downloadingTimer = setTimeout(setDownloadingLater, this._notDownloadingLinger, this, downloading)
   }
 
-  setDownloadingNow (downloading, session) {
+  setDownloadingNow (downloading) {
     this._downloadingTimer = null
     if (this.downloading === downloading) return
     this.downloading = downloading
@@ -1351,7 +1440,7 @@ module.exports = class Replicator {
       for (const protomux of this._attached) {
         if (!protomux.stream.handshakeHash) continue
         if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) continue
-        this._makePeer(protomux, session && session.session({ active: false }))
+        this._makePeer(protomux, true)
       }
     } else {
       for (const peer of this.peers) peer.closeIfIdle()
@@ -1605,21 +1694,20 @@ module.exports = class Replicator {
     this.onpeerupdate(true, peer)
   }
 
-  _removeInflight (id) {
-    this._inflight.remove(id)
+  _removeInflight (id, roundtrip) {
+    this._inflight.remove(id, roundtrip)
     if (this.isDownloading() === true) return
     for (const peer of this.peers) peer.signalUpgrade()
   }
 
   _removePeer (peer) {
     this.peers.splice(this.peers.indexOf(peer), 1)
-    peer.removed = true
 
     if (this._manifestPeer === peer) this._manifestPeer = null
 
     for (const req of this._inflight) {
       if (req.peer !== peer) continue
-      this._inflight.remove(req.id)
+      this._inflight.remove(req.id, true)
       this._clearRequest(peer, req)
     }
 
@@ -1634,7 +1722,7 @@ module.exports = class Replicator {
   }
 
   _resolveHashLocally (peer, req) {
-    this._removeInflight(req.id)
+    this._removeInflight(req.id, false)
     this._resolveBlockRequest(this._hashes, req.hash.index / 2, null, req)
     this.updatePeer(peer)
   }
@@ -2086,16 +2174,32 @@ module.exports = class Replicator {
     this._maybeResolveIfAvailableRanges()
   }
 
-  _closeSession (session) {
-    session.close().catch(safetyCatch)
+  _closeSession () {
+    this.core.active--
+
+    // we were the last active ref, so lets shut things down
+    if (this.core.active === 0 && this.core.sessions.length === 0) {
+      this.destroy()
+      this.core.close().catch(safetyCatch)
+      return
+    }
+
+    // in case one session is still alive but its been marked for auto close also kill it
+    if (this.core.sessions.length === 1 && this.core.active === 1 && this.core.sessions[0].autoClose) {
+      this.core.sessions[0].close().catch(safetyCatch)
+    }
   }
 
   attached (protomux) {
     return this._attached.has(protomux)
   }
 
-  attachTo (protomux, session) {
-    const makePeer = this._makePeer.bind(this, protomux, session)
+  attachTo (protomux, useSession) {
+    if (useSession) {
+      this.core.active++
+    }
+
+    const makePeer = this._makePeer.bind(this, protomux, useSession)
 
     this._attached.add(protomux)
     protomux.pair({ protocol: 'hypercore/alpha', id: this.discoveryKey }, makePeer)
@@ -2107,7 +2211,7 @@ module.exports = class Replicator {
       this._ifAvailable--
 
       if (opened && !this.destroyed) makePeer()
-      else if (session) this._closeSession(session)
+      else if (useSession) this._closeSession()
       this._checkUpgradeIfAvailable()
     })
   }
@@ -2134,7 +2238,7 @@ module.exports = class Replicator {
     }
   }
 
-  _makePeer (protomux, session) {
+  _makePeer (protomux, useSession) {
     const replicator = this
     if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) return onnochannel()
 
@@ -2163,7 +2267,7 @@ module.exports = class Replicator {
 
     if (channel === null) return onnochannel()
 
-    const peer = new Peer(replicator, protomux, channel, session, this.inflightRange)
+    const peer = new Peer(replicator, protomux, channel, useSession, this.inflightRange)
     const stream = protomux.stream
 
     peer.channel.open({
@@ -2174,7 +2278,7 @@ module.exports = class Replicator {
     return true
 
     function onnochannel () {
-      if (session) replicator._closeSession(session)
+      if (useSession) replicator._closeSession()
       return false
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.37.1",
+  "version": "10.37.3",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {