hypercore 10.17.0 → 10.18.1

package/errors.js

@@ -1,2 +1,2 @@
 // explicitly exposed as hypercore/errors
-module.exports = require('./lib/errors')
+module.exports = require('hypercore-errors')

package/index.js

@@ -23,7 +23,7 @@ const {
   SESSION_CLOSED,
   SESSION_NOT_WRITABLE,
   SNAPSHOT_NOT_AVAILABLE
-} = require('./lib/errors')
+} = require('hypercore-errors')
 
 const promises = Symbol.for('hypercore.promises')
 const inspect = Symbol.for('nodejs.util.inspect.custom')
@@ -359,7 +359,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.id = z32.encode(this.key)
 
     this.replicator = new Replicator(this.core, this.key, {
-      eagerUpdate: true,
+      eagerUpgrade: true,
       allowFork: opts.allowFork !== false,
       onpeerupdate: this._onpeerupdate.bind(this),
       onupload: this._onupload.bind(this)

package/lib/batch.js

@@ -1,4 +1,4 @@
-const { SESSION_NOT_WRITABLE, BATCH_ALREADY_FLUSHED } = require('./errors')
+const { SESSION_NOT_WRITABLE, BATCH_ALREADY_FLUSHED } = require('hypercore-errors')
 
 module.exports = class Batch {
   constructor (session) {

package/lib/core.js

@@ -6,7 +6,7 @@ const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
 const Info = require('./info')
-const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = require('./errors')
+const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = require('hypercore-errors')
 const m = require('./messages')
 
 module.exports = class Core {

package/lib/merkle-tree.js

@@ -4,7 +4,7 @@ const c = require('compact-encoding')
 const Xache = require('xache')
 const b4a = require('b4a')
 const caps = require('./caps')
-const { INVALID_PROOF, INVALID_CHECKSUM, INVALID_OPERATION, BAD_ARGUMENT } = require('./errors')
+const { INVALID_PROOF, INVALID_CHECKSUM, INVALID_OPERATION, BAD_ARGUMENT } = require('hypercore-errors')
 
 const BLANK_HASH = b4a.alloc(32)
 const OLD_TREE = b4a.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])

package/lib/messages.js

@@ -1,6 +1,6 @@
 const c = require('compact-encoding')
 const b4a = require('b4a')
-const { INVALID_OPLOG_VERSION } = require('./errors')
+const { INVALID_OPLOG_VERSION } = require('hypercore-errors')
 
 const EMPTY = b4a.alloc(0)
 
@@ -103,9 +103,10 @@ wire.request = {
     if (m.hash) requestBlock.preencode(state, m.hash)
     if (m.seek) requestSeek.preencode(state, m.seek)
     if (m.upgrade) requestUpgrade.preencode(state, m.upgrade)
+    if (m.priority) c.uint.preencode(state, m.priority)
   },
   encode (state, m) {
-    const flags = (m.block ? 1 : 0) | (m.hash ? 2 : 0) | (m.seek ? 4 : 0) | (m.upgrade ? 8 : 0)
+    const flags = (m.block ? 1 : 0) | (m.hash ? 2 : 0) | (m.seek ? 4 : 0) | (m.upgrade ? 8 : 0) | (m.priority ? 16 : 0)
 
     c.uint.encode(state, flags)
     c.uint.encode(state, m.id)
@@ -115,6 +116,7 @@ wire.request = {
     if (m.hash) requestBlock.encode(state, m.hash)
     if (m.seek) requestSeek.encode(state, m.seek)
     if (m.upgrade) requestUpgrade.encode(state, m.upgrade)
+    if (m.priority) c.uint.encode(state, m.priority)
   },
   decode (state) {
     const flags = c.uint.decode(state)
@@ -125,7 +127,8 @@ wire.request = {
       block: flags & 1 ? requestBlock.decode(state) : null,
       hash: flags & 2 ? requestBlock.decode(state) : null,
       seek: flags & 4 ? requestSeek.decode(state) : null,
-      upgrade: flags & 8 ? requestUpgrade.decode(state) : null
+      upgrade: flags & 8 ? requestUpgrade.decode(state) : null,
+      priority: flags & 16 ? c.uint.decode(state) : 0
     }
   }
 }

package/lib/oplog.js

@@ -1,7 +1,7 @@
 const cenc = require('compact-encoding')
 const b4a = require('b4a')
 const { crc32 } = require('crc-universal')
-const { OPLOG_CORRUPT } = require('./errors')
+const { OPLOG_CORRUPT } = require('hypercore-errors')
 
 module.exports = class Oplog {
   constructor (storage, { pageSize = 4096, headerEncoding = cenc.raw, entryEncoding = cenc.raw, readonly = false } = {}) {

package/lib/receiver-queue.js

@@ -0,0 +1,63 @@
+const FIFO = require('fast-fifo')
+
+module.exports = class ReceiverQueue {
+  constructor () {
+    this.queue = new FIFO()
+    this.priority = []
+    this.requests = new Map()
+    this.length = 0
+  }
+
+  push (req) {
+    // TODO: use a heap at some point if we wanna support multiple prios
+    if (req.priority > 0) this.priority.push(req)
+    else this.queue.push(req)
+
+    this.requests.set(req.id, req)
+    this.length++
+  }
+
+  shift () {
+    while (this.priority.length > 0) {
+      const msg = this.priority.pop()
+      const req = this._processRequest(msg)
+      if (req !== null) return req
+    }
+
+    while (this.queue.length > 0) {
+      const msg = this.queue.shift()
+      const req = this._processRequest(msg)
+      if (req !== null) return req
+    }
+
+    return null
+  }
+
+  _processRequest (req) {
+    if (req.block || req.hash || req.seek || req.upgrade) {
+      this.requests.delete(req.id)
+      this.length--
+      return req
+    }
+
+    return null
+  }
+
+  delete (id) {
+    const req = this.requests.get(id)
+    if (!req) return
+
+    req.block = null
+    req.hash = null
+    req.seek = null
+    req.upgrade = null
+
+    this.requests.delete(id)
+    this.length--
+
+    if (this.length === 0) {
+      this.queue.clear()
+      this.priority = []
+    }
+  }
+}

package/lib/replicator.js

@@ -1,8 +1,9 @@
 const b4a = require('b4a')
 const safetyCatch = require('safety-catch')
 const RandomIterator = require('random-array-iterator')
+const ReceiverQueue = require('./receiver-queue')
 const RemoteBitfield = require('./remote-bitfield')
-const { REQUEST_CANCELLED, REQUEST_TIMEOUT, INVALID_CAPABILITY, SNAPSHOT_NOT_AVAILABLE } = require('./errors')
+const { REQUEST_CANCELLED, REQUEST_TIMEOUT, INVALID_CAPABILITY, SNAPSHOT_NOT_AVAILABLE } = require('hypercore-errors')
 const m = require('./messages')
 const caps = require('./caps')
 
@@ -10,6 +11,12 @@ const DEFAULT_MAX_INFLIGHT = [32, 512]
 const SCALE_LATENCY = 50
 const DEFAULT_SEGMENT_SIZE = 128 * 1024 // 128 KiB
 
+const PRIORITY = {
+  NORMAL: 0,
+  HIGH: 1,
+  VERY_HIGH: 2
+}
+
 class Attachable {
   constructor () {
     this.resolved = false
@@ -95,17 +102,21 @@ class Attachable {
 }
 
 class BlockRequest extends Attachable {
-  constructor (tracker, index) {
+  constructor (tracker, index, priority) {
     super()
 
     this.index = index
+    this.priority = priority
     this.inflight = []
     this.queued = false
     this.tracker = tracker
   }
 
   _unref () {
-    if (this.inflight.length > 0) return
+    for (const req of this.inflight) {
+      req.peer._cancelRequest(req.id)
+    }
+
     this.tracker.remove(this.index)
   }
 }
@@ -233,11 +244,11 @@ class BlockTracker {
     return this._map.get(index) || null
   }
 
-  add (index) {
+  add (index, priority) {
     let b = this._map.get(index)
     if (b) return b
 
-    b = new BlockRequest(this, index)
+    b = new BlockRequest(this, index, priority)
     this._map.set(index, b)
 
     return b
@@ -267,7 +278,7 @@ class Peer {
 
     this.wireSync = this.channel.messages[0]
     this.wireRequest = this.channel.messages[1]
-    this.wireCancel = null
+    this.wireCancel = this.channel.messages[2]
     this.wireData = this.channel.messages[3]
     this.wireNoData = this.channel.messages[4]
     this.wireWant = this.channel.messages[5]
@@ -276,6 +287,9 @@ class Peer {
     this.wireRange = this.channel.messages[8]
     this.wireExtension = this.channel.messages[9]
 
+    this.receiverQueue = new ReceiverQueue()
+    this.receiverBusy = false
+
     this.inflight = 0
     this.inflightRange = DEFAULT_MAX_INFLIGHT
     this.dataProcessing = 0
@@ -311,6 +325,10 @@ class Peer {
     replicator._ifAvailable++
   }
 
+  get remoteContiguousLength () {
+    return this.remoteBitfield.findFirst(false, 0)
+  }
+
   getMaxInflight () {
     const stream = this.stream.rawStream
     if (!stream.udx) return Math.min(this.inflightRange[1], this.inflightRange[0] * 3)
@@ -504,6 +522,35 @@ class Peer {
   }
 
   async onrequest (msg) {
+    if (!this.protomux.drained || this.receiverQueue.length) {
+      this.receiverQueue.push(msg)
+      return
+    }
+
+    await this._handleRequest(msg)
+  }
+
+  oncancel (msg) {
+    this.receiverQueue.delete(msg.request)
+  }
+
+  ondrain () {
+    return this._handleRequests()
+  }
+
+  async _handleRequests () {
+    if (this.receiverBusy) return
+    this.receiverBusy = true
+
+    while (this.remoteOpened && this.protomux.drained && this.receiverQueue.length > 0) {
+      const msg = this.receiverQueue.shift()
+      await this._handleRequest(msg)
+    }
+
+    this.receiverBusy = false
+  }
+
+  async _handleRequest (msg) {
     let proof = null
 
     // TODO: could still be answerable if (index, fork) is an ancestor of the current fork
@@ -536,6 +583,16 @@ class Peer {
     })
   }
 
+  _cancelRequest (id) {
+    const exists = this.replicator._inflight.get(id)
+    if (!exists) return
+
+    this.inflight--
+    this.replicator._inflight.remove(id)
+
+    this.wireCancel.send({ request: id })
+  }
+
   _checkIfConflict (err) {
     this.paused = true
 
@@ -628,7 +685,16 @@ class Peer {
   onrange ({ drop, start, length }) {
     const has = drop === false
 
-    this.remoteBitfield.setRange(start, length, has)
+    if (length === 1) {
+      this.remoteBitfield.setRange(start, length, has)
+    } else {
+      const rangeStart = this.remoteBitfield.findFirst(!has, start)
+      const rangeLength = length - (rangeStart - start)
+
+      if (rangeLength > 0) {
+        this.remoteBitfield.setRange(rangeStart, rangeLength, has)
+      }
+    }
 
     if (drop === false) this._update()
   }
@@ -662,7 +728,7 @@ class Peer {
     this.protomux.uncork()
   }
 
-  _makeRequest (needsUpgrade) {
+  _makeRequest (needsUpgrade, priority) {
     if (needsUpgrade === true && this.replicator._shouldUpgrade(this) === false) {
       return null
     }
@@ -680,7 +746,8 @@ class Peer {
       seek: null,
       upgrade: needsUpgrade === false
         ? null
-        : { start: this.core.tree.length, length: this.remoteLength - this.core.tree.length }
+        : { start: this.core.tree.length, length: this.remoteLength - this.core.tree.length },
+      priority
     }
   }
 
@@ -727,10 +794,10 @@ class Peer {
       if (b !== null && b.inflight.length > 0) continue
 
       // Block is not inflight, but we only want the hash, check if that is inflight
-      const h = this.replicator._hashes.add(index)
+      const h = this.replicator._hashes.add(index, PRIORITY.NORMAL)
       if (h.inflight.length > 0) continue
 
-      const req = this._makeRequest(false)
+      const req = this._makeRequest(false, h.priority)
 
       req.hash = { index: 2 * index, nodes: 0 }
       req.seek = { bytes: s.seeker.bytes }
@@ -759,7 +826,7 @@ class Peer {
       return false
     }
 
-    const req = this._makeRequest(b.index >= length)
+    const req = this._makeRequest(b.index >= length, b.priority)
     if (req === null) return false
 
     req.block = { index: b.index, nodes: 0 }
@@ -792,10 +859,10 @@ class Peer {
       if (this.remoteBitfield.get(index) === false) continue
       if (this.core.bitfield.get(index) === true) continue
 
-      const b = this.replicator._blocks.add(index)
+      const b = this.replicator._blocks.add(index, PRIORITY.NORMAL)
       if (b.inflight.length > 0) continue
 
-      const req = this._makeRequest(index >= length)
+      const req = this._makeRequest(index >= length, b.priority)
 
       // If the request cannot be satisfied, dealloc the block request if no one is subscribed to it
       if (req === null) {
@@ -858,6 +925,8 @@ class Peer {
   }
 
   _maybeWant (start, length = 1) {
+    if (start + length <= this.remoteContiguousLength) return
+
     let i = Math.floor(start / DEFAULT_SEGMENT_SIZE)
     const n = Math.ceil((start + length) / DEFAULT_SEGMENT_SIZE)
 
@@ -911,6 +980,7 @@ module.exports = class Replicator {
     this.peers = []
     this.findingPeers = 0 // updateable from the outside
 
+    this._attached = new Set()
     this._inflight = new InflightTracker()
     this._blocks = new BlockTracker()
     this._hashes = new BlockTracker()
@@ -926,6 +996,13 @@ module.exports = class Replicator {
     this._ifAvailable = 0
     this._updatesPending = 0
     this._applyingReorg = null
+
+    const self = this
+    this._onstreamclose = onstreamclose
+
+    function onstreamclose () {
+      self.detachFrom(this.userData)
+    }
   }
 
   cork () {
@@ -1007,7 +1084,7 @@ module.exports = class Replicator {
   }
 
   addBlock (session, index) {
-    const b = this._blocks.add(index)
+    const b = this._blocks.add(index, PRIORITY.HIGH)
     const ref = b.attach(session)
 
     this._queueBlock(b)
@@ -1596,21 +1673,36 @@ module.exports = class Replicator {
   attachTo (protomux, session) {
     const makePeer = this._makePeer.bind(this, protomux, session)
 
+    this._attached.add(protomux)
     protomux.pair({ protocol: 'hypercore/alpha', id: this.discoveryKey }, makePeer)
+    protomux.stream.setMaxListeners(0)
+    protomux.stream.on('close', this._onstreamclose)
+
     this._ifAvailable++
     protomux.stream.opened.then((opened) => {
       this._ifAvailable--
+
       if (opened) makePeer()
       else if (session) session.close().catch(noop)
       this._checkUpgradeIfAvailable()
     })
   }
 
+  detachFrom (protomux) {
+    if (this._attached.delete(protomux)) {
+      protomux.stream.removeListener('close', this._onstreamclose)
+      protomux.unpair({ protocol: 'hypercore/alpha', id: this.discoveryKey })
+    }
+  }
+
   destroy () {
     for (const peer of this.peers) {
-      peer.protomux.unpair({ protocol: 'hypercore/alpha', id: this.discoveryKey })
+      this.detachFrom(peer.protomux)
       peer.channel.close()
     }
+    for (const protomux of this._attached) {
+      this.detachFrom(protomux)
+    }
   }
 
   _makePeer (protomux, session) {
@@ -1625,7 +1717,7 @@ module.exports = class Replicator {
       messages: [
         { encoding: m.wire.sync, onmessage: onwiresync },
         { encoding: m.wire.request, onmessage: onwirerequest },
-        null, // oncancel
+        { encoding: m.wire.cancel, onmessage: onwirecancel },
         { encoding: m.wire.data, onmessage: onwiredata },
         { encoding: m.wire.noData, onmessage: onwirenodata },
         { encoding: m.wire.want, onmessage: onwirewant },
@@ -1635,7 +1727,8 @@ module.exports = class Replicator {
         { encoding: m.wire.extension, onmessage: onwireextension }
       ],
       onopen: onwireopen,
-      onclose: onwireclose
+      onclose: onwireclose,
+      ondrain: onwiredrain
     })
 
     if (channel === null) return onnochannel()
@@ -1717,6 +1810,10 @@ function onwireclose (isRemote, c) {
   return c.userData.onclose(isRemote)
 }
 
+function onwiredrain (c) {
+  return c.userData.ondrain()
+}
+
 function onwiresync (m, c) {
   return c.userData.onsync(m)
 }
@@ -1725,6 +1822,10 @@ function onwirerequest (m, c) {
   return c.userData.onrequest(m)
 }
 
+function onwirecancel (m, c) {
+  return c.userData.oncancel(m)
+}
+
 function onwiredata (m, c) {
   return c.userData.ondata(m)
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.17.0",
+  "version": "10.18.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -41,10 +41,12 @@
     "compact-encoding": "^2.11.0",
     "crc-universal": "^1.0.2",
     "events": "^3.3.0",
+    "fast-fifo": "^1.3.0",
     "flat-tree": "^1.9.0",
     "hypercore-crypto": "^3.2.1",
+    "hypercore-errors": "^1.0.0",
     "is-options": "^1.0.1",
-    "protomux": "^3.4.0",
+    "protomux": "^3.5.0",
     "quickbit-universal": "^2.1.1",
     "random-access-file": "^4.0.0",
     "random-array-iterator": "^1.0.0",

package/lib/errors.js

@@ -1,90 +0,0 @@
-module.exports = class HypercoreError extends Error {
-  constructor (msg, code, fn = HypercoreError) {
-    super(`${code}: ${msg}`)
-    this.code = code
-
-    if (Error.captureStackTrace) {
-      Error.captureStackTrace(this, fn)
-    }
-  }
-
-  get name () {
-    return 'HypercoreError'
-  }
-
-  static BAD_ARGUMENT (msg) {
-    return new HypercoreError(msg, 'BAD_ARGUMENT', HypercoreError.BAD_ARGUMENT)
-  }
-
-  static STORAGE_EMPTY (msg) {
-    return new HypercoreError(msg, 'STORAGE_EMPTY', HypercoreError.STORAGE_EMPTY)
-  }
-
-  static STORAGE_CONFLICT (msg) {
-    return new HypercoreError(msg, 'STORAGE_CONFLICT', HypercoreError.STORAGE_CONFLICT)
-  }
-
-  static INVALID_SIGNATURE (msg) {
-    return new HypercoreError(msg, 'INVALID_SIGNATURE', HypercoreError.INVALID_SIGNATURE)
-  }
-
-  static INVALID_CAPABILITY (msg) {
-    return new HypercoreError(msg, 'INVALID_CAPABILITY', HypercoreError.INVALID_CAPABILITY)
-  }
-
-  static INVALID_CHECKSUM (msg = 'Invalid checksum') {
-    return new HypercoreError(msg, 'INVALID_CHECKSUM', HypercoreError.INVALID_CHECKSUM)
-  }
-
-  static INVALID_OPERATION (msg) {
-    return new HypercoreError(msg, 'INVALID_OPERATION', HypercoreError.INVALID_OPERATION)
-  }
-
-  static INVALID_PROOF (msg = 'Proof not verifiable') {
-    return new HypercoreError(msg, 'INVALID_PROOF', HypercoreError.INVALID_PROOF)
-  }
-
-  static BLOCK_NOT_AVAILABLE (msg = 'Block is not available') {
-    return new HypercoreError(msg, 'BLOCK_NOT_AVAILABLE', HypercoreError.BLOCK_NOT_AVAILABLE)
-  }
-
-  static SNAPSHOT_NOT_AVAILABLE (msg = 'Snapshot is not available') {
-    return new HypercoreError(msg, 'SNAPSHOT_NOT_AVAILABLE', HypercoreError.SNAPSHOT_NOT_AVAILABLE)
-  }
-
-  static REQUEST_CANCELLED (msg = 'Request was cancelled') {
-    return new HypercoreError(msg, 'REQUEST_CANCELLED', HypercoreError.REQUEST_CANCELLED)
-  }
-
-  static REQUEST_TIMEOUT (msg = 'Request timed out') {
-    return new HypercoreError(msg, 'REQUEST_TIMEOUT', HypercoreError.REQUEST_TIMEOUT)
-  }
-
-  static SESSION_NOT_WRITABLE (msg = 'Session is not writable') {
-    return new HypercoreError(msg, 'SESSION_NOT_WRITABLE', HypercoreError.SESSION_NOT_WRITABLE)
-  }
-
-  static SESSION_CLOSED (msg = 'Session is closed') {
-    return new HypercoreError(msg, 'SESSION_CLOSED', HypercoreError.SESSION_CLOSED)
-  }
-
-  static BATCH_UNFLUSHED (msg = 'Batch not yet flushed') {
-    return new HypercoreError(msg, 'BATCH_UNFLUSHED', HypercoreError.BATCH_UNFLUSHED)
-  }
-
-  static BATCH_ALREADY_EXISTS (msg = 'Batch already exists') {
-    return new HypercoreError(msg, 'BATCH_ALREADY_EXISTS', HypercoreError.BATCH_ALREADY_EXISTS)
-  }
-
-  static BATCH_ALREADY_FLUSHED (msg = 'Batch has already been flushed') {
-    return new HypercoreError(msg, 'BATCH_ALREADY_FLUSHED', HypercoreError.BATCH_ALREADY_FLUSHED)
-  }
-
-  static OPLOG_CORRUPT (msg = 'Oplog file appears corrupt or out of date') {
-    return new HypercoreError(msg, 'OPLOG_CORRUPT', HypercoreError.OPLOG_CORRUPT)
-  }
-
-  static INVALID_OPLOG_VERSION (msg = 'Invalid header version') {
-    return new HypercoreError(msg, 'INVALID_OPLOG_VERSION', HypercoreError.INVALID_OPLOG_VERSION)
-  }
-}