hypercore 10.0.0-alpha.44 → 10.0.0-alpha.47

package/index.js

@@ -113,6 +113,7 @@ module.exports = class Hypercore extends EventEmitter {
       indent + '  opened: ' + opts.stylize(this.opened, 'boolean') + '\n' +
       indent + '  closed: ' + opts.stylize(this.closed, 'boolean') + '\n' +
       indent + '  snapshotted: ' + opts.stylize(this.snapshotted, 'boolean') + '\n' +
+      indent + '  sparse: ' + opts.stylize(this.sparse, 'boolean') + '\n' +
       indent + '  writable: ' + opts.stylize(this.writable, 'boolean') + '\n' +
       indent + '  length: ' + opts.stylize(this.length, 'number') + '\n' +
       indent + '  byteLength: ' + opts.stylize(this.byteLength, 'number') + '\n' +
@@ -182,9 +183,11 @@ module.exports = class Hypercore extends EventEmitter {
       throw SESSION_CLOSED('Cannot make sessions on a closing core')
     }
 
+    const sparse = opts.sparse === false ? false : this.sparse
     const Clz = opts.class || Hypercore
     const s = new Clz(this.storage, this.key, {
       ...opts,
+      sparse,
       _opening: this.opening,
       _sessions: this.sessions
     })
@@ -297,7 +300,8 @@ module.exports = class Hypercore extends EventEmitter {
       crypto: this.crypto,
       legacy: opts.legacy,
       auth: opts.auth,
-      onupdate: this._oncoreupdate.bind(this)
+      onupdate: this._oncoreupdate.bind(this),
+      oncontigupdate: this._oncorecontigupdate.bind(this)
     })
 
     if (opts.userData) {
@@ -323,14 +327,27 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
-  _updateSnapshot () {
-    const prev = this._snapshot
-    const next = this._snapshot = {
-      length: this.core.tree.length,
-      byteLength: this.core.tree.byteLength,
+  _getSnapshot () {
+    if (this.sparse) {
+      return {
+        length: this.core.tree.length,
+        byteLength: this.core.tree.byteLength,
+        fork: this.core.tree.fork,
+        compatLength: this.core.tree.length
+      }
+    }
+
+    return {
+      length: this.core.header.contiguousLength,
+      byteLength: 0,
       fork: this.core.tree.fork,
-      compatLength: this.core.tree.length
+      compatLength: this.core.header.contiguousLength
     }
+  }
+
+  _updateSnapshot () {
+    const prev = this._snapshot
+    const next = this._snapshot = this._getSnapshot()
 
     if (!prev) return true
     return prev.length !== next.length || prev.fork !== next.fork
@@ -484,9 +501,10 @@ module.exports = class Hypercore extends EventEmitter {
           if (s._snapshot && bitfield.start < s._snapshot.compatLength) s._snapshot.compatLength = bitfield.start
         }
 
-        if (appended) {
-          s.emit('append')
-        }
+        // For sparse sessions, immediately emit appends. Non-sparse sessions
+        // are handled separately and only emit appends when their contiguous
+        // length is updated.
+        if (appended && s.sparse) s.emit('append')
       }
 
       this.replicator.onupgrade()
@@ -505,6 +523,16 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
+  _oncorecontigupdate () {
+    // For non-sparse sessions, emit appends only when the contiguous length is
+    // updated.
+    for (let i = 0; i < this.sessions.length; i++) {
+      const s = this.sessions[i]
+
+      if (!s.sparse) s.emit('append')
+    }
+  }
+
   _onpeerupdate (added, peer) {
     const name = added ? 'peer-add' : 'peer-remove'
 
@@ -561,10 +589,19 @@ module.exports = class Hypercore extends EventEmitter {
     const activeRequests = (opts && opts.activeRequests) || this.activeRequests
     const req = this.replicator.addUpgrade(activeRequests)
 
-    if (!this.snapshotted) return req.promise
-    if (!(await req.promise)) return false
+    const upgraded = await req.promise
+
+    if (!this.sparse) {
+      // Download all available blocks in non-sparse mode
+      const start = this.length
+      const end = this.core.tree.length
+
+      await this.download({ start, end, ifAvailable: true }).downloaded()
+    }
 
-    return this._updateSnapshot()
+    if (!upgraded) return false
+    if (this.snapshotted) return this._updateSnapshot()
+    return true
   }
 
   async seek (bytes, opts) {

package/lib/caps.js

@@ -4,9 +4,7 @@ const b4a = require('b4a')
 const c = require('compact-encoding')
 
 // TODO: rename this to "crypto" and move everything hashing related etc in here
-// Also lets move the tree stuff from hypercore-crypto here, and loose the types
-// from the hashes there - they are not needed since we lock the indexes in the tree
-// hash and just makes alignment etc harder in other languages
+// Also lets move the tree stuff from hypercore-crypto here
 
 const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER] = crypto.namespace('hypercore', 3)
 

package/lib/core.js

@@ -9,8 +9,9 @@ const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = req
 const m = require('./messages')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate) {
+  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate, oncontigupdate) {
     this.onupdate = onupdate
+    this.oncontigupdate = oncontigupdate
     this.header = header
     this.crypto = crypto
     this.oplog = oplog
@@ -164,7 +165,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop)
+    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop, opts.oncontigupdate || noop)
   }
 
   _shouldFlush () {
@@ -202,6 +203,7 @@ module.exports = class Core {
       while (this.bitfield.get(i)) i++
 
       this.header.contiguousLength = i
+      this.oncontigupdate()
     }
   }
 
@@ -285,6 +287,7 @@ module.exports = class Core {
       batch.commit()
 
       this.header.contiguousLength = batch.length
+      this.oncontigupdate()
       this.header.tree.length = batch.length
       this.header.tree.rootHash = hash
       this.header.tree.signature = batch.signature
@@ -472,6 +475,7 @@ module.exports = class Core {
     const appended = batch.length > batch.ancestors
 
     this.header.contiguousLength = Math.min(batch.ancestors, this.header.contiguousLength)
+    this.oncontigupdate()
     this.header.tree.fork = batch.fork
     this.header.tree.length = batch.length
     this.header.tree.rootHash = batch.hash()

package/lib/replicator.js

@@ -102,12 +102,13 @@ class BlockRequest extends Attachable {
 }
 
 class RangeRequest extends Attachable {
-  constructor (ranges, start, end, linear, blocks) {
+  constructor (ranges, start, end, linear, ifAvailable, blocks) {
     super()
 
     this.start = start
     this.end = end
     this.linear = linear
+    this.ifAvailable = ifAvailable
     this.blocks = blocks
     this.ranges = ranges
 
@@ -172,6 +173,10 @@ class InflightTracker {
     this._free = []
   }
 
+  get idle () {
+    return this._requests.length === this._free.length
+  }
+
   * [Symbol.iterator] () {
     for (const req of this._requests) {
       if (req !== null) yield req
@@ -260,6 +265,7 @@ class Peer {
 
     this.inflight = 0
     this.maxInflight = DEFAULT_MAX_INFLIGHT
+    this.dataProcessing = 0
 
     this.canUpgrade = true
 
@@ -515,6 +521,8 @@ class Peer {
 
     if (reorg === true) return this.replicator._onreorgdata(this, req, data)
 
+    this.dataProcessing++
+
     try {
       if (!matchingRequest(req, data) || !(await this.core.verify(data, this))) {
         this.replicator._onnodata(this, req)
@@ -523,6 +531,8 @@ class Peer {
     } catch (err) {
       this.replicator._onnodata(this, req)
       throw err
+    } finally {
+      this.dataProcessing--
     }
 
     this.replicator._ondata(this, req, data)
@@ -909,13 +919,21 @@ module.exports = class Replicator {
     return ref
   }
 
-  addRange (session, { start = 0, end = -1, length = toLength(start, end), blocks = null, linear = false } = {}) {
+  addRange (session, { start = 0, end = -1, length = toLength(start, end), blocks = null, linear = false, ifAvailable = false } = {}) {
     if (blocks !== null) { // if using blocks, start, end just acts as frames around the blocks array
       start = 0
       end = length = blocks.length
     }
 
-    const r = new RangeRequest(this._ranges, start, length === -1 ? -1 : start + length, linear, blocks)
+    const r = new RangeRequest(
+      this._ranges,
+      start,
+      length === -1 ? -1 : start + length,
+      linear,
+      ifAvailable,
+      blocks
+    )
+
     const ref = r.attach(session)
 
     this._ranges.push(r)
@@ -1115,6 +1133,14 @@ module.exports = class Replicator {
     return true
   }
 
+  _resolveRangeRequest (req, index) {
+    const head = this._ranges.pop()
+
+    if (index < this._ranges.length) this._ranges[index] = head
+
+    req.resolve(true)
+  }
+
   _clearInflightBlock (tracker, req) {
     const isBlock = tracker === this._blocks
     const index = isBlock === true ? req.block.index : req.hash.index / 2
@@ -1168,14 +1194,9 @@ module.exports = class Replicator {
         while (r.start < r.end && this.core.bitfield.get(mapIndex(r.blocks, r.start)) === true) r.start++
         while (r.start < r.end && this.core.bitfield.get(mapIndex(r.blocks, r.end - 1)) === true) r.end--
 
-        if (r.end === -1 || r.start < r.end) continue
-
-        if (i < this._ranges.length - 1) this._ranges[i] = this._ranges.pop()
-        else this._ranges.pop()
-
-        i--
-
-        r.resolve(true)
+        if (r.end !== -1 && r.start >= r.end) {
+          this._resolveRangeRequest(r, i--)
+        }
       }
 
       for (let i = 0; i < this._seeks.length; i++) {
@@ -1201,7 +1222,7 @@ module.exports = class Replicator {
         else s.resolve(res)
       }
 
-      this.updateAll()
+      if (this._inflight.idle) this.updateAll()
 
       // No additional updates scheduled - return
       if (--this._updatesPending === 0) return
@@ -1210,6 +1231,22 @@ module.exports = class Replicator {
     }
   }
 
+  _maybeResolveIfAvailableRanges () {
+    if (this._ifAvailable > 0 || !this._inflight.idle || !this._ranges.length) return
+
+    for (let i = 0; i < this.peers.length; i++) {
+      if (this.peers[i].dataProcessing > 0) return
+    }
+
+    for (let i = 0; i < this._ranges.length; i++) {
+      const r = this._ranges[i]
+
+      if (r.ifAvailable) {
+        this._resolveRangeRequest(r, i--)
+      }
+    }
+  }
+
   _clearRequest (peer, req) {
     if (req.block !== null) {
       this._clearInflightBlock(this._blocks, req)
@@ -1407,6 +1444,7 @@ module.exports = class Replicator {
     while (this._updatePeerNonPrimary(peer) === true);
 
     this._checkUpgradeIfAvailable()
+    this._maybeResolveIfAvailableRanges()
   }
 
   updateAll () {
@@ -1434,6 +1472,7 @@ module.exports = class Replicator {
     }
 
     this._checkUpgradeIfAvailable()
+    this._maybeResolveIfAvailableRanges()
   }
 
   attachTo (protomux) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.44",
+  "version": "10.0.0-alpha.47",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {