hypercore 10.0.0-alpha.45 → 10.0.0-alpha.48

package/README.md

@@ -66,7 +66,8 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
   valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
   encodeBatch: batch => { ... }, // optionally apply an encoding to complete batches
   keyPair: kp, // optionally pass the public key and secret key as a key pair
-  encryptionKey: k // optionally pass an encryption key to enable block encryption
+  encryptionKey: k, // optionally pass an encryption key to enable block encryption
+  onwait: () => {} // hook that is called if gets are waiting for download
 }
 ```
 

package/index.js

@@ -70,6 +70,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.sessions = opts._sessions || [this]
     this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
+    this.onwait = opts.onwait || null
 
     this.closing = null
     this.opening = this._openSession(key, storage, opts)
@@ -113,6 +114,7 @@ module.exports = class Hypercore extends EventEmitter {
       indent + '  opened: ' + opts.stylize(this.opened, 'boolean') + '\n' +
       indent + '  closed: ' + opts.stylize(this.closed, 'boolean') + '\n' +
       indent + '  snapshotted: ' + opts.stylize(this.snapshotted, 'boolean') + '\n' +
+      indent + '  sparse: ' + opts.stylize(this.sparse, 'boolean') + '\n' +
       indent + '  writable: ' + opts.stylize(this.writable, 'boolean') + '\n' +
       indent + '  length: ' + opts.stylize(this.length, 'number') + '\n' +
       indent + '  byteLength: ' + opts.stylize(this.byteLength, 'number') + '\n' +
@@ -182,9 +184,11 @@ module.exports = class Hypercore extends EventEmitter {
       throw SESSION_CLOSED('Cannot make sessions on a closing core')
     }
 
+    const sparse = opts.sparse === false ? false : this.sparse
     const Clz = opts.class || Hypercore
     const s = new Clz(this.storage, this.key, {
       ...opts,
+      sparse,
       _opening: this.opening,
       _sessions: this.sessions
     })
@@ -324,14 +328,27 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
-  _updateSnapshot () {
-    const prev = this._snapshot
-    const next = this._snapshot = {
-      length: this.core.tree.length,
-      byteLength: this.core.tree.byteLength,
+  _getSnapshot () {
+    if (this.sparse) {
+      return {
+        length: this.core.tree.length,
+        byteLength: this.core.tree.byteLength,
+        fork: this.core.tree.fork,
+        compatLength: this.core.tree.length
+      }
+    }
+
+    return {
+      length: this.core.header.contiguousLength,
+      byteLength: 0,
       fork: this.core.tree.fork,
-      compatLength: this.core.tree.length
+      compatLength: this.core.header.contiguousLength
     }
+  }
+
+  _updateSnapshot () {
+    const prev = this._snapshot
+    const next = this._snapshot = this._getSnapshot()
 
     if (!prev) return true
     return prev.length !== next.length || prev.fork !== next.fork
@@ -573,10 +590,22 @@ module.exports = class Hypercore extends EventEmitter {
     const activeRequests = (opts && opts.activeRequests) || this.activeRequests
     const req = this.replicator.addUpgrade(activeRequests)
 
-    if (!this.snapshotted) return req.promise
-    if (!(await req.promise)) return false
+    let upgraded = await req.promise
+
+    if (!this.sparse) {
+      // Download all available blocks in non-sparse mode
+      const start = this.length
+      const end = this.core.tree.length
+      const contig = this.contiguousLength
+
+      await this.download({ start, end, ifAvailable: true }).downloaded()
+
+      if (!upgraded) upgraded = this.contiguousLength !== contig
+    }
 
-    return this._updateSnapshot()
+    if (!upgraded) return false
+    if (this.snapshotted) return this._updateSnapshot()
+    return true
   }
 
   async seek (bytes, opts) {
@@ -634,6 +663,7 @@ module.exports = class Hypercore extends EventEmitter {
     } else {
       if (opts && opts.wait === false) return null
       if (opts && opts.onwait) opts.onwait(index)
+      else if (this.onwait) this.onwait(index)
 
       const activeRequests = (opts && opts.activeRequests) || this.activeRequests
 

package/lib/caps.js

@@ -4,9 +4,7 @@ const b4a = require('b4a')
 const c = require('compact-encoding')
 
 // TODO: rename this to "crypto" and move everything hashing related etc in here
-// Also lets move the tree stuff from hypercore-crypto here, and loose the types
-// from the hashes there - they are not needed since we lock the indexes in the tree
-// hash and just makes alignment etc harder in other languages
+// Also lets move the tree stuff from hypercore-crypto here
 
 const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER] = crypto.namespace('hypercore', 3)
 

package/lib/replicator.js

@@ -102,12 +102,13 @@ class BlockRequest extends Attachable {
 }
 
 class RangeRequest extends Attachable {
-  constructor (ranges, start, end, linear, blocks) {
+  constructor (ranges, start, end, linear, ifAvailable, blocks) {
     super()
 
     this.start = start
     this.end = end
     this.linear = linear
+    this.ifAvailable = ifAvailable
     this.blocks = blocks
     this.ranges = ranges
 
@@ -172,6 +173,10 @@ class InflightTracker {
     this._free = []
   }
 
+  get idle () {
+    return this._requests.length === this._free.length
+  }
+
   * [Symbol.iterator] () {
     for (const req of this._requests) {
       if (req !== null) yield req
@@ -260,6 +265,7 @@ class Peer {
 
     this.inflight = 0
     this.maxInflight = DEFAULT_MAX_INFLIGHT
+    this.dataProcessing = 0
 
     this.canUpgrade = true
 
@@ -515,6 +521,8 @@ class Peer {
 
     if (reorg === true) return this.replicator._onreorgdata(this, req, data)
 
+    this.dataProcessing++
+
     try {
       if (!matchingRequest(req, data) || !(await this.core.verify(data, this))) {
         this.replicator._onnodata(this, req)
@@ -523,6 +531,8 @@ class Peer {
     } catch (err) {
       this.replicator._onnodata(this, req)
       throw err
+    } finally {
+      this.dataProcessing--
     }
 
     this.replicator._ondata(this, req, data)
@@ -909,13 +919,21 @@ module.exports = class Replicator {
     return ref
   }
 
-  addRange (session, { start = 0, end = -1, length = toLength(start, end), blocks = null, linear = false } = {}) {
+  addRange (session, { start = 0, end = -1, length = toLength(start, end), blocks = null, linear = false, ifAvailable = false } = {}) {
     if (blocks !== null) { // if using blocks, start, end just acts as frames around the blocks array
       start = 0
       end = length = blocks.length
     }
 
-    const r = new RangeRequest(this._ranges, start, length === -1 ? -1 : start + length, linear, blocks)
+    const r = new RangeRequest(
+      this._ranges,
+      start,
+      length === -1 ? -1 : start + length,
+      linear,
+      ifAvailable,
+      blocks
+    )
+
     const ref = r.attach(session)
 
     this._ranges.push(r)
@@ -1115,6 +1133,14 @@ module.exports = class Replicator {
     return true
   }
 
+  _resolveRangeRequest (req, index) {
+    const head = this._ranges.pop()
+
+    if (index < this._ranges.length) this._ranges[index] = head
+
+    req.resolve(true)
+  }
+
   _clearInflightBlock (tracker, req) {
     const isBlock = tracker === this._blocks
     const index = isBlock === true ? req.block.index : req.hash.index / 2
@@ -1168,14 +1194,9 @@ module.exports = class Replicator {
         while (r.start < r.end && this.core.bitfield.get(mapIndex(r.blocks, r.start)) === true) r.start++
         while (r.start < r.end && this.core.bitfield.get(mapIndex(r.blocks, r.end - 1)) === true) r.end--
 
-        if (r.end === -1 || r.start < r.end) continue
-
-        if (i < this._ranges.length - 1) this._ranges[i] = this._ranges.pop()
-        else this._ranges.pop()
-
-        i--
-
-        r.resolve(true)
+        if (r.end !== -1 && r.start >= r.end) {
+          this._resolveRangeRequest(r, i--)
+        }
       }
 
       for (let i = 0; i < this._seeks.length; i++) {
@@ -1201,7 +1222,7 @@ module.exports = class Replicator {
         else s.resolve(res)
       }
 
-      this.updateAll()
+      if (this._inflight.idle) this.updateAll()
 
       // No additional updates scheduled - return
       if (--this._updatesPending === 0) return
@@ -1210,6 +1231,22 @@ module.exports = class Replicator {
     }
   }
 
+  _maybeResolveIfAvailableRanges () {
+    if (this._ifAvailable > 0 || !this._inflight.idle || !this._ranges.length) return
+
+    for (let i = 0; i < this.peers.length; i++) {
+      if (this.peers[i].dataProcessing > 0) return
+    }
+
+    for (let i = 0; i < this._ranges.length; i++) {
+      const r = this._ranges[i]
+
+      if (r.ifAvailable) {
+        this._resolveRangeRequest(r, i--)
+      }
+    }
+  }
+
   _clearRequest (peer, req) {
     if (req.block !== null) {
       this._clearInflightBlock(this._blocks, req)
@@ -1407,6 +1444,7 @@ module.exports = class Replicator {
     while (this._updatePeerNonPrimary(peer) === true);
 
     this._checkUpgradeIfAvailable()
+    this._maybeResolveIfAvailableRanges()
   }
 
   updateAll () {
@@ -1434,6 +1472,7 @@ module.exports = class Replicator {
     }
 
     this._checkUpgradeIfAvailable()
+    this._maybeResolveIfAvailableRanges()
   }
 
   attachTo (protomux) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.45",
+  "version": "10.0.0-alpha.48",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {