hypercore 10.0.0-alpha.53 → 10.0.0-alpha.56

package/README.md

@@ -152,7 +152,7 @@ core.download({ start: 0, end: -1 })
 To downloaded a discrete range of blocks pass a list of indices.
 
 ```js
-core.download({ blocks: [4, 9, 7] });
+core.download({ blocks: [4, 9, 7] })
 ```
 
 To cancel downloading a range simply destroy the range instance.

package/index.js

@@ -69,6 +69,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
     this.onwait = opts.onwait || null
+    this.wait = opts.wait !== false
 
     this.closing = null
     this.opening = this._openSession(key, storage, opts)
@@ -194,11 +195,13 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     const sparse = opts.sparse === false ? false : this.sparse
+    const wait = opts.wait === false ? false : this.wait
     const onwait = opts.onwait === undefined ? this.onwait : opts.onwait
     const Clz = opts.class || Hypercore
     const s = new Clz(this.storage, this.key, {
       ...opts,
       sparse,
+      wait,
       onwait,
       _opening: this.opening,
       _sessions: this.sessions
@@ -312,8 +315,7 @@ module.exports = class Hypercore extends EventEmitter {
       crypto: this.crypto,
       legacy: opts.legacy,
       auth: opts.auth,
-      onupdate: this._oncoreupdate.bind(this),
-      oncontigupdate: this._oncorecontigupdate.bind(this)
+      onupdate: this._oncoreupdate.bind(this)
     })
 
     if (opts.userData) {
@@ -496,30 +498,50 @@ module.exports = class Hypercore extends EventEmitter {
 
   _oncoreupdate (status, bitfield, value, from) {
     if (status !== 0) {
-      const truncated = (status & 0b10) !== 0
-      const appended = (status & 0b01) !== 0
+      const truncatedNonSparse = (status & 0b1000) !== 0
+      const appendedNonSparse = (status & 0b0100) !== 0
+      const truncated = (status & 0b0010) !== 0
+      const appended = (status & 0b0001) !== 0
 
       if (truncated) {
         this.replicator.ontruncate(bitfield.start)
       }
 
+      if ((status & 0b0011) !== 0) {
+        this.replicator.onupgrade()
+      }
+
       for (let i = 0; i < this.sessions.length; i++) {
         const s = this.sessions[i]
 
         if (truncated) {
           if (s.cache) s.cache.clear()
-          s.emit('truncate', bitfield.start, this.core.tree.fork)
+
           // If snapshotted, make sure to update our compat so we can fail gets
           if (s._snapshot && bitfield.start < s._snapshot.compatLength) s._snapshot.compatLength = bitfield.start
         }
 
-        // For sparse sessions, immediately emit appends. Non-sparse sessions
-        // are handled separately and only emit appends when their contiguous
-        // length is updated.
-        if (appended && s.sparse) s.emit('append')
+        if (s.sparse ? truncated : truncatedNonSparse) {
+          s.emit('truncate', bitfield.start, this.core.tree.fork)
+        }
+
+        // For sparse sessions, immediately emit appends. If non-sparse, emit if contig length has updated
+        if (s.sparse ? appended : appendedNonSparse) {
+          s.emit('append')
+        }
       }
 
-      this.replicator.onupgrade()
+      const contig = this.core.header.contiguousLength
+
+      // When the contig length catches up, broadcast the non-sparse length to peers
+      if (appendedNonSparse && contig === this.core.tree.length) {
+        for (const peer of this.peers) {
+          if (peer.broadcastedNonSparse) continue
+
+          peer.broadcastRange(0, contig)
+          peer.broadcastedNonSparse = true
+        }
+      }
     }
 
     if (bitfield) {
@@ -535,16 +557,6 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
-  _oncorecontigupdate () {
-    // For non-sparse sessions, emit appends only when the contiguous length is
-    // updated.
-    for (let i = 0; i < this.sessions.length; i++) {
-      const s = this.sessions[i]
-
-      if (!s.sparse) s.emit('append')
-    }
-  }
-
   _onpeerupdate (added, peer) {
     const name = added ? 'peer-add' : 'peer-remove'
 
@@ -665,29 +677,26 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   async _get (index, opts) {
-    let req
+    let block
 
     if (this.core.bitfield.get(index)) {
-      req = this.core.blocks.get(index)
+      block = this.core.blocks.get(index)
 
-      if (this.cache) this.cache.set(index, req)
+      if (this.cache) this.cache.set(index, block)
     } else {
       if (opts && opts.wait === false) return null
+      if (this.wait === false && (!opts || !opts.wait)) return null
       if (opts && opts.onwait) opts.onwait(index, this)
-      else if (this.onwait) this.onwait(index, this)
+      if (this.onwait) this.onwait(index, this)
 
       const activeRequests = (opts && opts.activeRequests) || this.activeRequests
 
-      req = this._cacheOnResolve(
-        index,
-        this.replicator
-          .addBlock(activeRequests, index)
-          .promise,
-        this.core.tree.fork
-      )
+      const req = this.replicator.addBlock(activeRequests, index)
+
+      block = this._cacheOnResolve(index, req.promise, this.core.tree.fork)
     }
 
-    return req
+    return block
   }
 
   async _cacheOnResolve (index, req, fork) {
@@ -728,7 +737,9 @@ module.exports = class Hypercore extends EventEmitter {
 
   async _download (range) {
     if (this.opened === false) await this.opening
+
     const activeRequests = (range && range.activeRequests) || this.activeRequests
+
     return this.replicator.addRange(activeRequests, range)
   }
 

package/lib/bitfield.js

@@ -1,7 +1,6 @@
-// TODO: needs massive improvements obvs
-
 const BigSparseArray = require('big-sparse-array')
 const b4a = require('b4a')
+const bits = require('bits-to-bytes')
 
 class FixedBitfield {
   constructor (index, bitfield) {
@@ -11,26 +10,17 @@ class FixedBitfield {
   }
 
   get (index) {
-    const j = index & 31
-    const i = (index - j) / 32
-
-    return i < this.bitfield.length && (this.bitfield[i] & (1 << j)) !== 0
+    return bits.get(this.bitfield, index)
   }
 
   set (index, val) {
-    const j = index & 31
-    const i = (index - j) / 32
-    const v = this.bitfield[i]
-
-    if (val === ((v & (1 << j)) !== 0)) return false
-
-    const u = val
-      ? v | (1 << j)
-      : v ^ (1 << j)
-
-    if (u === v) return false
+    return bits.set(this.bitfield, index, val)
+  }
 
-    this.bitfield[i] = u
+  setRange (start, length, val) {
+    // Using fill instead of setRange is ~2 orders of magnitude faster, but does
+    // have the downside of not being able to tell if any bits actually changed.
+    bits.fill(this.bitfield, val, start, start + length)
     return true
   }
 }
@@ -44,7 +34,11 @@ module.exports = class Bitfield {
     this.resumed = !!(buf && buf.byteLength >= 4)
 
     const all = this.resumed
-      ? new Uint32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4))
+      ? new Uint32Array(
+          buf.buffer,
+          buf.byteOffset,
+          Math.floor(buf.byteLength / 4)
+        )
       : new Uint32Array(1024)
 
     for (let i = 0; i < all.length; i += 1024) {
@@ -55,40 +49,53 @@ module.exports = class Bitfield {
   }
 
   get (index) {
-    const j = index & 32767
-    const i = (index - j) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i)
 
     return p ? p.get(j) : false
   }
 
   set (index, val) {
-    const j = index & 32767
-    const i = (index - j) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
 
     let p = this.pages.get(i)
 
-    if (!p) {
-      if (!val) return
+    if (!p && val) {
       p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
     }
 
-    if (!p.set(j, val) || p.dirty) return
-
-    p.dirty = true
-    this.unflushed.push(p)
+    if (p && p.set(j, val) && !p.dirty) {
+      p.dirty = true
+      this.unflushed.push(p)
+    }
   }
 
   setRange (start, length, val) {
-    for (let i = 0; i < length; i++) {
-      this.set(start + i, val)
-    }
-  }
+    let j = start & (this.pageSize - 1)
+    let i = (start - j) / this.pageSize
 
-  // Should prob be removed, when/if we re-add compression
-  page (i) {
-    const p = this.pages.get(i)
-    return p ? p.bitfield : new Uint32Array(1024)
+    while (length > 0) {
+      let p = this.pages.get(i)
+
+      if (!p && val) {
+        p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
+      }
+
+      const end = Math.min(j + length, this.pageSize)
+      const range = end - j
+
+      if (p && p.setRange(j, range, val) && !p.dirty) {
+        p.dirty = true
+        this.unflushed.push(p)
+      }
+
+      j = 0
+      i++
+      length -= range
+    }
   }
 
   clear () {
@@ -120,7 +127,12 @@ module.exports = class Bitfield {
       let error = null
 
       for (const page of this.unflushed) {
-        const buf = b4a.from(page.bitfield.buffer, page.bitfield.byteOffset, page.bitfield.byteLength)
+        const buf = b4a.from(
+          page.bitfield.buffer,
+          page.bitfield.byteOffset,
+          page.bitfield.byteLength
+        )
+
         page.dirty = false
         this.storage.write(page.index * 4096, buf, done)
       }
@@ -151,7 +163,7 @@ module.exports = class Bitfield {
 }
 
 function ensureSize (uint32, size) {
-  if (uint32.length === size) return uint32
+  if (uint32.byteLength === size) return uint32
   const a = new Uint32Array(1024)
   a.set(uint32, 0)
   return a

package/lib/core.js

@@ -9,9 +9,8 @@ const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = req
 const m = require('./messages')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate, oncontigupdate) {
+  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate) {
     this.onupdate = onupdate
-    this.oncontigupdate = oncontigupdate
     this.header = header
     this.crypto = crypto
     this.oplog = oplog
@@ -27,8 +26,6 @@ module.exports = class Core {
     this._verifiesFlushed = null
     this._mutex = new Mutex()
     this._legacy = legacy
-
-    this._updateContiguousLength(header.contiguousLength)
   }
 
   static async open (storage, opts = {}) {
@@ -134,6 +131,11 @@ module.exports = class Core {
       await bitfield.clear()
     }
 
+    // compat from earlier version that do not store contig length
+    if (header.contiguousLength === 0) {
+      while (bitfield.get(header.contiguousLength)) header.contiguousLength++
+    }
+
     const auth = opts.auth || this.createAuth(crypto, header.signer)
 
     for (const e of entries) {
@@ -149,6 +151,7 @@ module.exports = class Core {
 
       if (e.bitfield) {
         bitfield.setRange(e.bitfield.start, e.bitfield.length, !e.bitfield.drop)
+        updateContig(header, e.bitfield, bitfield)
       }
 
       if (e.treeUpgrade) {
@@ -165,7 +168,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop, opts.oncontigupdate || noop)
+    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop)
   }
 
   _shouldFlush () {
@@ -196,17 +199,6 @@ module.exports = class Core {
     await this.blocks.put(index, value, byteOffset)
   }
 
-  _updateContiguousLength (index, length = 0) {
-    if (index === this.header.contiguousLength) {
-      let i = this.header.contiguousLength + length
-
-      while (this.bitfield.get(i)) i++
-
-      this.header.contiguousLength = i
-      this.oncontigupdate()
-    }
-  }
-
   async userData (key, value) {
     // TODO: each oplog append can set user data, so we should have a way
     // to just hitch a ride on one of the other ongoing appends?
@@ -286,12 +278,12 @@ module.exports = class Core {
       this.bitfield.setRange(batch.ancestors, batch.length - batch.ancestors, true)
       batch.commit()
 
-      this.header.contiguousLength = batch.length
-      this.oncontigupdate()
       this.header.tree.length = batch.length
       this.header.tree.rootHash = hash
       this.header.tree.signature = batch.signature
-      this.onupdate(0b01, entry.bitfield, null, null)
+
+      const status = 0b0001 | updateContig(this.header, entry.bitfield, this.bitfield)
+      this.onupdate(status, entry.bitfield, null, null)
 
       if (this._shouldFlush()) await this._flushOplog()
 
@@ -329,9 +321,11 @@ module.exports = class Core {
 
       await this.oplog.append([entry], false)
 
+      let status = 0b0001
+
       if (bitfield) {
         this.bitfield.set(bitfield.start, true)
-        this._updateContiguousLength(bitfield.start, bitfield.length)
+        status |= updateContig(this.header, bitfield, this.bitfield)
       }
 
       batch.commit()
@@ -340,7 +334,8 @@ module.exports = class Core {
       this.header.tree.length = batch.length
       this.header.tree.rootHash = batch.rootHash
       this.header.tree.signature = batch.signature
-      this.onupdate(0b01, bitfield, value, from)
+
+      this.onupdate(status, bitfield, value, from)
 
       if (this._shouldFlush()) await this._flushOplog()
     } finally {
@@ -387,14 +382,16 @@ module.exports = class Core {
           continue
         }
 
+        let status = 0
+
         if (bitfield) {
           this.bitfield.set(bitfield.start, true)
-          this._updateContiguousLength(bitfield.start, bitfield.length)
+          status = updateContig(this.header, bitfield, this.bitfield)
         }
 
         batch.commit()
 
-        this.onupdate(0, bitfield, value, from)
+        this.onupdate(status, bitfield, value, from)
       }
 
       if (this._shouldFlush()) await this._flushOplog()
@@ -472,15 +469,15 @@ module.exports = class Core {
     addReorgHint(this.header.hints.reorgs, this.tree, batch)
     batch.commit()
 
-    const appended = batch.length > batch.ancestors
+    const contigStatus = updateContig(this.header, entry.bitfield, this.bitfield)
+    const status = ((batch.length > batch.ancestors) ? 0b0011 : 0b0010) | contigStatus
 
-    this.header.contiguousLength = Math.min(batch.ancestors, this.header.contiguousLength)
-    this.oncontigupdate()
     this.header.tree.fork = batch.fork
     this.header.tree.length = batch.length
     this.header.tree.rootHash = batch.hash()
     this.header.tree.signature = batch.signature
-    this.onupdate(appended ? 0b11 : 0b10, entry.bitfield, null, from)
+
+    this.onupdate(status, entry.bitfield, null, from)
 
     // TODO: there is a bug in the merkle tree atm where it cannot handle unflushed
     // truncates if we append or download anything after the truncation point later on
@@ -501,6 +498,36 @@ module.exports = class Core {
   }
 }
 
+function updateContig (header, upd, bitfield) {
+  const end = upd.start + upd.length
+
+  let c = header.contiguousLength
+
+  if (upd.drop) {
+    // If we dropped a block in the current contig range, "downgrade" it
+    if (c <= end && c > upd.start) {
+      c = upd.start
+    }
+  } else {
+    if (c <= end && c >= upd.start) {
+      c = end
+      while (bitfield.get(c)) c++
+    }
+  }
+
+  if (c === header.contiguousLength) {
+    return 0b0000
+  }
+
+  if (c > header.contiguousLength) {
+    header.contiguousLength = c
+    return 0b0100
+  }
+
+  header.contiguousLength = c
+  return 0b1000
+}
+
 function addReorgHint (list, tree, batch) {
   if (tree.length === 0 || tree.fork === batch.fork) return
 

package/lib/remote-bitfield.js

@@ -1,24 +1,45 @@
 const BigSparseArray = require('big-sparse-array')
+const bits = require('bits-to-bytes')
 
 module.exports = class RemoteBitfield {
   constructor () {
+    this.pageSize = 32768
     this.pages = new BigSparseArray()
   }
 
   get (index) {
-    const r = index & 32767
-    const i = (index - r) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i)
 
-    return p ? (p[r >>> 5] & (1 << (r & 31))) !== 0 : false
+    return p ? bits.get(p, j) : false
   }
 
   set (index, val) {
-    const r = index & 32767
-    const i = (index - r) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i) || this.pages.set(i, new Uint32Array(1024))
 
-    if (val) p[r >>> 5] |= (1 << (r & 31))
-    else p[r >>> 5] &= ~(1 << (r & 31))
+    bits.set(p, j, val)
+  }
+
+  setRange (start, length, val) {
+    let j = start & (this.pageSize - 1)
+    let i = (start - j) / this.pageSize
+
+    while (length > 0) {
+      const p = this.pages.get(i) || this.pages.set(i, new Uint32Array(1024))
+
+      const end = Math.min(j + length, this.pageSize)
+      const range = end - j
+
+      bits.fill(p, val, j, end)
+
+      j = 0
+      i++
+      length -= range
+    }
   }
 }

package/lib/replicator.js

@@ -7,6 +7,7 @@ const m = require('./messages')
 const caps = require('./caps')
 
 const DEFAULT_MAX_INFLIGHT = 32
+const DEFAULT_SEGMENT_SIZE = 128 * 1024 // 128 KiB
 
 class Attachable {
   constructor () {
@@ -286,6 +287,9 @@ class Peer {
     this.remoteDownloading = true
     this.remoteSynced = false
 
+    this.segmentsWanted = new Set()
+    this.broadcastedNonSparse = false
+
     this.lengthAcked = 0
 
     this.extensions = new Map()
@@ -356,13 +360,13 @@ class Peer {
 
     this.sendSync()
 
-    const p = pages(this.core)
+    const contig = this.core.header.contiguousLength
+    if (contig > 0) {
+      this.broadcastRange(0, contig, false)
 
-    for (let index = 0; index < p.length; index++) {
-      this.wireBitfield.send({
-        start: index * this.core.bitfield.pageSize,
-        bitfield: p[index]
-      })
+      if (contig === this.core.tree.length) {
+        this.broadcastedNonSparse = true
+      }
     }
 
     this.replicator._ifAvailable--
@@ -552,8 +556,8 @@ class Peer {
     this.replicator._onnodata(this, req)
   }
 
-  onwant () {
-    // TODO
+  onwant ({ start, length }) {
+    this.replicator._onwant(this, start, length)
   }
 
   onunwant () {
@@ -577,9 +581,7 @@ class Peer {
   onrange ({ drop, start, length }) {
     const has = drop === false
 
-    for (const end = start + length; start < end; start++) {
-      this.remoteBitfield.set(start, has)
-    }
+    this.remoteBitfield.setRange(start, length, has)
 
     if (drop === false) this._update()
   }
@@ -674,6 +676,7 @@ class Peer {
       return true
     }
 
+    this._maybeWant(s.seeker.start, len)
     return false
   }
 
@@ -685,7 +688,10 @@ class Peer {
   _requestBlock (b) {
     const { length, fork } = this.core.tree
 
-    if (this.remoteBitfield.get(b.index) === false || fork !== this.remoteFork) return false
+    if (this.remoteBitfield.get(b.index) === false || fork !== this.remoteFork) {
+      this._maybeWant(b.index)
+      return false
+    }
 
     const req = this._makeRequest(b.index >= length)
     if (req === null) return false
@@ -743,6 +749,7 @@ class Peer {
       return true
     }
 
+    this._maybeWant(r.start, len)
     return false
   }
 
@@ -780,9 +787,25 @@ class Peer {
       return true
     }
 
+    this._maybeWant(f.batch.want.start, len)
     return false
   }
 
+  _maybeWant (start, length = 1) {
+    let i = Math.floor(start / DEFAULT_SEGMENT_SIZE)
+    const n = Math.ceil((start + length) / DEFAULT_SEGMENT_SIZE)
+
+    for (; i < n; i++) {
+      if (this.segmentsWanted.has(i)) continue
+      this.segmentsWanted.add(i)
+
+      this.wireWant.send({
+        start: i * DEFAULT_SEGMENT_SIZE,
+        length: DEFAULT_SEGMENT_SIZE
+      })
+    }
+  }
+
   async _send (req) {
     const fork = this.core.tree.fork
 
@@ -1299,6 +1322,28 @@ module.exports = class Replicator {
     else this.updatePeer(peer)
   }
 
+  _onwant (peer, start, length) {
+    length = Math.min(length, this.core.tree.length - start)
+
+    peer.protomux.cork()
+
+    let i = Math.floor(start / this.core.bitfield.pageSize)
+    const n = Math.ceil((start + length) / this.core.bitfield.pageSize)
+
+    for (; i < n; i++) {
+      const p = this.core.bitfield.pages.get(i)
+
+      if (p) {
+        peer.wireBitfield.send({
+          start: i * this.core.bitfield.pageSize,
+          bitfield: p.bitfield
+        })
+      }
+    }
+
+    peer.protomux.uncork()
+  }
+
   async _onreorgdata (peer, req, data) {
     const f = this._addReorg(data.fork, peer)
 
@@ -1525,17 +1570,6 @@ module.exports = class Replicator {
   }
 }
 
-function pages (core) {
-  const res = []
-
-  for (let i = 0; i < core.tree.length; i += core.bitfield.pageSize) {
-    const p = core.bitfield.page(i / core.bitfield.pageSize)
-    res.push(p)
-  }
-
-  return res
-}
-
 function matchingRequest (req, data) {
   if (data.block !== null && (req.block === null || req.block.index !== data.block.index)) return false
   if (data.hash !== null && (req.hash === null || req.hash.index !== data.hash.index)) return false

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.53",
+  "version": "10.0.0-alpha.56",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -35,6 +35,7 @@
     "@hyperswarm/secret-stream": "^6.0.0",
     "b4a": "^1.1.0",
     "big-sparse-array": "^1.0.2",
+    "bits-to-bytes": "^1.2.0",
     "codecs": "^3.0.0",
     "compact-encoding": "^2.5.0",
     "crc32-universal": "^1.0.1",