hypercore 10.0.0-alpha.54 → 10.0.0-alpha.55

package/README.md

@@ -152,7 +152,7 @@ core.download({ start: 0, end: -1 })
 To downloaded a discrete range of blocks pass a list of indices.
 
 ```js
-core.download({ blocks: [4, 9, 7] });
+core.download({ blocks: [4, 9, 7] })
 ```
 
 To cancel downloading a range simply destroy the range instance.

package/index.js

@@ -527,6 +527,18 @@ module.exports = class Hypercore extends EventEmitter {
           s.emit('append')
         }
       }
+
+      const contig = this.core.header.contiguousLength
+
+      // When the contig length catches up, broadcast the non-sparse length to peers
+      if (appendedNonSparse && contig === this.core.tree.length) {
+        for (const peer of this.peers) {
+          if (peer.broadcastedNonSparse) continue
+
+          peer.broadcastRange(0, contig)
+          peer.broadcastedNonSparse = true
+        }
+      }
     }
 
     if (bitfield) {
@@ -662,29 +674,25 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   async _get (index, opts) {
-    let req
+    let block
 
     if (this.core.bitfield.get(index)) {
-      req = this.core.blocks.get(index)
+      block = this.core.blocks.get(index)
 
-      if (this.cache) this.cache.set(index, req)
+      if (this.cache) this.cache.set(index, block)
     } else {
       if (opts && opts.wait === false) return null
       if (opts && opts.onwait) opts.onwait(index, this)
-      else if (this.onwait) this.onwait(index, this)
+      if (this.onwait) this.onwait(index, this)
 
       const activeRequests = (opts && opts.activeRequests) || this.activeRequests
 
-      req = this._cacheOnResolve(
-        index,
-        this.replicator
-          .addBlock(activeRequests, index)
-          .promise,
-        this.core.tree.fork
-      )
+      const req = this.replicator.addBlock(activeRequests, index)
+
+      block = this._cacheOnResolve(index, req.promise, this.core.tree.fork)
     }
 
-    return req
+    return block
   }
 
   async _cacheOnResolve (index, req, fork) {
@@ -725,7 +733,9 @@ module.exports = class Hypercore extends EventEmitter {
 
   async _download (range) {
     if (this.opened === false) await this.opening
+
     const activeRequests = (range && range.activeRequests) || this.activeRequests
+
     return this.replicator.addRange(activeRequests, range)
   }
 

package/lib/bitfield.js

@@ -1,7 +1,6 @@
-// TODO: needs massive improvements obvs
-
 const BigSparseArray = require('big-sparse-array')
 const b4a = require('b4a')
+const bits = require('bits-to-bytes')
 
 class FixedBitfield {
   constructor (index, bitfield) {
@@ -11,26 +10,17 @@ class FixedBitfield {
   }
 
   get (index) {
-    const j = index & 31
-    const i = (index - j) / 32
-
-    return i < this.bitfield.length && (this.bitfield[i] & (1 << j)) !== 0
+    return bits.get(this.bitfield, index)
   }
 
   set (index, val) {
-    const j = index & 31
-    const i = (index - j) / 32
-    const v = this.bitfield[i]
-
-    if (val === ((v & (1 << j)) !== 0)) return false
-
-    const u = val
-      ? v | (1 << j)
-      : v ^ (1 << j)
-
-    if (u === v) return false
+    return bits.set(this.bitfield, index, val)
+  }
 
-    this.bitfield[i] = u
+  setRange (start, length, val) {
+    // Using fill instead of setRange is ~2 orders of magnitude faster, but does
+    // have the downside of not being able to tell if any bits actually changed.
+    bits.fill(this.bitfield, val, start, start + length)
     return true
   }
 }
@@ -44,7 +34,11 @@ module.exports = class Bitfield {
     this.resumed = !!(buf && buf.byteLength >= 4)
 
     const all = this.resumed
-      ? new Uint32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4))
+      ? new Uint32Array(
+          buf.buffer,
+          buf.byteOffset,
+          Math.floor(buf.byteLength / 4)
+        )
       : new Uint32Array(1024)
 
     for (let i = 0; i < all.length; i += 1024) {
@@ -55,40 +49,53 @@ module.exports = class Bitfield {
   }
 
   get (index) {
-    const j = index & 32767
-    const i = (index - j) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i)
 
     return p ? p.get(j) : false
   }
 
   set (index, val) {
-    const j = index & 32767
-    const i = (index - j) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
 
     let p = this.pages.get(i)
 
-    if (!p) {
-      if (!val) return
+    if (!p && val) {
       p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
     }
 
-    if (!p.set(j, val) || p.dirty) return
-
-    p.dirty = true
-    this.unflushed.push(p)
+    if (p && p.set(j, val) && !p.dirty) {
+      p.dirty = true
+      this.unflushed.push(p)
+    }
   }
 
   setRange (start, length, val) {
-    for (let i = 0; i < length; i++) {
-      this.set(start + i, val)
-    }
-  }
+    let j = start & (this.pageSize - 1)
+    let i = (start - j) / this.pageSize
 
-  // Should prob be removed, when/if we re-add compression
-  page (i) {
-    const p = this.pages.get(i)
-    return p ? p.bitfield : new Uint32Array(1024)
+    while (length > 0) {
+      let p = this.pages.get(i)
+
+      if (!p && val) {
+        p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
+      }
+
+      const end = Math.min(j + length, this.pageSize)
+      const range = end - j
+
+      if (p && p.setRange(j, range, val) && !p.dirty) {
+        p.dirty = true
+        this.unflushed.push(p)
+      }
+
+      j = 0
+      i++
+      length -= range
+    }
   }
 
   clear () {
@@ -120,7 +127,12 @@ module.exports = class Bitfield {
       let error = null
 
       for (const page of this.unflushed) {
-        const buf = b4a.from(page.bitfield.buffer, page.bitfield.byteOffset, page.bitfield.byteLength)
+        const buf = b4a.from(
+          page.bitfield.buffer,
+          page.bitfield.byteOffset,
+          page.bitfield.byteLength
+        )
+
         page.dirty = false
         this.storage.write(page.index * 4096, buf, done)
       }
@@ -151,7 +163,7 @@ module.exports = class Bitfield {
 }
 
 function ensureSize (uint32, size) {
-  if (uint32.length === size) return uint32
+  if (uint32.byteLength === size) return uint32
   const a = new Uint32Array(1024)
   a.set(uint32, 0)
   return a

package/lib/remote-bitfield.js

@@ -1,24 +1,45 @@
 const BigSparseArray = require('big-sparse-array')
+const bits = require('bits-to-bytes')
 
 module.exports = class RemoteBitfield {
   constructor () {
+    this.pageSize = 32768
     this.pages = new BigSparseArray()
   }
 
   get (index) {
-    const r = index & 32767
-    const i = (index - r) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i)
 
-    return p ? (p[r >>> 5] & (1 << (r & 31))) !== 0 : false
+    return p ? bits.get(p, j) : false
   }
 
   set (index, val) {
-    const r = index & 32767
-    const i = (index - r) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i) || this.pages.set(i, new Uint32Array(1024))
 
-    if (val) p[r >>> 5] |= (1 << (r & 31))
-    else p[r >>> 5] &= ~(1 << (r & 31))
+    bits.set(p, j, val)
+  }
+
+  setRange (start, length, val) {
+    let j = start & (this.pageSize - 1)
+    let i = (start - j) / this.pageSize
+
+    while (length > 0) {
+      const p = this.pages.get(i) || this.pages.set(i, new Uint32Array(1024))
+
+      const end = Math.min(j + length, this.pageSize)
+      const range = end - j
+
+      bits.fill(p, val, j, end)
+
+      j = 0
+      i++
+      length -= range
+    }
   }
 }

package/lib/replicator.js

@@ -7,6 +7,7 @@ const m = require('./messages')
 const caps = require('./caps')
 
 const DEFAULT_MAX_INFLIGHT = 32
+const DEFAULT_SEGMENT_SIZE = 128 * 1024 // 128 KiB
 
 class Attachable {
   constructor () {
@@ -286,6 +287,9 @@ class Peer {
     this.remoteDownloading = true
     this.remoteSynced = false
 
+    this.segmentsWanted = new Set()
+    this.broadcastedNonSparse = false
+
     this.lengthAcked = 0
 
     this.extensions = new Map()
@@ -356,13 +360,13 @@ class Peer {
 
     this.sendSync()
 
-    const p = pages(this.core)
+    const contig = this.core.header.contiguousLength
+    if (contig > 0) {
+      this.broadcastRange(0, contig, false)
 
-    for (let index = 0; index < p.length; index++) {
-      this.wireBitfield.send({
-        start: index * this.core.bitfield.pageSize,
-        bitfield: p[index]
-      })
+      if (contig === this.core.tree.length) {
+        this.broadcastedNonSparse = true
+      }
     }
 
     this.replicator._ifAvailable--
@@ -552,8 +556,8 @@ class Peer {
     this.replicator._onnodata(this, req)
   }
 
-  onwant () {
-    // TODO
+  onwant ({ start, length }) {
+    this.replicator._onwant(this, start, length)
   }
 
   onunwant () {
@@ -577,9 +581,7 @@ class Peer {
   onrange ({ drop, start, length }) {
     const has = drop === false
 
-    for (const end = start + length; start < end; start++) {
-      this.remoteBitfield.set(start, has)
-    }
+    this.remoteBitfield.setRange(start, length, has)
 
     if (drop === false) this._update()
   }
@@ -674,6 +676,7 @@ class Peer {
       return true
     }
 
+    this._maybeWant(s.seeker.start, len)
     return false
   }
 
@@ -685,7 +688,10 @@ class Peer {
   _requestBlock (b) {
     const { length, fork } = this.core.tree
 
-    if (this.remoteBitfield.get(b.index) === false || fork !== this.remoteFork) return false
+    if (this.remoteBitfield.get(b.index) === false || fork !== this.remoteFork) {
+      this._maybeWant(b.index)
+      return false
+    }
 
     const req = this._makeRequest(b.index >= length)
     if (req === null) return false
@@ -743,6 +749,7 @@ class Peer {
       return true
     }
 
+    this._maybeWant(r.start, len)
     return false
   }
 
@@ -780,9 +787,25 @@ class Peer {
       return true
     }
 
+    this._maybeWant(f.batch.want.start, len)
     return false
   }
 
+  _maybeWant (start, length = 1) {
+    let i = Math.floor(start / DEFAULT_SEGMENT_SIZE)
+    const n = Math.ceil((start + length) / DEFAULT_SEGMENT_SIZE)
+
+    for (; i < n; i++) {
+      if (this.segmentsWanted.has(i)) continue
+      this.segmentsWanted.add(i)
+
+      this.wireWant.send({
+        start: i * DEFAULT_SEGMENT_SIZE,
+        length: DEFAULT_SEGMENT_SIZE
+      })
+    }
+  }
+
   async _send (req) {
     const fork = this.core.tree.fork
 
@@ -1299,6 +1322,28 @@ module.exports = class Replicator {
     else this.updatePeer(peer)
   }
 
+  _onwant (peer, start, length) {
+    length = Math.min(length, this.core.tree.length - start)
+
+    peer.protomux.cork()
+
+    let i = Math.floor(start / this.core.bitfield.pageSize)
+    const n = Math.ceil((start + length) / this.core.bitfield.pageSize)
+
+    for (; i < n; i++) {
+      const p = this.core.bitfield.pages.get(i)
+
+      if (p) {
+        peer.wireBitfield.send({
+          start: i * this.core.bitfield.pageSize,
+          bitfield: p.bitfield
+        })
+      }
+    }
+
+    peer.protomux.uncork()
+  }
+
   async _onreorgdata (peer, req, data) {
     const f = this._addReorg(data.fork, peer)
 
@@ -1525,17 +1570,6 @@ module.exports = class Replicator {
   }
 }
 
-function pages (core) {
-  const res = []
-
-  for (let i = 0; i < core.tree.length; i += core.bitfield.pageSize) {
-    const p = core.bitfield.page(i / core.bitfield.pageSize)
-    res.push(p)
-  }
-
-  return res
-}
-
 function matchingRequest (req, data) {
   if (data.block !== null && (req.block === null || req.block.index !== data.block.index)) return false
   if (data.hash !== null && (req.hash === null || req.hash.index !== data.hash.index)) return false

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.54",
+  "version": "10.0.0-alpha.55",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -35,6 +35,7 @@
     "@hyperswarm/secret-stream": "^6.0.0",
     "b4a": "^1.1.0",
     "big-sparse-array": "^1.0.2",
+    "bits-to-bytes": "^1.2.0",
     "codecs": "^3.0.0",
     "compact-encoding": "^2.5.0",
     "crc32-universal": "^1.0.1",