hypercore 10.31.5 → 10.31.7

package/index.js

@@ -658,7 +658,7 @@ module.exports = class Hypercore extends EventEmitter {
       const appended = (status & 0b0001) !== 0
 
       if (truncated) {
-        this.replicator.ontruncate(bitfield.start)
+        this.replicator.ontruncate(bitfield.start, bitfield.length)
       }
 
       if ((status & 0b10011) !== 0) {

package/lib/bitfield.js

@@ -40,10 +40,10 @@ class BitfieldPage {
   setRange (start, length, val) {
     quickbit.fill(this.bitfield, val, start, start + length)
 
-    let i = Math.floor(start / 32)
-    const n = i + Math.ceil(length / 32)
+    let i = Math.floor(start / 128)
+    const n = i + Math.ceil(length / 128)
 
-    while (i < n) this.tree.update(this.offset * 8 + i++ * 32)
+    while (i <= n) this.tree.update(this.offset * 8 + i++ * 128)
   }
 
   findFirst (val, position) {
@@ -213,6 +213,14 @@ module.exports = class Bitfield {
     }
   }
 
+  getBitfield (index, length) {
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
+
+    const p = this._pages.get(i)
+    return p || null
+  }
+
   get (index) {
     const j = index & (BITS_PER_PAGE - 1)
     const i = (index - j) / BITS_PER_PAGE

package/lib/compat.js

@@ -3,7 +3,8 @@
 let quickbit = require('quickbit-universal')
 if (
   typeof quickbit.findFirst !== 'function' ||
-  typeof quickbit.findLast !== 'function'
+  typeof quickbit.findLast !== 'function' ||
+  typeof quickbit.clear !== 'function'
 ) {
   // This should always load the fallback from the locally installed version
   quickbit = require('quickbit-universal/fallback')

package/lib/remote-bitfield.js

@@ -35,10 +35,10 @@ class RemoteBitfieldPage {
   setRange (start, length, val) {
     quickbit.fill(this.bitfield, val, start, start + length)
 
-    let i = Math.floor(start / 32)
-    const n = i + Math.ceil(length / 32)
+    let i = Math.floor(start / 128)
+    const n = i + Math.ceil(length / 128)
 
-    while (i < n) this.tree.update(this.offset * 8 + i++ * 32)
+    while (i <= n) this.tree.update(this.offset * 8 + i++ * 128)
   }
 
   findFirst (val, position) {
@@ -51,6 +51,11 @@ class RemoteBitfieldPage {
 
   insert (start, bitfield) {
     this.bitfield.set(bitfield, start / 32)
+    this.segment.refresh()
+  }
+
+  clear (start, bitfield) {
+    quickbit.clear(this.bitfield, { field: bitfield, offset: start })
   }
 }
 
@@ -60,14 +65,22 @@ class RemoteBitfieldSegment {
     this.offset = index * BYTES_PER_SEGMENT
     this.tree = quickbit.Index.from([], BYTES_PER_SEGMENT)
     this.pages = new Array(PAGES_PER_SEGMENT)
+    this.pagesLength = 0
   }
 
   get chunks () {
     return this.tree.chunks
   }
 
+  refresh () {
+    this.tree = quickbit.Index.from(this.tree.chunks, BYTES_PER_SEGMENT)
+  }
+
   add (page) {
-    this.pages[page.index - this.index * PAGES_PER_SEGMENT] = page
+    const pageIndex = page.index - this.index * PAGES_PER_SEGMENT
+    if (pageIndex >= this.pagesLength) this.pagesLength = pageIndex + 1
+
+    this.pages[pageIndex] = page
 
     const chunk = { field: page.bitfield, offset: page.offset }
 
@@ -89,7 +102,7 @@ class RemoteBitfieldSegment {
 
     if (i >= PAGES_PER_SEGMENT) return -1
 
-    while (i < this.pages.length) {
+    while (i < this.pagesLength) {
       const p = this.pages[i]
 
       let index = -1
@@ -138,6 +151,14 @@ module.exports = class RemoteBitfield {
     this._segments = new BigSparseArray()
   }
 
+  getBitfield (index) {
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
+
+    const p = this._pages.get(i)
+    return p || null
+  }
+
   get (index) {
     const j = index & (BITS_PER_PAGE - 1)
     const i = (index - j) / BITS_PER_PAGE
@@ -278,4 +299,37 @@ module.exports = class RemoteBitfield {
 
     return true
   }
+
+  clear (start, bitfield) {
+    if (start % 32 !== 0) return false
+
+    let length = bitfield.byteLength * 8
+
+    let j = start & (BITS_PER_PAGE - 1)
+    let i = (start - j) / BITS_PER_PAGE
+
+    while (length > 0) {
+      let p = this._pages.get(i)
+
+      if (!p) {
+        const k = Math.floor(i / PAGES_PER_SEGMENT)
+        const s = this._segments.get(k) || this._segments.set(k, new RemoteBitfieldSegment(k))
+
+        p = this._pages.set(i, new RemoteBitfieldPage(i, new Uint32Array(WORDS_PER_PAGE), s))
+      }
+
+      const end = Math.min(j + length, BITS_PER_PAGE)
+      const range = end - j
+
+      p.clear(j, bitfield.subarray(0, range / 32))
+
+      bitfield = bitfield.subarray(range / 32)
+
+      j = 0
+      i++
+      length -= range
+    }
+
+    return true
+  }
 }

package/lib/replicator.js

@@ -310,7 +310,7 @@ class Peer {
 
     this.remoteOpened = false
     this.remoteBitfield = new RemoteBitfield()
-    this.skipList = new RemoteBitfield()
+    this.missingBlocks = new RemoteBitfield()
 
     this.remoteFork = 0
     this.remoteLength = 0
@@ -350,7 +350,9 @@ class Peer {
   }
 
   broadcastRange (start, length, drop) {
-    if (drop) this.skipList.setRange(start, length, false)
+    if (drop) this._unclearLocalRange(start, length)
+    else this._clearLocalRange(start, length)
+
     this.wireRange.send({
       drop,
       start,
@@ -699,6 +701,7 @@ class Peer {
         // might be a fork, verify
         this._checkIfConflict()
       }
+
       this.replicator._onnodata(this, req)
       this.replicator.oninvalid(err, req, data, this)
       return
@@ -732,25 +735,85 @@ class Peer {
   }
 
   onbitfield ({ start, bitfield }) {
-    if (this.remoteBitfield.insert(start, bitfield)) {
-      this.skipList.setRange(start, bitfield.byteLength * 8, false)
-      this._update()
+    this.remoteBitfield.insert(start, bitfield)
+    this.missingBlocks.insert(start, bitfield)
+    this._clearLocalRange(start, bitfield.byteLength * 8)
+    this._update()
+  }
+
+  _clearLocalRange (start, length) {
+    if (length === 1) {
+      this.missingBlocks.set(start, this.remoteBitfield.get(start) && !this.core.bitfield.get(start))
+      return
+    }
+
+    const contig = Math.min(this.core.tree.length, this.core.header.hints.contiguousLength)
+
+    if (start + length < contig) {
+      const delta = contig - start
+      this.missingBlocks.setRange(start, delta, false)
+      return
+    }
+
+    const rem = start & 32767
+    if (rem > 0) {
+      start -= rem
+      length += rem
+    }
+
+    const end = start + Math.min(length, this.core.tree.length)
+    while (start < end) {
+      const local = this.core.bitfield.getBitfield(start)
+
+      if (local && local.bitfield) {
+        this.missingBlocks.clear(start, local.bitfield)
+      }
+
+      start += 32768
     }
   }
 
+  _unclearLocalRange (start, length) {
+    if (length === 1) {
+      this.missingBlocks.set(start, this.remoteBitfield.get(start) && !this.core.bitfield.get(start))
+      return
+    }
+
+    const rem = start & 2097151
+    if (rem > 0) {
+      start -= rem
+      length += rem
+    }
+
+    const fixedStart = start
+
+    const end = start + Math.min(length, this.remoteLength)
+    while (start < end) {
+      const remote = this.remoteBitfield.getBitfield(start)
+      if (remote && remote.bitfield) {
+        this.missingBlocks.insert(start, remote.bitfield)
+      }
+
+      start += 2097152
+    }
+
+    this._clearLocalRange(fixedStart, length)
+  }
+
   onrange ({ drop, start, length }) {
     const has = drop === false
 
     if (length === 1) {
-      this.remoteBitfield.setRange(start, length, has)
-      this.skipList.set(start, false)
+      this.remoteBitfield.set(start, has)
+      this.missingBlocks.set(start, has && !this.core.bitfield.get(start))
     } else {
       const rangeStart = this.remoteBitfield.findFirst(!has, start)
       const rangeLength = length - (rangeStart - start)
 
       if (rangeLength > 0) {
         this.remoteBitfield.setRange(rangeStart, rangeLength, has)
-        this.skipList.setRange(rangeStart, rangeLength, false)
+        this.missingBlocks.setRange(rangeStart, rangeLength, has)
+        if (has) this._clearLocalRange(rangeStart, rangeLength)
       }
     }
 
@@ -942,74 +1005,74 @@ class Peer {
     return true
   }
 
-  _requestRange (r) {
-    const { length, fork } = this.core.tree
+  _requestRangeBlock (index, length) {
+    if (this.core.bitfield.get(index) === true || !this._hasTreeParent(index)) return false
 
-    const end = Math.min(r.end === -1 ? this.remoteLength : r.end, this.remoteLength)
-    if (end < r.start || fork !== this.remoteFork) return false
+    const b = this.replicator._blocks.add(index, PRIORITY.NORMAL)
+    if (b.inflight.length > 0) return false
 
-    const len = end - r.start
-    const off = r.start + (r.linear ? 0 : Math.floor(Math.random() * len))
+    const req = this._makeRequest(index >= length, b.priority)
 
-    // TODO: we should weight this to request blocks < .length first
-    // as they are "cheaper" and will trigger an auto upgrade if possible
-    // If no blocks < .length is avaible then try the "needs upgrade" range
+    // If the request cannot be satisfied, dealloc the block request if no one is subscribed to it
+    if (req === null) {
+      b.gc()
+      return false
+    }
 
-    let wrapped = 0
+    req.block = { index, nodes: 0 }
 
-    for (let i = 0; i < len && wrapped < 2; i++) {
-      let index = off + i
-      if (index >= end) index -= len
+    b.inflight.push(req)
+    this._send(req)
 
-      if (r.blocks !== null) {
-        index = r.blocks[index]
-      } else { // TODO: make this loop better (something like a for loop that skips with the skip list)
-        index = this.skipList.findFirst(false, index)
-        if (index === -1 || index >= end) {
-          wrapped++
-          index = this.skipList.findFirst(false, r.start)
-          if (index === -1 || index >= end) {
-            return false
-          }
-        }
-      }
+    // Don't think this will ever happen, as the pending queue is drained before the range queue
+    // but doesn't hurt to check this explicitly here also.
+    if (b.queued) b.queued = false
+    return true
+  }
 
-      if (this.remoteBitfield.get(index) === false || this.core.bitfield.get(index) === true) {
-        this.skipList.set(index, true)
-        continue
-      }
+  _requestRange (r) {
+    const { length, fork } = this.core.tree
 
-      if (!this._hasTreeParent(index)) {
-        continue
+    if (r.blocks) {
+      let min = -1
+      let max = -1
+
+      for (let i = r.start; i < r.end; i++) {
+        const index = r.blocks[i]
+        if (min === -1 || index < min) min = index
+        if (max === -1 || index > max) max = index
+        if (this.missingBlocks.get(index) === true && this._requestRangeBlock(index, length)) return true
       }
 
-      const b = this.replicator._blocks.add(index, PRIORITY.NORMAL)
+      if (min > -1) this._maybeWant(min, max - min)
+      return false
+    }
 
-      if (b.inflight.length > 0) {
-        this.skipList.set(index, true)
-        continue
-      }
+    const end = Math.min(r.end === -1 ? this.remoteLength : r.end, this.remoteLength)
+    if (end < r.start || fork !== this.remoteFork) return false
 
-      const req = this._makeRequest(index >= length, b.priority)
+    const len = end - r.start
+    const off = r.start + (r.linear ? 0 : Math.floor(Math.random() * len))
 
-      // If the request cannot be satisfied, dealloc the block request if no one is subscribed to it
-      if (req === null) {
-        b.gc()
-        return false
-      }
+    let i = off
 
-      this.skipList.set(index, true)
+    while (true) {
+      i = this.missingBlocks.findFirst(true, i)
+      if (i === -1 || i >= end) break
+
+      if (this._requestRangeBlock(i, length)) return true
+      i++
+    }
 
-      req.block = { index, nodes: 0 }
+    i = r.start
 
-      b.inflight.push(req)
-      this._send(req)
+    while (true) {
+      i = this.missingBlocks.findFirst(true, i)
 
-      // Don't think this will ever happen, as the pending queue is drained before the range queue
-      // but doesn't hurt to check this explicitly here also.
-      if (b.queued) b.queued = false
+      if (i === -1 || i >= off) break
 
-      return true
+      if (this._requestRangeBlock(i, length)) return true
+      i++
     }
 
     this._maybeWant(r.start, len)
@@ -1197,7 +1260,7 @@ module.exports = class Replicator {
   }
 
   // Called externally when a truncation upgrade has been processed
-  ontruncate (newLength) {
+  ontruncate (newLength, truncated) {
     const notify = []
 
     for (const blk of this._blocks) {
@@ -1211,6 +1274,8 @@ module.exports = class Replicator {
         blk.detach(r, SNAPSHOT_NOT_AVAILABLE())
       }
     }
+
+    for (const peer of this.peers) peer._unclearLocalRange(newLength, truncated)
   }
 
   // Called externally when a upgrade has been processed
@@ -1526,9 +1591,9 @@ module.exports = class Replicator {
 
     if (b === null || removeInflight(b.inflight, req) === false) return
 
-    if (isBlock && this.core.bitfield.get(index) === false) {
-      for (const peer of this.peers) peer.skipList.set(index, false)
-    }
+    // if (isBlock && this.core.bitfield.get(index) === false) {
+    //   for (const peer of this.peers) peer.skipList.set(index, false)
+    // }
 
     if (b.refs.length > 0 && isBlock === true) {
       this._queueBlock(b)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.31.5",
+  "version": "10.31.7",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -48,7 +48,7 @@
     "hypercore-id-encoding": "^1.2.0",
     "is-options": "^1.0.1",
     "protomux": "^3.5.0",
-    "quickbit-universal": "^2.1.1",
+    "quickbit-universal": "^2.2.0",
     "random-access-file": "^4.0.0",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",