hypercore 10.31.6 → 10.31.8

package/index.js

@@ -389,6 +389,7 @@ module.exports = class Hypercore extends EventEmitter {
 
     this.replicator = new Replicator(this.core, this.key, {
       eagerUpgrade: true,
+      notDownloadingLinger: opts.notDownloadingLinger,
       allowFork: opts.allowFork !== false,
       onpeerupdate: this._onpeerupdate.bind(this),
       onupload: this._onupload.bind(this),

package/lib/bitfield.js

@@ -43,7 +43,7 @@ class BitfieldPage {
     let i = Math.floor(start / 128)
     const n = i + Math.ceil(length / 128)
 
-    while (i < n) this.tree.update(this.offset * 8 + i++ * 128)
+    while (i <= n) this.tree.update(this.offset * 8 + i++ * 128)
   }
 
   findFirst (val, position) {

package/lib/remote-bitfield.js

@@ -38,7 +38,7 @@ class RemoteBitfieldPage {
     let i = Math.floor(start / 128)
     const n = i + Math.ceil(length / 128)
 
-    while (i < n) this.tree.update(this.offset * 8 + i++ * 128)
+    while (i <= n) this.tree.update(this.offset * 8 + i++ * 128)
   }
 
   findFirst (val, position) {
@@ -65,6 +65,7 @@ class RemoteBitfieldSegment {
     this.offset = index * BYTES_PER_SEGMENT
     this.tree = quickbit.Index.from([], BYTES_PER_SEGMENT)
     this.pages = new Array(PAGES_PER_SEGMENT)
+    this.pagesLength = 0
   }
 
   get chunks () {
@@ -76,7 +77,10 @@ class RemoteBitfieldSegment {
   }
 
   add (page) {
-    this.pages[page.index - this.index * PAGES_PER_SEGMENT] = page
+    const pageIndex = page.index - this.index * PAGES_PER_SEGMENT
+    if (pageIndex >= this.pagesLength) this.pagesLength = pageIndex + 1
+
+    this.pages[pageIndex] = page
 
     const chunk = { field: page.bitfield, offset: page.offset }
 
@@ -98,7 +102,7 @@ class RemoteBitfieldSegment {
 
     if (i >= PAGES_PER_SEGMENT) return -1
 
-    while (i < this.pages.length) {
+    while (i < this.pagesLength) {
       const p = this.pages[i]
 
       let index = -1

package/lib/replicator.js

@@ -11,6 +11,7 @@ const caps = require('./caps')
 const DEFAULT_MAX_INFLIGHT = [32, 512]
 const SCALE_LATENCY = 50
 const DEFAULT_SEGMENT_SIZE = 256 * 1024 * 8 // 256 KiB in bits
+const NOT_DOWNLOADING_SLACK = 4000 + (Math.random() * 4000) | 0
 
 const PRIORITY = {
   NORMAL: 0,
@@ -743,20 +744,23 @@ class Peer {
 
   _clearLocalRange (start, length) {
     if (length === 1) {
-      this.missingBlocks.set(start, false)
+      this.missingBlocks.set(start, this.remoteBitfield.get(start) && !this.core.bitfield.get(start))
       return
     }
 
     const contig = Math.min(this.core.tree.length, this.core.header.hints.contiguousLength)
 
-    if (start < contig) {
+    if (start + length < contig) {
       const delta = contig - start
-      this.missingBlocks.setRange(start, delta)
-      start = contig
-      length -= delta
+      this.missingBlocks.setRange(start, delta, false)
+      return
     }
 
-    if ((start & 31) > 0) start -= (start & 31)
+    const rem = start & 32767
+    if (rem > 0) {
+      start -= rem
+      length += rem
+    }
 
     const end = start + Math.min(length, this.core.tree.length)
     while (start < end) {
@@ -772,11 +776,17 @@ class Peer {
 
   _unclearLocalRange (start, length) {
     if (length === 1) {
-      this.missingBlocks.set(start, this.remoteBitfield.get(start))
+      this.missingBlocks.set(start, this.remoteBitfield.get(start) && !this.core.bitfield.get(start))
       return
     }
 
-    if ((start & 31) > 0) start -= (start & 31)
+    const rem = start & 2097151
+    if (rem > 0) {
+      start -= rem
+      length += rem
+    }
+
+    const fixedStart = start
 
     const end = start + Math.min(length, this.remoteLength)
     while (start < end) {
@@ -788,14 +798,14 @@ class Peer {
       start += 2097152
     }
 
-    this._clearLocalRange(start, length)
+    this._clearLocalRange(fixedStart, length)
   }
 
   onrange ({ drop, start, length }) {
     const has = drop === false
 
     if (length === 1) {
-      this.remoteBitfield.setRange(start, length, has)
+      this.remoteBitfield.set(start, has)
       this.missingBlocks.set(start, has && !this.core.bitfield.get(start))
     } else {
       const rangeStart = this.remoteBitfield.findFirst(!has, start)
@@ -1027,12 +1037,14 @@ class Peer {
     if (r.blocks) {
       let min = -1
       let max = -1
+
       for (let i = r.start; i < r.end; i++) {
         const index = r.blocks[i]
         if (min === -1 || index < min) min = index
         if (max === -1 || index > max) max = index
         if (this.missingBlocks.get(index) === true && this._requestRangeBlock(index, length)) return true
       }
+
       if (min > -1) this._maybeWant(min, max - min)
       return false
     }
@@ -1047,7 +1059,6 @@ class Peer {
 
     while (true) {
       i = this.missingBlocks.findFirst(true, i)
-
       if (i === -1 || i >= end) break
 
       if (this._requestRangeBlock(i, length)) return true
@@ -1061,7 +1072,7 @@ class Peer {
 
       if (i === -1 || i >= off) break
 
-      if (this.core.bitfield.get(i) === false && this._hasTreeParent(i) && this._requestRangeBlock(i, length)) return true
+      if (this._requestRangeBlock(i, length)) return true
       i++
     }
 
@@ -1165,7 +1176,14 @@ class Peer {
 }
 
 module.exports = class Replicator {
-  constructor (core, key, { eagerUpgrade = true, allowFork = true, onpeerupdate = noop, onupload = noop, oninvalid = noop } = {}) {
+  constructor (core, key, {
+    notDownloadingLinger = NOT_DOWNLOADING_SLACK,
+    eagerUpgrade = true,
+    allowFork = true,
+    onpeerupdate = noop,
+    onupload = noop,
+    oninvalid = noop
+  } = {}) {
     this.key = key
     this.discoveryKey = core.crypto.discoveryKey(key)
     this.core = core
@@ -1198,6 +1216,8 @@ module.exports = class Replicator {
     this._updatesPending = 0
     this._applyingReorg = null
     this._manifestPeer = null
+    this._notDownloadingLinger = notDownloadingLinger
+    this._downloadingTimer = null
 
     const self = this
     this._onstreamclose = onstreamclose
@@ -1217,6 +1237,19 @@ module.exports = class Replicator {
   }
 
   setDownloading (downloading, session) {
+    clearTimeout(this._downloadingTimer)
+
+    if (this.destroyed) return
+    if (downloading || this._notDownloadingLinger === 0) {
+      this.setDownloadingNow(downloading, session)
+      return
+    }
+
+    this._downloadingTimer = setTimeout(setDownloadingLater, this._notDownloadingLinger, this, downloading, session)
+  }
+
+  setDownloadingNow (downloading, session) {
+    this._downloadingTimer = null
     if (this.downloading === downloading) return
     this.downloading = downloading
     if (!downloading && this.isDownloading()) return
@@ -1973,6 +2006,10 @@ module.exports = class Replicator {
 
   destroy () {
     this.destroyed = true
+    if (this._downloadingTimer) {
+      clearTimeout(this._downloadingTimer)
+      this._downloadingTimer = null
+    }
     for (const peer of this.peers) {
       this.detachFrom(peer.protomux)
       peer.channel.close()
@@ -2137,3 +2174,7 @@ function onwirerange (m, c) {
 function onwireextension (m, c) {
   return c.userData.onextension(m)
 }
+
+function setDownloadingLater (repl, downloading, session) {
+  repl.setDownloadingNow(downloading, session)
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.31.6",
+  "version": "10.31.8",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {