hypercore 10.31.9 → 10.31.11

package/lib/core.js

@@ -230,6 +230,8 @@ module.exports = class Core {
     }
 
     try {
+      const updates = []
+
       let pos = 0
 
       while (pos < length) {
@@ -252,6 +254,12 @@ module.exports = class Core {
         this.bitfield.setRange(segmentStart, segmentEnd - segmentStart, true)
 
         pos = segmentEnd + 1
+
+        updates.push({
+          drop: false,
+          start: segmentStart,
+          length: segmentEnd - segmentStart
+        })
       }
 
       for (let i = 0; i < length * 2; i++) {
@@ -295,6 +303,10 @@ module.exports = class Core {
       this.header.userData = src.header.userData.slice(0)
       this.header.hints.contiguousLength = Math.min(src.header.hints.contiguousLength, this.header.tree.length)
 
+      for (const bitfield of updates) {
+        this.onupdate(0b0001, bitfield, null, null)
+      }
+
       await this._flushOplog()
     } finally {
       src._mutex.unlock()

package/lib/replicator.js

@@ -11,7 +11,7 @@ const caps = require('./caps')
 const DEFAULT_MAX_INFLIGHT = [32, 512]
 const SCALE_LATENCY = 50
 const DEFAULT_SEGMENT_SIZE = 256 * 1024 * 8 // 256 KiB in bits
-const NOT_DOWNLOADING_SLACK = 4000 + (Math.random() * 4000) | 0
+const NOT_DOWNLOADING_SLACK = 20000 + (Math.random() * 20000) | 0
 
 const PRIORITY = {
   NORMAL: 0,
@@ -305,6 +305,8 @@ class Peer {
     this.needsSync = false
     this.syncsProcessing = 0
 
+    this._remoteContiguousLength = 0
+
     // TODO: tweak pipelining so that data sent BEFORE remoteOpened is not cap verified!
     // we might wanna tweak that with some crypto, ie use the cap to encrypt it...
     // or just be aware of that, to only push non leaky data
@@ -334,7 +336,7 @@ class Peer {
   }
 
   get remoteContiguousLength () {
-    return this.remoteBitfield.findFirst(false, 0)
+    return this.remoteBitfield.findFirst(false, this._remoteContiguousLength)
   }
 
   getMaxInflight () {
@@ -736,6 +738,7 @@ class Peer {
   }
 
   onbitfield ({ start, bitfield }) {
+    if (start < this._remoteContiguousLength) this._remoteContiguousLength = start // bitfield is always the truth
     this.remoteBitfield.insert(start, bitfield)
     this.missingBlocks.insert(start, bitfield)
     this._clearLocalRange(start, bitfield.byteLength * 8)
@@ -744,7 +747,7 @@ class Peer {
 
   _clearLocalRange (start, length) {
     if (length === 1) {
-      this.missingBlocks.set(start, this.remoteBitfield.get(start) && !this.core.bitfield.get(start))
+      this.missingBlocks.set(start, this._remoteHasBlock(start) && !this.core.bitfield.get(start))
       return
     }
 
@@ -776,7 +779,7 @@ class Peer {
 
   _unclearLocalRange (start, length) {
     if (length === 1) {
-      this.missingBlocks.set(start, this.remoteBitfield.get(start) && !this.core.bitfield.get(start))
+      this.missingBlocks.set(start, this._remoteHasBlock(start) && !this.core.bitfield.get(start))
       return
     }
 
@@ -804,7 +807,13 @@ class Peer {
   onrange ({ drop, start, length }) {
     const has = drop === false
 
-    if (length === 1) {
+    if (drop === true && start < this._remoteContiguousLength) {
+      this._remoteContiguousLength = start
+    }
+
+    if (start === 0 && drop === false && length > this._remoteContiguousLength) {
+      this._remoteContiguousLength = length
+    } else if (length === 1) {
       this.remoteBitfield.set(start, has)
       this.missingBlocks.set(start, has && !this.core.bitfield.get(start))
     } else {
@@ -947,11 +956,6 @@ class Peer {
     return false
   }
 
-  // mb turn this into a YES/NO/MAYBE enum, could simplify ifavail logic
-  _blockAvailable (b) { // TODO: fork also
-    return this.remoteBitfield.get(b.index)
-  }
-
   _hasTreeParent (index) {
     if (this.remoteLength >= this.core.tree.length) return true
 
@@ -984,13 +988,18 @@ class Peer {
     return false
   }
 
+  _remoteHasBlock (index) {
+    return index < this._remoteContiguousLength || this.remoteBitfield.get(index) === true
+  }
+
   _requestBlock (b) {
     const { length, fork } = this.core.tree
 
-    if (this.remoteBitfield.get(b.index) === false || fork !== this.remoteFork) {
+    if (this._remoteHasBlock(b.index) === false || fork !== this.remoteFork) {
       this._maybeWant(b.index)
       return false
     }
+
     if (!this._hasTreeParent(b.index)) {
       return false
     }
@@ -1031,6 +1040,16 @@ class Peer {
     return true
   }
 
+  _findNext (i) {
+    if (i < this._remoteContiguousLength) {
+      i = this.core.bitfield.findFirst(false, i)
+      if (i < this._remoteContiguousLength && i > -1) return i
+      i = this._remoteContiguousLength
+    }
+
+    return this.missingBlocks.findFirst(true, i)
+  }
+
   _requestRange (r) {
     const { length, fork } = this.core.tree
 
@@ -1042,7 +1061,8 @@ class Peer {
         const index = r.blocks[i]
         if (min === -1 || index < min) min = index
         if (max === -1 || index > max) max = index
-        if (this.missingBlocks.get(index) === true && this._requestRangeBlock(index, length)) return true
+        const has = index < this._remoteContiguousLength || this.missingBlocks.get(index) === true
+        if (has === true && this._requestRangeBlock(index, length)) return true
       }
 
       if (min > -1) this._maybeWant(min, max - min)
@@ -1058,7 +1078,7 @@ class Peer {
     let i = off
 
     while (true) {
-      i = this.missingBlocks.findFirst(true, i)
+      i = this._findNext(i)
       if (i === -1 || i >= end) break
 
       if (this._requestRangeBlock(i, length)) return true
@@ -1068,8 +1088,7 @@ class Peer {
     i = r.start
 
     while (true) {
-      i = this.missingBlocks.findFirst(true, i)
-
+      i = this._findNext(i)
       if (i === -1 || i >= off) break
 
       if (this._requestRangeBlock(i, length)) return true
@@ -1103,7 +1122,7 @@ class Peer {
       let index = off + i
       if (index >= end) index -= len
 
-      if (this.remoteBitfield.get(index) === false) continue
+      if (this._remoteHasBlock(index) === false) continue
 
       const req = this._makeRequest(false, 0)
 

package/lib/streams.js

@@ -105,7 +105,7 @@ class ByteStream extends Readable {
     }
 
     this._predownload(this._index + 1)
-    data = await this._core.get(this._index++)
+    data = await this._core.get(this._index++, { valueEncoding: 'binary' })
 
     if (relativeOffset > 0) data = data.subarray(relativeOffset)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.31.9",
+  "version": "10.31.11",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {