hypercore 10.35.1 → 10.35.3

package/index.js

@@ -60,7 +60,6 @@ module.exports = class Hypercore extends EventEmitter {
     this.crypto = opts.crypto || hypercoreCrypto
     this.core = null
     this.replicator = null
-    this.inflightRange = opts.inflightRange || null
     this.encryption = null
     this.extensions = new Map()
     this.cache = createCache(opts.cache)
@@ -405,7 +404,7 @@ module.exports = class Hypercore extends EventEmitter {
       eagerUpgrade: true,
       notDownloadingLinger: opts.notDownloadingLinger,
       allowFork: opts.allowFork !== false,
-      inflightRange: this.inflightRange,
+      inflightRange: opts.inflightRange,
       onpeerupdate: this._onpeerupdate.bind(this),
       onupload: this._onupload.bind(this),
       oninvalid: this._oninvalid.bind(this)

package/lib/bitfield.js

@@ -213,6 +213,30 @@ module.exports = class Bitfield {
     }
   }
 
+  toBuffer (length) {
+    const pages = Math.ceil(length / BITS_PER_PAGE)
+    const buffer = b4a.allocUnsafe(pages * BYTES_PER_PAGE)
+
+    for (let i = 0; i < pages; i++) {
+      const page = this._pages.get(i)
+      const offset = i * BYTES_PER_PAGE
+
+      if (page) {
+        const buf = b4a.from(
+          page.bitfield.buffer,
+          page.bitfield.byteOffset,
+          page.bitfield.byteLength
+        )
+
+        buffer.set(buf, offset)
+      } else {
+        buffer.fill(0, offset, offset + BYTES_PER_PAGE)
+      }
+    }
+
+    return buffer
+  }
+
   getBitfield (index, length) {
     const j = index & (BITS_PER_PAGE - 1)
     const i = (index - j) / BITS_PER_PAGE
@@ -303,7 +327,7 @@ module.exports = class Bitfield {
       i++
     }
 
-    return val ? -1 : position
+    return val ? -1 : this._segments.maxLength * BITS_PER_SEGMENT
   }
 
   firstSet (position) {

package/lib/core.js

@@ -6,6 +6,7 @@ const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
+const RemoteBitfield = require('./remote-bitfield')
 const Info = require('./info')
 const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_OPERATION, INVALID_SIGNATURE, INVALID_CHECKSUM } = require('hypercore-errors')
 const m = require('./messages')
@@ -30,6 +31,7 @@ module.exports = class Core {
     this.truncating = 0
     this.updating = false
     this.closed = false
+    this.skipBitfield = null
 
     this._manifestFlushed = !!header.manifest
     this._maxOplogSize = 65536
@@ -142,7 +144,7 @@ module.exports = class Core {
     const prologue = header.manifest ? header.manifest.prologue : null
 
     const tree = await MerkleTree.open(treeFile, { crypto, prologue, ...header.tree })
-    const bitfield = await Bitfield.open(bitfieldFile, tree)
+    const bitfield = await Bitfield.open(bitfieldFile)
     const blocks = new BlockStore(dataFile, tree)
 
     if (overwrite) {
@@ -258,7 +260,7 @@ module.exports = class Core {
         const [offset] = await src.tree.byteRange(2 * segmentStart)
         await this.blocks.putBatch(segmentStart, segment, offset)
 
-        this.bitfield.setRange(segmentStart, segmentEnd - segmentStart, true)
+        this._setBitfieldRange(segmentStart, segmentEnd - segmentStart, true)
 
         pos = segmentEnd + 1
 
@@ -300,7 +302,7 @@ module.exports = class Core {
         const blocks = additional.length === missing ? additional : additional.slice(offset, offset + missing)
 
         await this.blocks.putBatch(source.length, blocks, source.byteLength)
-        this.bitfield.setRange(source.length, missing, true)
+        this._setBitfieldRange(source.length, missing, true)
 
         updates.push({
           drop: false,
@@ -464,7 +466,7 @@ module.exports = class Core {
 
       await this.oplog.append([entry], false)
 
-      this.bitfield.setRange(batch.ancestors, len - batch.ancestors, false)
+      this._setBitfieldRange(batch.ancestors, len - batch.ancestors, false)
       batch.commit()
 
       // TODO: (see below todo)
@@ -491,7 +493,7 @@ module.exports = class Core {
 
       await this.oplog.append([entry], false)
 
-      this.bitfield.setRange(start, end - start, false)
+      this._setBitfieldRange(start, end - start, false)
 
       if (start < this.header.hints.contiguousLength) {
         this.header.hints.contiguousLength = start
@@ -588,7 +590,7 @@ module.exports = class Core {
       await this.blocks.putBatch(treeLength, adding > values.length ? values.slice(0, adding) : values, byteOffset)
       await this.oplog.append([entry], false)
 
-      this.bitfield.setRange(entry.bitfield.start, entry.bitfield.length, true)
+      this._setBitfieldRange(entry.bitfield.start, entry.bitfield.length, true)
       batch.commit()
 
       if (batch.upgraded) {
@@ -654,7 +656,7 @@ module.exports = class Core {
 
       await this.oplog.append([entry], false)
 
-      this.bitfield.setRange(batch.ancestors, batch.length - batch.ancestors, true)
+      this._setBitfieldRange(batch.ancestors, batch.length - batch.ancestors, true)
       batch.commit()
 
       this.header.tree.length = batch.length
@@ -719,7 +721,7 @@ module.exports = class Core {
       let status = 0b0001
 
       if (bitfield) {
-        this.bitfield.set(bitfield.start, true)
+        this._setBitfield(bitfield.start, true)
         status |= updateContig(this.header, bitfield, this.bitfield)
       }
 
@@ -781,7 +783,7 @@ module.exports = class Core {
         let status = 0
 
         if (bitfield) {
-          this.bitfield.set(bitfield.start, true)
+          this._setBitfield(bitfield.start, true)
           status = updateContig(this.header, bitfield, this.bitfield)
         }
 
@@ -898,7 +900,7 @@ module.exports = class Core {
 
     await this.oplog.append([entry], false)
 
-    this.bitfield.setRange(batch.ancestors, this.tree.length - batch.ancestors, false)
+    this._setBitfieldRange(batch.ancestors, this.tree.length - batch.ancestors, false)
     addReorgHint(this.header.hints.reorgs, this.tree, batch)
     batch.commit()
 
@@ -920,6 +922,23 @@ module.exports = class Core {
     await this._flushOplog()
   }
 
+  openSkipBitfield () {
+    if (this.skipBitfield !== null) return this.skipBitfield
+    this.skipBitfield = new RemoteBitfield()
+    this.skipBitfield.insert(0, this.bitfield.toBuffer(this.tree.length))
+    return this.skipBitfield
+  }
+
+  _setBitfield (index, value) {
+    this.bitfield.set(index, value)
+    if (this.skipBitfield !== null) this.skipBitfield.set(index, value)
+  }
+
+  _setBitfieldRange (start, length, value) {
+    this.bitfield.setRange(start, length, value)
+    if (this.skipBitfield !== null) this.skipBitfield.setRange(start, length, value)
+  }
+
   async close () {
     this.closed = true
     await this._mutex.destroy()

package/lib/remote-bitfield.js

@@ -116,7 +116,7 @@ class RemoteBitfieldSegment {
       i++
     }
 
-    return -1
+    return (val || this.pagesLength === PAGES_PER_SEGMENT) ? -1 : this.pagesLength * BITS_PER_PAGE
   }
 
   findLast (val, position) {
@@ -230,7 +230,7 @@ module.exports = class RemoteBitfield {
       i++
     }
 
-    return val ? -1 : position
+    return val ? -1 : this._maxSegments * BITS_PER_SEGMENT
   }
 
   firstSet (position) {

package/lib/replicator.js

@@ -33,7 +33,7 @@ const m = require('./messages')
 const caps = require('./caps')
 const { createTracer } = require('hypertrace')
 
-const DEFAULT_MAX_INFLIGHT = [32, 512]
+const DEFAULT_MAX_INFLIGHT = [16, 512]
 const SCALE_LATENCY = 50
 const DEFAULT_SEGMENT_SIZE = 256 * 1024 * 8 // 256 KiB in bits
 const NOT_DOWNLOADING_SLACK = 20000 + (Math.random() * 20000) | 0
@@ -369,8 +369,8 @@ class Peer {
     const stream = this.stream.rawStream
     if (!stream.udx) return Math.min(this.inflightRange[1], this.inflightRange[0] * 3)
 
-    const scale = stream.rtt <= SCALE_LATENCY ? 1 : stream.rtt / SCALE_LATENCY
-    return Math.round(Math.min(this.inflightRange[1], this.inflightRange[0] * scale))
+    const scale = stream.rtt <= SCALE_LATENCY ? 1 : stream.rtt / SCALE_LATENCY * Math.min(1, 2 / this.replicator.peers.length)
+    return Math.max(this.inflightRange[0], Math.round(Math.min(this.inflightRange[1], this.inflightRange[0] * scale)))
   }
 
   signalUpgrade () {
@@ -378,6 +378,10 @@ class Peer {
     else this.sendSync()
   }
 
+  _markInflight (index) {
+    this.missingBlocks.set(index, false)
+  }
+
   broadcastRange (start, length, drop) {
     if (drop) this._unclearLocalRange(start, length)
     else this._clearLocalRange(start, length)
@@ -662,11 +666,12 @@ class Peer {
   }
 
   _cancelRequest (id) {
-    const exists = this.replicator._inflight.get(id)
-    if (!exists) return
+    const req = this.replicator._inflight.get(id)
+    if (!req) return
 
     this.inflight--
     this.replicator._removeInflight(id)
+    if (isBlockRequest(req)) this.replicator._unmarkInflight(req.block.index)
 
     this.wireCancel.send({ request: id })
   }
@@ -716,6 +721,8 @@ class Peer {
       if (reorg === true) return await this.replicator._onreorgdata(this, req, data)
     } catch (err) {
       safetyCatch(err)
+      if (isBlockRequest(req)) this.replicator._unmarkInflight(req.block.index)
+
       this.paused = true
       this.replicator.oninvalid(err, req, data, this)
       return
@@ -730,6 +737,7 @@ class Peer {
       }
     } catch (err) {
       safetyCatch(err)
+      if (isBlockRequest(req)) this.replicator._unmarkInflight(req.block.index)
 
       if (err.code === 'WRITE_FAILED') {
         // For example, we don't want to keep pulling data when storage is full
@@ -788,8 +796,10 @@ class Peer {
   }
 
   _clearLocalRange (start, length) {
+    const bitfield = this.core.skipBitfield === null ? this.core.bitfield : this.core.skipBitfield
+
     if (length === 1) {
-      this.missingBlocks.set(start, this._remoteHasBlock(start) && !this.core.bitfield.get(start))
+      this.missingBlocks.set(start, this._remoteHasBlock(start) && !bitfield.get(start))
       return
     }
 
@@ -809,7 +819,7 @@ class Peer {
 
     const end = start + Math.min(length, this.core.tree.length)
     while (start < end) {
-      const local = this.core.bitfield.getBitfield(start)
+      const local = bitfield.getBitfield(start)
 
       if (local && local.bitfield) {
         this.missingBlocks.clear(start, local.bitfield)
@@ -819,9 +829,14 @@ class Peer {
     }
   }
 
+  _resetMissingBlock (index) {
+    const bitfield = this.core.skipBitfield === null ? this.core.bitfield : this.core.skipBitfield
+    this.missingBlocks.set(index, this._remoteHasBlock(index) && !bitfield.get(index))
+  }
+
   _unclearLocalRange (start, length) {
     if (length === 1) {
-      this.missingBlocks.set(start, this._remoteHasBlock(start) && !this.core.bitfield.get(start))
+      this._resetMissingBlock(start)
       return
     }
 
@@ -856,8 +871,9 @@ class Peer {
     if (start === 0 && drop === false && length > this._remoteContiguousLength) {
       this._remoteContiguousLength = length
     } else if (length === 1) {
+      const bitfield = this.core.skipBitfield === null ? this.core.bitfield : this.core.skipBitfield
       this.remoteBitfield.set(start, has)
-      this.missingBlocks.set(start, has && !this.core.bitfield.get(start))
+      this.missingBlocks.set(start, has && !bitfield.get(start))
     } else {
       const rangeStart = this.remoteBitfield.findFirst(!has, start)
       const rangeLength = length - (rangeStart - start)
@@ -1034,6 +1050,14 @@ class Peer {
     return index < this._remoteContiguousLength || this.remoteBitfield.get(index) === true
   }
 
+  _sendBlockRequest (req, b) {
+    req.block = { index: b.index, nodes: 0 }
+    this.replicator._markInflight(b.index)
+
+    b.inflight.push(req)
+    this._send(req)
+  }
+
   _requestBlock (b) {
     const { length, fork } = this.core.tree
 
@@ -1049,10 +1073,7 @@ class Peer {
     const req = this._makeRequest(b.index >= length, b.priority)
     if (req === null) return false
 
-    req.block = { index: b.index, nodes: 0 }
-
-    b.inflight.push(req)
-    this._send(req)
+    this._sendBlockRequest(req, b)
 
     return true
   }
@@ -1061,7 +1082,10 @@ class Peer {
     if (this.core.bitfield.get(index) === true || !this._hasTreeParent(index)) return false
 
     const b = this.replicator._blocks.add(index, PRIORITY.NORMAL)
-    if (b.inflight.length > 0) return false
+    if (b.inflight.length > 0) {
+      this.missingBlocks.set(index, false) // in case we missed some states just set them ondemand, nbd
+      return false
+    }
 
     const req = this._makeRequest(index >= length, b.priority)
 
@@ -1071,10 +1095,7 @@ class Peer {
       return false
     }
 
-    req.block = { index, nodes: 0 }
-
-    b.inflight.push(req)
-    this._send(req)
+    this._sendBlockRequest(req, b)
 
     // Don't think this will ever happen, as the pending queue is drained before the range queue
     // but doesn't hurt to check this explicitly here also.
@@ -1084,7 +1105,8 @@ class Peer {
 
   _findNext (i) {
     if (i < this._remoteContiguousLength) {
-      i = this.core.bitfield.findFirst(false, i)
+      if (this.core.skipBitfield === null) this.replicator._openSkipBitfield()
+      i = this.core.skipBitfield.findFirst(false, i)
       if (i < this._remoteContiguousLength && i > -1) return i
       i = this._remoteContiguousLength
     }
@@ -1583,9 +1605,8 @@ module.exports = class Replicator {
 
   _removeInflight (id) {
     this._inflight.remove(id)
-    if (this.isDownloading() === false) {
-      for (const peer of this.peers) peer.signalUpgrade()
-    }
+    if (this.isDownloading() === true) return
+    for (const peer of this.peers) peer.signalUpgrade()
   }
 
   _removePeer (peer) {
@@ -1788,6 +1809,7 @@ module.exports = class Replicator {
   _clearRequest (peer, req) {
     if (req.block !== null) {
       this._clearInflightBlock(this._blocks, req)
+      this._unmarkInflight(req.block.index)
     }
 
     if (req.hash !== null) {
@@ -1812,6 +1834,28 @@ module.exports = class Replicator {
     this.updateAll()
   }
 
+  _openSkipBitfield () {
+    // technically the skip bitfield gets bits cleared if .clear() is called
+    // also which might be in inflight also, but that just results in that section being overcalled shortly
+    // worst case, so ok for now
+
+    const bitfield = this.core.openSkipBitfield()
+
+    for (const req of this._inflight) {
+      if (req.block) bitfield.set(req.block.index, true) // skip
+    }
+  }
+
+  _markInflight (index) {
+    if (this.core.skipBitfield !== null) this.core.skipBitfield.set(index, true)
+    for (const peer of this.peers) peer._markInflight(index)
+  }
+
+  _unmarkInflight (index) {
+    if (this.core.skipBitfield !== null) this.core.skipBitfield.set(index, this.core.bitfield.get(index))
+    for (const peer of this.peers) peer._resetMissingBlock(index)
+  }
+
   _ondata (peer, req, data) {
     if (data.block !== null) {
       this._resolveBlockRequest(this._blocks, data.block.index, data.block.value, req)
@@ -2247,3 +2291,7 @@ function onwireextension (m, c) {
 function setDownloadingLater (repl, downloading, session) {
   repl.setDownloadingNow(downloading, session)
 }
+
+function isBlockRequest (req) {
+  return req !== null && req.block !== null
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.35.1",
+  "version": "10.35.3",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {