hypercore 11.22.1 → 11.23.0

package/lib/bitfield.js

@@ -420,6 +420,26 @@ module.exports = class Bitfield {
     return this.findLast(false, position)
   }
 
+  hasUnset(start, length) {
+    const end = start + length
+
+    let j = start & (BITS_PER_SEGMENT - 1)
+    let i = (start - j) / BITS_PER_SEGMENT
+
+    while (i < this._segments.maxLength) {
+      const s = this._segments.get(i)
+      const index = s ? s.findFirst(false, j) : j
+      if (index !== -1) return i * BITS_PER_SEGMENT + index < end
+
+      j = 0
+      i++
+
+      if (i * BITS_PER_SEGMENT >= end) return false
+    }
+
+    return true
+  }
+
   hasSet(start, length) {
     const end = start + length
 
@@ -475,23 +495,23 @@ module.exports = class Bitfield {
   }
 
   *want(start, length) {
-    const j = start & (BITS_PER_SEGMENT - 1)
+    let j = start & (BITS_PER_SEGMENT - 1)
     let i = (start - j) / BITS_PER_SEGMENT
 
     while (length > 0) {
       const s = this._segments.get(i)
 
       if (s) {
-        // We always send at least 4 KiB worth of bitfield in a want, rounding
-        // to the nearest 4 KiB.
-        const end = ceilTo(clamp(length / 8, 4096, BYTES_PER_SEGMENT), 4096)
+        let end = Math.min(j + length, BITS_PER_SEGMENT)
+        if (end & 31) end += 32 - (end & 31)
 
         yield {
-          start: i * BITS_PER_SEGMENT,
-          bitfield: s.bitfield.subarray(0, end / 4)
+          start: i * BITS_PER_SEGMENT + j,
+          bitfield: s.bitfield.subarray(j / 32, end / 32)
         }
       }
 
+      j = 0
       i++
       length -= BITS_PER_SEGMENT
     }
@@ -521,13 +541,3 @@ module.exports = class Bitfield {
     return new Bitfield(buffer)
   }
 }
-
-function clamp(n, min, max) {
-  return Math.min(Math.max(n, min), max)
-}
-
-function ceilTo(n, multiple = 1) {
-  const remainder = n % multiple
-  if (remainder === 0) return n
-  return n + multiple - remainder
-}

package/lib/merkle-tree.js

@@ -723,6 +723,9 @@ class MerkleTree {
       if (r === index) return 0
     }
 
+    // need missing nodes not inclusive of index
+    ite.parent()
+
     let cnt = 0
     while (!ite.contains(head)) {
       cnt++

package/lib/messages.js

@@ -543,19 +543,32 @@ wire.noData = {
   }
 }
 
+// TODO: we should move to having these have a handle like in data, easier since they
+// allocate state in practice
 wire.want = {
   preencode(state, m) {
     c.uint.preencode(state, m.start)
     c.uint.preencode(state, m.length)
+
+    const flags = m.any ? 1 : 0
+    if (flags !== 0) c.uint.preencode(state, flags)
   },
   encode(state, m) {
     c.uint.encode(state, m.start)
     c.uint.encode(state, m.length)
+
+    const flags = m.any ? 1 : 0
+    if (flags !== 0) c.uint.encode(state, flags)
   },
   decode(state) {
+    const start = c.uint.decode(state)
+    const length = c.uint.decode(state)
+    const flags = state.start < state.end ? c.uint.decode(state) : 0
+
     return {
-      start: c.uint.decode(state),
-      length: c.uint.decode(state)
+      start,
+      length,
+      any: (flags & 1) !== 0
     }
   }
 }
@@ -564,15 +577,26 @@ wire.unwant = {
   preencode(state, m) {
     c.uint.preencode(state, m.start)
     c.uint.preencode(state, m.length)
+
+    const flags = m.any ? 1 : 0
+    if (flags !== 0) c.uint.preencode(state, flags)
   },
   encode(state, m) {
     c.uint.encode(state, m.start)
     c.uint.encode(state, m.length)
+
+    const flags = m.any ? 1 : 0
+    if (flags !== 0) c.uint.encode(state, flags)
   },
   decode(state, m) {
+    const start = c.uint.decode(state)
+    const length = c.uint.decode(state)
+    const flags = state.start < state.end ? c.uint.decode(state) : 0
+
     return {
-      start: c.uint.decode(state),
-      length: c.uint.decode(state)
+      start,
+      length,
+      any: (flags & 1) !== 0
     }
   }
 }

package/lib/remote-bitfield.js

@@ -147,6 +147,7 @@ class RemoteBitfieldSegment {
 
 module.exports = class RemoteBitfield {
   static BITS_PER_PAGE = BITS_PER_PAGE
+  static BITS_PER_SEGMENT = BITS_PER_SEGMENT
 
   constructor() {
     this._pages = new BigSparseArray()

package/lib/replicator.js

@@ -31,6 +31,7 @@ const ReceiverQueue = require('./receiver-queue')
 const HotswapQueue = require('./hotswap-queue')
 const RemoteBitfield = require('./remote-bitfield')
 const { MerkleTree } = require('./merkle-tree')
+const { LocalWants, RemoteWants, WANT_BATCH } = require('./wants')
 const {
   REQUEST_CANCELLED,
   REQUEST_TIMEOUT,
@@ -43,11 +44,9 @@ const caps = require('./caps')
 
 const DEFAULT_MAX_INFLIGHT = [16, 512]
 const SCALE_LATENCY = 50
-const DEFAULT_SEGMENT_SIZE = 256 * 1024 * 8 // 256 KiB in bits
 const NOT_DOWNLOADING_SLACK = (20000 + Math.random() * 20000) | 0
 const MAX_PEERS_UPGRADE = 3
 const LAST_BLOCKS = 256
-const MAX_REMOTE_SEGMENTS = 2048
 
 const MAX_RANGES = 64
 
@@ -69,6 +68,17 @@ class Attachable {
     this.resolved = false
     this.processing = false
     this.refs = []
+    this.wants = null
+  }
+
+  addWant(w) {
+    if (this.wants === null) this.wants = new Set()
+    this.wants.add(w)
+  }
+
+  removeWant(w) {
+    if (this.wants === null) return
+    this.wants.delete(w)
   }
 
   attach(session) {
@@ -118,7 +128,37 @@ class Attachable {
   }
 
   gc() {
-    if (this.refs.length === 0 && !this.processing) this._unref()
+    if (this.refs.length > 0 || this.processing) return
+    this._unref()
+
+    if (this.wants === null) return
+
+    const wants = this.wants
+    this.wants = null
+
+    let update = null
+
+    for (const w of wants) {
+      if (w.any) {
+        const removed = w.wants.removeAnyRange(w.start, w.length, this)
+        if (!removed) continue
+        if (update === null) update = new Set()
+        update.add(w.wants.peer)
+        // TODO: should also use a free list for simplicity
+        w.wants.peer.wireUnwant.send(removed)
+        continue
+      }
+
+      if (w.wants.removeBatch(w.start, this)) {
+        if (update === null) update = new Set()
+        update.add(w.wants.peer)
+      }
+    }
+
+    if (update === null) return
+    for (const peer of update) {
+      this.replicator.updatePeer(peer)
+    }
   }
 
   processed() {
@@ -139,6 +179,7 @@ class Attachable {
     while (this.refs.length > 0) {
       this._detach(this.refs[this.refs.length - 1]).resolve(val)
     }
+    this.gc()
   }
 
   reject(err) {
@@ -146,6 +187,7 @@ class Attachable {
     while (this.refs.length > 0) {
       this._detach(this.refs[this.refs.length - 1]).reject(err)
     }
+    this.gc()
   }
 
   setTimeout(r, ms) {
@@ -167,6 +209,7 @@ class BlockRequest extends Attachable {
   }
 
   _unref() {
+    if (this.resolved) return
     this.queued = false
 
     for (const req of this.inflight) {
@@ -178,6 +221,15 @@ class BlockRequest extends Attachable {
   }
 }
 
+class RangeBatchRequest extends Attachable {
+  constructor(replicator, batch) {
+    super(replicator)
+
+    this.batch = batch
+    this.index = 0
+  }
+}
+
 class RangeRequest extends Attachable {
   constructor(replicator, ranges, start, end, linear, ifAvailable, blocks) {
     super(replicator)
@@ -188,6 +240,8 @@ class RangeRequest extends Attachable {
     this.ifAvailable = ifAvailable
     this.blocks = blocks
     this.ranges = ranges
+    this.batches = []
+
     ranges.push(this)
 
     if (this.end === -1) {
@@ -199,6 +253,28 @@ class RangeRequest extends Attachable {
     this.userEnd = end
   }
 
+  addBatch(batch) {
+    for (let i = 0; i < this.batches.length; i++) {
+      const b = this.batches[i]
+      if (b.batch === batch) return false
+    }
+    const b = new RangeBatchRequest(this.replicator, batch)
+    b.index = this.batches.length
+    this.batches.push(b)
+    return true
+  }
+
+  removeBatch(batch) {
+    const head = this.batches.pop()
+
+    if (head !== batch) {
+      head.index = batch.index
+      this.batches[head.index] = head
+    }
+
+    batch.gc()
+  }
+
   _unref() {
     const rangeIndex = this.ranges.indexOf(this)
     if (rangeIndex === -1) return
@@ -211,6 +287,10 @@ class RangeRequest extends Attachable {
     if (this.end === -1) {
       this.replicator._alwaysLatestBlock--
     }
+
+    while (this.batches.length) {
+      this.batches.pop().gc()
+    }
   }
 
   _cancel(r) {
@@ -228,6 +308,7 @@ class UpgradeRequest extends Attachable {
   }
 
   _unref() {
+    if (this.resolved) return
     if (this.replicator.eagerUpgrade === true || this.inflight.length > 0) return
     this.replicator._upgrade = null
   }
@@ -247,6 +328,7 @@ class SeekRequest extends Attachable {
   }
 
   _unref() {
+    if (this.resolved) return
     if (this.inflight.length > 0) return
     const i = this.seeks.indexOf(this)
     if (i === -1) return
@@ -255,6 +337,15 @@ class SeekRequest extends Attachable {
   }
 }
 
+class ForkRequest extends Attachable {
+  constructor(replicator, fork) {
+    super(replicator)
+    this.fork = fork
+    this.inflight = []
+    this.batch = null
+  }
+}
+
 class InflightTracker {
   constructor() {
     this._requests = []
@@ -400,7 +491,6 @@ class Peer {
     this.remotePublicKey = this.stream.remotePublicKey
     this.remoteSupportsSeeks = false
     this.inflightRange = inflightRange
-    this.remoteSegmentsWanted = new Set()
     this.fullyDownloadedSignaled = 0
 
     this.paused = false
@@ -427,6 +517,7 @@ class Peer {
       wireCancel: { tx: 0, rx: 0 },
       wireData: { tx: 0, rx: 0 },
       wireWant: { tx: 0, rx: 0 },
+      wireUnwant: { tx: 0, rx: 0 },
       wireBitfield: { tx: 0, rx: 0 },
       wireRange: { tx: 0, rx: 0 },
       wireExtension: { tx: 0, rx: 0 },
@@ -477,6 +568,9 @@ class Peer {
     this.segmentsWanted = new Set()
     this.broadcastedNonSparse = false
 
+    this.wants = new LocalWants(this)
+    this.remoteWants = new RemoteWants(this)
+
     this.lengthAcked = 0
 
     this.extensions = new Map()
@@ -525,16 +619,10 @@ class Peer {
     if (drop) this._unclearLocalRange(start, length)
     else this._clearLocalRange(start, length)
 
-    const i = Math.floor(start / DEFAULT_SEGMENT_SIZE)
     const fullyContig = this.core.header.hints.contiguousLength === this.core.state.length
 
-    if (
-      start + LAST_BLOCKS < this.core.state.length &&
-      !this.remoteSegmentsWanted.has(i) &&
-      !drop &&
-      !fullyContig
-    ) {
-      return
+    if (start + LAST_BLOCKS < this.core.state.length && !drop && !fullyContig) {
+      if (!this.remoteWants.hasRange(start, length)) return
     }
 
     let force = false
@@ -915,7 +1003,7 @@ class Peer {
       nodes: MerkleTree.maxMissingNodes(2 * index, remoteLength)
     }
 
-    if (index >= this.remoteLength) {
+    if (index >= remoteLength) {
       msg.upgrade = {
         start: remoteLength,
         length: this.core.state.length - remoteLength
@@ -926,7 +1014,8 @@ class Peer {
     const batch = this.core.storage.read()
     try {
       req = await this._getProof(batch, msg, true)
-    } catch {
+    } catch (err) {
+      this.replicator._oninvalidrequest(err, msg, this)
       return
     }
 
@@ -1180,15 +1269,13 @@ class Peer {
     for (const id of flushed) this.replicator._inflight.reusable(id)
   }
 
-  onwant({ start, length }) {
-    const i = Math.floor(start / DEFAULT_SEGMENT_SIZE)
-    if (this.remoteSegmentsWanted.size >= MAX_REMOTE_SEGMENTS) this.remoteSegmentsWanted.clear() // just in case of abuse
-    this.remoteSegmentsWanted.add(i)
-    this.replicator._onwant(this, start, length)
+  onwant(range) {
+    if (!this.remoteWants.add(range)) return
+    this.replicator._onwant(this, range.start, range.length, range.any)
   }
 
-  onunwant() {
-    // TODO
+  onunwant(range) {
+    this.remoteWants.remove(range)
   }
 
   onbitfield({ start, bitfield }) {
@@ -1243,7 +1330,7 @@ class Peer {
       return
     }
 
-    const rem = start & 2097151
+    const rem = start & (RemoteBitfield.BITS_PER_SEGMENT - 1)
     if (rem > 0) {
       start -= rem
       length += rem
@@ -1258,7 +1345,7 @@ class Peer {
         this.missingBlocks.insert(start, remote.bitfield)
       }
 
-      start += 2097152
+      start += RemoteBitfield.BITS_PER_SEGMENT
     }
 
     this._clearLocalRange(fixedStart, length)
@@ -1458,7 +1545,7 @@ class Peer {
       return true
     }
 
-    this._maybeWant(s.seeker.start, len)
+    this._maybeWantAnyRange(s.seeker.start, len, s)
     return false
   }
 
@@ -1497,7 +1584,7 @@ class Peer {
     const { length, fork } = this.core.state
 
     if (this._remoteHasBlock(b.index) === false || fork !== this.remoteFork) {
-      this._maybeWant(b.index)
+      if (!this.core.bitfield.get(b.index)) this._maybeWant(b.index, b)
       return false
     }
 
@@ -1511,6 +1598,43 @@ class Peer {
     return true
   }
 
+  _maybeWant(index, handle) {
+    if (index < this.remoteContiguousLength) return
+
+    const res = this.wants.add(index, handle)
+    if (!res) return
+
+    const batch = res.want && res.unwant
+    if (batch) this.protomux.cork()
+
+    if (res.unwant) {
+      this.wireUnwant.send(res.unwant)
+      incrementTx(this.stats.wireUnwant, this.replicator.stats.wireUnwant)
+    }
+    if (res.want) {
+      this.wireWant.send(res.want)
+      incrementTx(this.stats.wireWant, this.replicator.stats.wireWant)
+    }
+
+    if (batch) this.protomux.uncork()
+  }
+
+  _maybeWantAnyRange(start, length, handle) {
+    if (handle.wants) {
+      for (const w of handle.wants) {
+        if ((w.start !== start || w.length !== length) && w.any) {
+          const req = this.wants.removeAnyRange(w.start, w.length, handle)
+          if (req) this.wireUnwant.send(req)
+        }
+      }
+    }
+
+    if (start + length <= this.remoteContiguousLength) return
+
+    const req = this.wants.addAnyRange(start, length, handle)
+    if (req) this.wireWant.send(req)
+  }
+
   _requestRangeBlock(index, length) {
     if (this.core.bitfield.get(index) === true || !this._canRequest(index)) return false
 
@@ -1547,68 +1671,119 @@ class Peer {
     return this.missingBlocks.findFirst(true, i)
   }
 
-  _requestRange(r) {
-    if (this.syncsProcessing > 0) return false
+  _addRangeBatch(r, offset, end) {
+    while (offset < end) {
+      const next = this.core.bitfield.findFirst(false, offset)
+      if (next === -1 || next >= end) return false
 
-    const { length, fork } = this.core.state
+      const b = (next - (next & (WANT_BATCH - 1))) / WANT_BATCH
 
-    if (r.blocks) {
-      let min = -1
-      let max = -1
+      if (r.addBatch(b)) return true
 
-      for (let i = r.start; i < r.end; i++) {
-        const index = r.blocks[i]
-        if (min === -1 || index < min) min = index
-        if (max === -1 || index > max) max = index
-        const has = index < this._remoteContiguousLength || this.missingBlocks.get(index) === true
-        if (has === true && this._requestRangeBlock(index, length)) return true
-      }
+      offset = (b + 1) * WANT_BATCH
+    }
 
-      if (min > -1) this._maybeWant(min, max - min)
-      return false
+    return false
+  }
+
+  _populateRangeBatches(r) {
+    for (let i = r.batches.length - 1; i >= 0; i--) {
+      const b = r.batches[i]
+      const minStart = b.batch * WANT_BATCH
+      const maxEnd = minStart + WANT_BATCH
+
+      const start = Math.max(r.start, minStart)
+      const end = Math.min(r.end === -1 ? maxEnd : r.end, maxEnd)
+
+      if (start < end && this.core.bitfield.hasUnset(start, end - start)) continue
+
+      r.removeBatch(b)
     }
 
+    while (r.batches.length < 3) {
+      const end = r.end === -1 ? this.core.state.length : r.end
+      if (end <= r.start) return
+
+      const len = end - r.start
+      const off = r.start + (r.linear ? 0 : Math.floor(Math.random() * len))
+
+      if (!this._addRangeBatch(r, off, end) && !this._addRangeBatch(r, r.start, end)) break
+    }
+  }
+
+  _requestRangeBatch(start, end, length) {
+    let tries = this.inflight
+
+    do {
+      start = this._findNext(start)
+      if (start === -1 || start >= end) break
+
+      if (this._requestRangeBlock(start, length)) return true
+      start++
+    } while (tries-- > 0)
+
+    return false
+  }
+
+  _getValidEnd(maxEnd) {
     // if we can upgrade the remote, or the remote is ahead, then all the remotes blocks are valid
     // otherwise truncate to the last length the remote has acked for us
     const maxLocalLength =
       this.canUpgrade || this.remoteLength >= this.core.state.length
         ? this.core.state.length
-        : fork === this.lastUpgradableFork
+        : this.core.state.fork === this.lastUpgradableFork
           ? Math.min(this.lastUpgradableLength, this.core.state.length)
           : 0
 
-    const end = Math.min(
+    return Math.min(
       maxLocalLength,
-      Math.min(r.end === -1 ? this.remoteLength : r.end, this.remoteLength)
+      Math.min(maxEnd === -1 ? this.remoteLength : maxEnd, this.remoteLength)
     )
-    if (end <= r.start || fork !== this.remoteFork) return false
+  }
 
-    const len = end - r.start
-    const off = r.start + (r.linear ? 0 : Math.floor(Math.random() * len))
+  _requestRange(r) {
+    if (this.syncsProcessing > 0) return false
 
-    let i = off
-    // should be way less than this, but this is worst case upper bound for the skiplist
-    let tries = this.inflight
+    const { length, fork } = this.core.state
 
-    do {
-      i = this._findNext(i)
-      if (i === -1 || i >= end) break
+    if (r.blocks) {
+      for (let i = r.start; i < r.end; i++) {
+        const index = r.blocks[i]
+        const has = index < this._remoteContiguousLength || this.missingBlocks.get(index) === true
+        if (has === true && this._requestRangeBlock(index, length)) return true
+        this._maybeWant(index, r)
+      }
 
-      if (this._requestRangeBlock(i, length)) return true
-      i++
-    } while (tries-- > 0)
+      return false
+    }
 
-    i = r.start
+    this._populateRangeBatches(r)
 
-    do {
-      i = this._findNext(i)
-      if (i === -1 || i >= off) break
+    const end = this._getValidEnd(r.end)
 
-      if (this._requestRangeBlock(i, length)) return true
-      i++
-    } while (tries-- > 0)
+    if (end <= r.start || fork !== this.remoteFork) return false
+
+    for (let i = 0; i < r.batches.length; i++) {
+      const b = r.batches[i]
+
+      const batchStart = Math.max(b.batch * WANT_BATCH, r.start)
+      const batchEnd = Math.min((b.batch + 1) * WANT_BATCH, end)
+      const len = batchEnd - batchStart
+      const off = batchStart + (r.linear ? 0 : Math.floor(Math.random() * len))
+
+      if (
+        this._requestRangeBatch(off, end, length) ||
+        this._requestRangeBatch(batchStart, off, length)
+      ) {
+        return true
+      }
+
+      this._maybeWant(batchStart, b)
+    }
+
+    // Here we should consider making an OOB request (ie if the remote says they have a newer block)
+    // but the upgrade code already does that, so no big deal for now
 
-    this._maybeWant(r.start, len)
     return false
   }
 
@@ -1649,28 +1824,10 @@ class Peer {
       return true
     }
 
-    this._maybeWant(f.batch.want.start, len)
+    this._maybeWantAnyRange(f.batch.want.start, len, f)
     return false
   }
 
-  _maybeWant(start, length = 1) {
-    if (start + length <= this.remoteContiguousLength) return
-
-    let i = Math.floor(start / DEFAULT_SEGMENT_SIZE)
-    const n = Math.ceil((start + length) / DEFAULT_SEGMENT_SIZE)
-
-    for (; i < n; i++) {
-      if (this.segmentsWanted.has(i)) continue
-      this.segmentsWanted.add(i)
-
-      this.wireWant.send({
-        start: i * DEFAULT_SEGMENT_SIZE,
-        length: DEFAULT_SEGMENT_SIZE
-      })
-      incrementTx(this.stats.wireWant, this.replicator.stats.wireWant)
-    }
-  }
-
   isActive() {
     if (this.paused || this.removed || this.core.header.frozen) return false
     return true
@@ -1757,6 +1914,7 @@ module.exports = class Replicator {
       wireCancel: { tx: 0, rx: 0 },
       wireData: { tx: 0, rx: 0 },
       wireWant: { tx: 0, rx: 0 },
+      wireUnwant: { tx: 0, rx: 0 },
       wireBitfield: { tx: 0, rx: 0 },
       wireRange: { tx: 0, rx: 0 },
       wireExtension: { tx: 0, rx: 0 },
@@ -2158,11 +2316,7 @@ module.exports = class Replicator {
       if (f.fork === fork) return f
     }
 
-    const f = {
-      fork,
-      inflight: [],
-      batch: null
-    }
+    const f = new ForkRequest(this, fork)
 
     this._reorgs.push(f)
 
@@ -2208,6 +2362,7 @@ module.exports = class Replicator {
 
   _removePeer(peer) {
     this.peers.splice(this.peers.indexOf(peer), 1)
+    peer.wants.destroy()
 
     if (this._manifestPeer === peer) this._manifestPeer = null
 
@@ -2548,12 +2703,12 @@ module.exports = class Replicator {
     this.updatePeer(peer)
   }
 
-  _onwant(peer, start, length) {
+  _onwant(peer, start, length, any) {
     if (!peer.isActive()) return
 
     const contig = Math.min(this.core.state.length, this.core.header.hints.contiguousLength)
 
-    if (start + length < contig || this.core.state.length === contig) {
+    if (start + length < contig || this.core.state.length === contig || (any && start < contig)) {
       peer.wireRange.send({
         drop: false,
         start: 0,
@@ -2565,14 +2720,22 @@ module.exports = class Replicator {
 
     length = Math.min(length, this.core.state.length - start)
 
-    peer.protomux.cork()
+    if (any) {
+      const bit = this.core.bitfield.firstSet(start)
+      if (bit > -1 && bit < start + length) {
+        peer.wireRange.send({
+          drop: false,
+          start: bit,
+          length: 1
+        })
+      }
+      return
+    }
 
     for (const msg of this.core.bitfield.want(start, length)) {
       peer.wireBitfield.send(msg)
       incrementTx(peer.stats.wireBitfield, this.stats.wireBitfield)
     }
-
-    peer.protomux.uncork()
   }
 
   async _onreorgdata(peer, req, data) {
@@ -2612,6 +2775,7 @@ module.exports = class Replicator {
     // should investigate the complexity in going the other way
 
     const u = this._upgrade
+    const old = this._reorgs
 
     this._reorgs = [] // clear all as the nodes are against the old tree - easier
     this._applyingReorg = this.core.reorg(f.batch, null) // TODO: null should be the first/last peer?
@@ -2637,6 +2801,9 @@ module.exports = class Replicator {
       r.end = r.userEnd
     }
 
+    for (const f of old) f.resolve()
+    f.resolve()
+
     this.updateAll()
   }
 

package/lib/wants.js

@@ -0,0 +1,307 @@
+const BATCH_UINTS = 128
+const BATCH_BYTES = BATCH_UINTS * 4
+const BATCH = BATCH_BYTES * 8 // in bits
+const MAX_REMOTE_BATCHES = 4
+const MAX_RANGE = 8 * 256 * 1024
+const MIN_RANGE = 32 // 4bit ints
+const MAX = 512
+const MAX_ANY = 16
+
+class LocalWants {
+  constructor(peer) {
+    this.destroyed = false
+    this.peer = peer
+    this.wants = new Map()
+    this.free = new Set()
+    this.any = null
+  }
+
+  add(index, handle) {
+    const b = (index - (index & (BATCH - 1))) / BATCH
+    return this.addBatch(b, handle)
+  }
+
+  addAnyRange(start, length, handle) {
+    if (this.destroyed) return null
+    if (this.any === null) this.any = []
+
+    for (let i = 0; i < this.any.length; i++) {
+      const w = this.any[i]
+      if (w.start === start && w.length === length) {
+        w.handles.add(handle)
+        handle.addWant(w)
+        return null
+      }
+    }
+
+    const w = { wants: this, start, length, any: true, handles: new Set([handle]) }
+    this.any.push(w)
+    handle.addWant(w)
+
+    return { start, length, any: true }
+  }
+
+  removeAnyRange(start, length, handle) {
+    if (this.any === null) return null
+
+    for (let i = 0; i < this.any.length; i++) {
+      const w = this.any[i]
+      if (w.start !== start || w.length !== length) continue
+
+      w.handles.delete(handle)
+      handle.removeWant(w)
+      if (w.handles.size > 0) return null
+
+      const head = this.any.pop()
+      if (head !== w) this.any[i] = head
+      return { start, length, any: true }
+    }
+
+    return null
+  }
+
+  addBatch(index, handle) {
+    if (this.destroyed) return null
+    let w = this.wants.get(index)
+
+    if (w) {
+      const size = w.handles.size
+      w.handles.add(handle)
+      if (w.handles.size !== size) {
+        handle.addWant(w)
+      }
+      return null
+    }
+
+    // start here is the batch number for simplicity....
+    w = { wants: this, start: index, length: 0, any: false, handles: new Set([handle]) }
+
+    if (this.free.has(index)) {
+      this.free.delete(index)
+      this.wants.set(index, w)
+      handle.addWant(w)
+      return null
+    }
+
+    let unwant = null
+    if (this.wants.size + this.free.size === MAX) {
+      if (this.free.size === 0) {
+        return null
+      }
+      unwant = this._unwant()
+    }
+
+    this.wants.set(index, w)
+    handle.addWant(w)
+
+    return {
+      want: { start: index * BATCH, length: BATCH, any: false },
+      unwant
+    }
+  }
+
+  remove(index, handle) {
+    const b = (index - (index & (BATCH - 1))) / BATCH
+    return this.removeBatch(b, handle)
+  }
+
+  removeBatch(index, handle) {
+    if (this.destroyed) return false
+
+    const w = this.wants.get(index)
+    if (!w) {
+      return false
+    }
+
+    w.handles.delete(handle)
+    handle.removeWant(w)
+
+    if (w.handles.size === 0) {
+      this.free.add(index)
+      this.wants.delete(index)
+      return this.free.size === 1 && this.wants.size === MAX - 1
+    }
+
+    return false
+  }
+
+  destroy() {
+    if (this.destroyed) return
+    this.destroyed = true
+
+    for (const w of this.wants.values()) {
+      for (const handle of w.handles) {
+        handle.removeWant(w)
+      }
+    }
+
+    this.wants.clear()
+  }
+
+  _unwant() {
+    for (const index of this.free) {
+      this.free.delete(index)
+      return { start: index * BATCH, length: BATCH, any: false }
+    }
+    return null
+  }
+}
+
+class RemoteWants {
+  constructor() {
+    this.any = null
+    this.all = false
+    this.size = 0
+    this.batches = []
+  }
+
+  hasAny(start, length) {
+    for (let i = 0; i < this.any.length; i++) {
+      const a = this.any[i]
+      const e = start + length
+      const end = a.start + a.length
+
+      if (a.start <= start && start < end) return true
+      if (a.start < e && e <= end) return true
+    }
+
+    return this.all
+  }
+
+  hasRange(start, length) {
+    if (this.any !== null && this.hasAny(start, length)) return true
+    if (length === 1) return this.has(start)
+
+    let smallest = -1
+
+    for (let i = 0; i < this.batches.length; i++) {
+      const b = this.batches[i]
+      if (b.length < smallest || smallest === -1) smallest = b.length
+    }
+
+    if (smallest === -1) return this.all
+
+    const r = start & (smallest - 1)
+    const end = start + length
+
+    let max = 3
+
+    for (let i = start - r; i < end; i += smallest) {
+      if (max-- === 0) return true // just to save work
+      if (this.has(i)) return true
+    }
+
+    return this.all
+  }
+
+  has(index) {
+    for (let i = 0; i < this.batches.length; i++) {
+      const b = this.batches[i]
+      const block = (index - (index & (b.length - 1))) / b.length
+      if (b.ranges.has(block)) return true
+    }
+
+    return this.all
+  }
+
+  add(range) {
+    if (range.any) {
+      if (this.any === null) this.any = []
+      if (this.any.length < MAX_ANY) this.any.push(range)
+      else this.all = true
+      return true
+    }
+
+    if (range.length > MAX_RANGE) return false
+
+    if (!validateBatchRange(range)) {
+      this.all = true
+      return true
+    }
+
+    if (this.size >= MAX) {
+      this.all = true
+      return true
+    }
+
+    for (let i = 0; i < this.batches.length; i++) {
+      const b = this.batches[i]
+      if (b.length === range.length) {
+        b.ranges.add(range.start / range.length)
+        this.size++
+        return true
+      }
+    }
+
+    if (this.batches.length >= MAX_REMOTE_BATCHES) {
+      this.all = true
+      return true
+    }
+
+    this.batches.push({
+      length: range.length,
+      ranges: new Set([range.start / range.length])
+    })
+    this.size++
+
+    return true
+  }
+
+  remove(range) {
+    if (range.any) {
+      if (this.any === null) return false
+
+      for (let i = 0; i < this.any.length; i++) {
+        const a = this.any[i]
+
+        if (a.start === range.start && a.length === range.length) {
+          const head = this.any.pop()
+          if (head !== a) this.any[i] = head
+          return true
+        }
+      }
+
+      return false
+    }
+
+    if (!validateBatchRange(range)) return false
+
+    for (let i = 0; i < this.batches.length; i++) {
+      const b = this.batches[i]
+      if (b.length !== range.length) continue
+
+      const size = b.ranges.size
+      b.ranges.delete(range.start / range.length)
+      if (b.ranges.size !== size) this.size--
+
+      if (b.ranges.size === 0) {
+        const head = this.batches.pop()
+        if (head !== b) this.batches[i] = head
+      }
+
+      return true
+    }
+
+    return false
+  }
+}
+
+exports.LocalWants = LocalWants
+exports.RemoteWants = RemoteWants
+exports.WANT_BATCH = BATCH
+
+function validateBatchRange(range) {
+  if (range.length > MAX_RANGE || range.length < MIN_RANGE) {
+    return false
+  }
+  // check if power of two
+  if ((range.length & (range.length - 1)) !== 0) {
+    return false
+  }
+  // start must be a multiple of the length
+  if (range.start & (range.length - 1)) {
+    return false
+  }
+
+  return true
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.22.1",
+  "version": "11.23.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {