hypercore 10.3.2 → 10.4.1

package/index.js

@@ -429,29 +429,30 @@ module.exports = class Hypercore extends EventEmitter {
     const protocolStream = Hypercore.createProtocolStream(isInitiator, opts)
     const noiseStream = protocolStream.noiseStream
     const protocol = noiseStream.userData
+    const useSession = !!opts.session
 
-    this._attachToMuxer(protocol, opts)
+    this._attachToMuxer(protocol, useSession)
 
     return protocolStream
   }
 
-  _attachToMuxer (mux, opts) {
-    // If the user wants to, we can make this replication run in a session
-    // that way the core wont close "under them" during replication
-    if (opts.session) {
-      const s = this.session()
-      mux.stream.on('close', () => s.close().catch(noop))
-    }
-
+  _attachToMuxer (mux, useSession) {
     if (this.opened) {
-      this.replicator.attachTo(mux)
+      this._attachToMuxerOpened(mux, useSession)
     } else {
-      this.opening.then(() => this.replicator.attachTo(mux), mux.destroy.bind(mux))
+      this.opening.then(this._attachToMuxerOpened.bind(this, mux, useSession), mux.destroy.bind(mux))
     }
 
     return mux
   }
 
+  _attachToMuxerOpened (mux, useSession) {
+    // If the user wants to, we can make this replication run in a session
+    // that way the core wont close "under them" during replication
+    const session = useSession ? this.session() : null
+    this.replicator.attachTo(mux, session)
+  }
+
   get discoveryKey () {
     return this.replicator === null ? null : this.replicator.discoveryKey
   }
@@ -616,7 +617,7 @@ module.exports = class Hypercore extends EventEmitter {
   async info () {
     if (this.opened === false) await this.opening
 
-    return Info.from(this.core, this.padding, this._snapshot)
+    return Info.from(this)
   }
 
   async update (opts) {

package/lib/bitfield.js

@@ -1,99 +1,244 @@
 const BigSparseArray = require('big-sparse-array')
 const b4a = require('b4a')
-const bits = require('bits-to-bytes')
-
-class FixedBitfield {
-  constructor (index, bitfield) {
+const quickbit = require('./compat').quickbit
+
+const BITS_PER_PAGE = 32768
+const BYTES_PER_PAGE = BITS_PER_PAGE / 8
+const WORDS_PER_PAGE = BYTES_PER_PAGE / 4
+const BITS_PER_SEGMENT = 2097152
+const BYTES_PER_SEGMENT = BITS_PER_SEGMENT / 8
+const WORDS_PER_SEGMENT = BYTES_PER_SEGMENT / 4
+const INITIAL_WORDS_PER_SEGMENT = 1024
+const PAGES_PER_SEGMENT = BITS_PER_SEGMENT / BITS_PER_PAGE
+const SEGMENT_GROWTH_FACTOR = 4
+
+class BitfieldPage {
+  constructor (index, segment) {
     this.dirty = false
     this.index = index
-    this.bitfield = bitfield
+    this.bitfield = null
+    this.segment = segment
+
+    segment.add(this)
+  }
+
+  get tree () {
+    return this.segment.tree
   }
 
   get (index) {
-    return bits.get(this.bitfield, index)
+    return quickbit.get(this.bitfield, index)
   }
 
   set (index, val) {
-    return bits.set(this.bitfield, index, val)
+    if (quickbit.set(this.bitfield, index, val)) {
+      this.tree.update(this.offset * 8 + index)
+    }
   }
 
   setRange (start, length, val) {
-    // Using fill instead of setRange is ~2 orders of magnitude faster, but does
-    // have the downside of not being able to tell if any bits actually changed.
-    bits.fill(this.bitfield, val, start, start + length)
-    return true
+    quickbit.fill(this.bitfield, val, start, start + length)
+
+    let i = Math.floor(start / 32)
+    const n = i + Math.ceil(length / 32)
+
+    while (i < n) this.tree.update(this.offset * 8 + i++ * 32)
   }
 
-  firstSet (position) {
-    return bits.indexOf(this.bitfield, true, position)
+  findFirst (val, position) {
+    return quickbit.findFirst(this.bitfield, val, position)
   }
 
-  lastSet (position) {
-    return bits.lastIndexOf(this.bitfield, true, position)
+  findLast (val, position) {
+    return quickbit.findLast(this.bitfield, val, position)
+  }
+}
+
+class BitfieldSegment {
+  constructor (index, bitfield) {
+    this.index = index
+    this.offset = index * BYTES_PER_SEGMENT
+    this.tree = quickbit.Index.from(bitfield)
+    this.pages = new Array(PAGES_PER_SEGMENT)
+  }
+
+  get bitfield () {
+    return this.tree.field
+  }
+
+  add (page) {
+    const i = page.index - this.index * PAGES_PER_SEGMENT
+    this.pages[i] = page
+
+    const start = i * WORDS_PER_PAGE
+    const end = start + WORDS_PER_PAGE
+
+    if (end >= this.bitfield.length) this.reallocate(end)
+
+    page.bitfield = this.bitfield.subarray(start, end)
+  }
+
+  reallocate (length) {
+    let target = this.bitfield.length
+    while (target < length) target *= SEGMENT_GROWTH_FACTOR
+
+    const bitfield = new Uint32Array(target)
+    bitfield.set(this.bitfield)
+
+    this.tree = quickbit.Index.from(bitfield)
+
+    for (let i = 0; i < this.pages.length; i++) {
+      const page = this.pages[i]
+      if (!page) continue
+
+      const start = i * WORDS_PER_PAGE
+      const end = start + WORDS_PER_PAGE
+
+      page.bitfield = bitfield.subarray(start, end)
+    }
+  }
+
+  findFirst (val, position) {
+    position = this.tree.skipFirst(!val, position)
+
+    const j = position & (BITS_PER_PAGE - 1)
+    const i = (position - j) / BITS_PER_PAGE
+
+    if (i >= PAGES_PER_SEGMENT) return -1
+
+    const p = this.pages[i]
+
+    if (p) {
+      const index = p.findFirst(val, j)
+
+      if (index !== -1) {
+        return i * BITS_PER_PAGE + index
+      }
+    }
+
+    return -1
+  }
+
+  findLast (val, position) {
+    position = this.tree.skipLast(!val, position)
+
+    const j = position & (BITS_PER_PAGE - 1)
+    const i = (position - j) / BITS_PER_PAGE
+
+    if (i >= PAGES_PER_SEGMENT) return -1
+
+    const p = this.pages[i]
+
+    if (p) {
+      const index = p.findLast(val, j)
+
+      if (index !== -1) {
+        return i * BITS_PER_PAGE + index
+      }
+    }
+
+    return -1
   }
 }
 
 module.exports = class Bitfield {
-  constructor (storage, buf) {
-    this.pageSize = 32768
-    this.pages = new BigSparseArray()
+  constructor (storage, buffer) {
     this.unflushed = []
     this.storage = storage
-    this.resumed = !!(buf && buf.byteLength >= 4)
+    this.resumed = !!(buffer && buffer.byteLength >= 4)
+
+    this._pages = new BigSparseArray()
+    this._segments = new BigSparseArray()
+
+    const view = this.resumed
+      ? new Uint32Array(
+        buffer.buffer,
+        buffer.byteOffset,
+        Math.floor(buffer.byteLength / 4)
+      )
+      : new Uint32Array(INITIAL_WORDS_PER_SEGMENT)
+
+    for (let i = 0; i < view.length; i += WORDS_PER_SEGMENT) {
+      let bitfield = view.subarray(i, i + (WORDS_PER_SEGMENT))
+      let length = WORDS_PER_SEGMENT
+
+      if (i === 0) {
+        length = INITIAL_WORDS_PER_SEGMENT
+        while (length < bitfield.length) length *= SEGMENT_GROWTH_FACTOR
+      }
 
-    const all = this.resumed
-      ? new Uint32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4))
-      : new Uint32Array(1024)
+      if (bitfield.length !== length) {
+        const copy = new Uint32Array(length)
+        copy.set(bitfield, 0)
+        bitfield = copy
+      }
+
+      const segment = new BitfieldSegment(i / (WORDS_PER_SEGMENT), bitfield)
+      this._segments.set(segment.index, segment)
 
-    for (let i = 0; i < all.length; i += 1024) {
-      const bitfield = ensureSize(all.subarray(i, i + 1024), 1024)
-      const page = new FixedBitfield(i / 1024, bitfield)
-      this.pages.set(page.index, page)
+      for (let j = 0; j < bitfield.length; j += WORDS_PER_PAGE) {
+        const page = new BitfieldPage((i + j) / WORDS_PER_PAGE, segment)
+        this._pages.set(page.index, page)
+      }
     }
   }
 
   get (index) {
-    const j = index & (this.pageSize - 1)
-    const i = (index - j) / this.pageSize
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
 
-    const p = this.pages.get(i)
+    const p = this._pages.get(i)
 
     return p ? p.get(j) : false
   }
 
   set (index, val) {
-    const j = index & (this.pageSize - 1)
-    const i = (index - j) / this.pageSize
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
 
-    let p = this.pages.get(i)
+    let p = this._pages.get(i)
 
     if (!p && val) {
-      p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
+      const k = Math.floor(i / PAGES_PER_SEGMENT)
+      const s = this._segments.get(k) || this._segments.set(k, new BitfieldSegment(k, new Uint32Array(k === 0 ? INITIAL_WORDS_PER_SEGMENT : WORDS_PER_SEGMENT)))
+
+      p = this._pages.set(i, new BitfieldPage(i, s))
     }
 
-    if (p && p.set(j, val) && !p.dirty) {
-      p.dirty = true
-      this.unflushed.push(p)
+    if (p) {
+      p.set(j, val)
+
+      if (!p.dirty) {
+        p.dirty = true
+        this.unflushed.push(p)
+      }
     }
   }
 
   setRange (start, length, val) {
-    let j = start & (this.pageSize - 1)
-    let i = (start - j) / this.pageSize
+    let j = start & (BITS_PER_PAGE - 1)
+    let i = (start - j) / BITS_PER_PAGE
 
     while (length > 0) {
-      let p = this.pages.get(i)
+      let p = this._pages.get(i)
 
       if (!p && val) {
-        p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
+        const k = Math.floor(i / PAGES_PER_SEGMENT)
+        const s = this._segments.get(k) || this._segments.set(k, new BitfieldSegment(k, new Uint32Array(k === 0 ? INITIAL_WORDS_PER_SEGMENT : WORDS_PER_SEGMENT)))
+
+        p = this._pages.set(i, new BitfieldPage(i, s))
       }
 
-      const end = Math.min(j + length, this.pageSize)
+      const end = Math.min(j + length, BITS_PER_PAGE)
       const range = end - j
 
-      if (p && p.setRange(j, range, val) && !p.dirty) {
-        p.dirty = true
-        this.unflushed.push(p)
+      if (p) {
+        p.setRange(j, range, val)
+
+        if (!p.dirty) {
+          p.dirty = true
+          this.unflushed.push(p)
+        }
       }
 
       j = 0
@@ -102,18 +247,18 @@ module.exports = class Bitfield {
     }
   }
 
-  firstSet (position) {
-    let j = position & (this.pageSize - 1)
-    let i = (position - j) / this.pageSize
+  findFirst (val, position) {
+    let j = position & (BITS_PER_SEGMENT - 1)
+    let i = (position - j) / BITS_PER_SEGMENT
 
-    while (i < this.pages.factor) {
-      const p = this.pages.get(i)
+    while (i < this._segments.maxLength) {
+      const s = this._segments.get(i)
 
-      if (p) {
-        const index = p.firstSet(j)
+      if (s) {
+        const index = s.findFirst(val, j)
 
         if (index !== -1) {
-          return i * this.pageSize + index
+          return i * BITS_PER_SEGMENT + index
         }
       }
 
@@ -124,33 +269,72 @@ module.exports = class Bitfield {
     return -1
   }
 
-  lastSet (position) {
-    let j = position & (this.pageSize - 1)
-    let i = (position - j) / this.pageSize
+  firstSet (position) {
+    return this.findFirst(true, position)
+  }
+
+  firstUnset (position) {
+    return this.findFirst(false, position)
+  }
+
+  findLast (val, position) {
+    let j = position & (BITS_PER_SEGMENT - 1)
+    let i = (position - j) / BITS_PER_SEGMENT
 
     while (i >= 0) {
-      const p = this.pages.get(i)
+      const s = this._segments.get(i)
 
-      if (p) {
-        const index = p.lastSet(j)
+      if (s) {
+        const index = s.findLast(val, j)
 
         if (index !== -1) {
-          return i * this.pageSize + index
+          return i * BITS_PER_SEGMENT + index
         }
       }
 
-      j = this.pageSize - 1
+      j = BITS_PER_SEGMENT - 1
       i--
     }
 
     return -1
   }
 
+  lastSet (position) {
+    return this.findLast(true, position)
+  }
+
+  lastUnset (position) {
+    return this.findLast(false, position)
+  }
+
+  * want (start, length) {
+    const j = start & (BITS_PER_SEGMENT - 1)
+    let i = (start - j) / BITS_PER_SEGMENT
+
+    while (length > 0) {
+      const s = this._segments.get(i)
+
+      if (s) {
+        // We always send at least 4 KiB worth of bitfield in a want, rounding
+        // to the nearest 4 KiB.
+        const end = ceilTo(clamp(length / 8, 4096, BYTES_PER_SEGMENT), 4096)
+
+        yield {
+          start: i * BITS_PER_SEGMENT,
+          bitfield: s.bitfield.subarray(0, end / 4)
+        }
+      }
+
+      i++
+      length -= BITS_PER_SEGMENT
+    }
+  }
+
   clear () {
     return new Promise((resolve, reject) => {
       this.storage.truncate(0, (err) => {
         if (err) return reject(err)
-        this.pages = new BigSparseArray()
+        this._pages = new BigSparseArray()
         this.unflushed = []
         resolve()
       })
@@ -182,7 +366,7 @@ module.exports = class Bitfield {
         )
 
         page.dirty = false
-        this.storage.write(page.index * 4096, buf, done)
+        this.storage.write(page.index * BYTES_PER_PAGE, buf, done)
       }
 
       function done (err) {
@@ -195,12 +379,13 @@ module.exports = class Bitfield {
     })
   }
 
-  static open (storage) {
+  static open (storage, tree = null) {
     return new Promise((resolve, reject) => {
       storage.stat((err, st) => {
         if (err) return resolve(new Bitfield(storage, null))
-        const size = st.size - (st.size & 3)
+        let size = st.size - (st.size & 3)
         if (!size) return resolve(new Bitfield(storage, null))
+        if (tree) size = Math.min(size, ceilTo(tree.length / 8, 4096))
         storage.read(0, size, (err, data) => {
           if (err) return reject(err)
           resolve(new Bitfield(storage, data))
@@ -210,9 +395,12 @@ module.exports = class Bitfield {
   }
 }
 
-function ensureSize (uint32, size) {
-  if (uint32.byteLength === size) return uint32
-  const a = new Uint32Array(1024)
-  a.set(uint32, 0)
-  return a
+function clamp (n, min, max) {
+  return Math.min(Math.max(n, min), max)
+}
+
+function ceilTo (n, multiple = 1) {
+  const remainder = n % multiple
+  if (remainder === 0) return n
+  return n + multiple - remainder
 }

package/lib/compat.js

@@ -0,0 +1,11 @@
+// Export the appropriate version of `quickbit-universal` as the plain import
+// may resolve to an older version in some environments
+let quickbit = require('quickbit-universal')
+if (
+  typeof quickbit.findFirst !== 'function' ||
+  typeof quickbit.findLast !== 'function'
+) {
+  // This should always load the fallback from the locally installed version
+  quickbit = require('quickbit-universal/fallback')
+}
+exports.quickbit = quickbit

package/lib/core.js

@@ -108,7 +108,7 @@ module.exports = class Core {
     }
 
     const tree = await MerkleTree.open(treeFile, { crypto, ...header.tree })
-    const bitfield = await Bitfield.open(bitfieldFile)
+    const bitfield = await Bitfield.open(bitfieldFile, tree)
     const blocks = new BlockStore(dataFile, tree)
 
     if (overwrite) {

package/lib/info.js

@@ -1,24 +1,23 @@
 module.exports = class Info {
   constructor (opts = {}) {
+    this.key = opts.key
+    this.discoveryKey = opts.discoveryKey
     this.length = opts.length || 0
     this.contiguousLength = opts.contiguousLength || 0
     this.byteLength = opts.byteLength || 0
+    this.fork = opts.fork || 0
     this.padding = opts.padding || 0
   }
 
-  static async from (core, padding, snapshot) {
+  static async from (session) {
     return new Info({
-      key: core.key,
-      discoveryKey: core.discoveryKey,
-      length: snapshot
-        ? snapshot.length
-        : core.tree.length,
-      contiguousLength: core.header.contiguousLength,
-      byteLength: snapshot
-        ? snapshot.byteLength
-        : (core.tree.byteLength - (core.tree.length * padding)),
-      fork: core.tree.fork,
-      padding
+      key: session.key,
+      discoveryKey: session.discoveryKey,
+      length: session.length,
+      contiguousLength: session.contiguousLength,
+      byteLength: session.byteLength,
+      fork: session.fork,
+      padding: session.padding
     })
   }
 }

package/lib/oplog.js

@@ -1,6 +1,6 @@
 const cenc = require('compact-encoding')
 const b4a = require('b4a')
-const crc32 = require('crc32-universal')
+const { crc32 } = require('crc-universal')
 
 module.exports = class Oplog {
   constructor (storage, { pageSize = 4096, headerEncoding = cenc.raw, entryEncoding = cenc.raw } = {}) {

package/lib/remote-bitfield.js

@@ -1,45 +1,275 @@
 const BigSparseArray = require('big-sparse-array')
-const bits = require('bits-to-bytes')
+const quickbit = require('./compat').quickbit
+
+const BITS_PER_PAGE = 32768
+const BYTES_PER_PAGE = BITS_PER_PAGE / 8
+const WORDS_PER_PAGE = BYTES_PER_PAGE / 4
+const BITS_PER_SEGMENT = 2097152
+const BYTES_PER_SEGMENT = BITS_PER_SEGMENT / 8
+const PAGES_PER_SEGMENT = BITS_PER_SEGMENT / BITS_PER_PAGE
+
+class RemoteBitfieldPage {
+  constructor (index, bitfield, segment) {
+    this.index = index
+    this.offset = index * BYTES_PER_PAGE - segment.offset
+    this.bitfield = bitfield
+    this.segment = segment
+
+    segment.add(this)
+  }
+
+  get tree () {
+    return this.segment.tree
+  }
+
+  get (index) {
+    return quickbit.get(this.bitfield, index)
+  }
+
+  set (index, val) {
+    if (quickbit.set(this.bitfield, index, val)) {
+      this.tree.update(this.offset * 8 + index)
+    }
+  }
+
+  setRange (start, length, val) {
+    quickbit.fill(this.bitfield, val, start, start + length)
+
+    let i = Math.floor(start / 32)
+    const n = i + Math.ceil(length / 32)
+
+    while (i < n) this.tree.update(this.offset * 8 + i++ * 32)
+  }
+
+  findFirst (val, position) {
+    return quickbit.findFirst(this.bitfield, val, position)
+  }
+
+  findLast (val, position) {
+    return quickbit.findLast(this.bitfield, val, position)
+  }
+
+  insert (start, bitfield) {
+    this.bitfield.set(bitfield, start / 32)
+  }
+}
+
+class RemoteBitfieldSegment {
+  constructor (index) {
+    this.index = index
+    this.offset = index * BYTES_PER_SEGMENT
+    this.tree = quickbit.Index.from([])
+    this.pages = new Array(PAGES_PER_SEGMENT)
+  }
+
+  get chunks () {
+    return this.tree.chunks
+  }
+
+  add (page) {
+    this.pages[page.index - this.index * PAGES_PER_SEGMENT] = page
+
+    const chunk = { field: page.bitfield, offset: page.offset }
+
+    this.chunks.push(chunk)
+
+    for (let i = this.chunks.length - 2; i >= 0; i--) {
+      const prev = this.chunks[i]
+      if (prev.offset <= chunk.offset) break
+      this.chunks[i] = chunk
+      this.chunks[i + 1] = prev
+    }
+  }
+
+  findFirst (val, position) {
+    position = this.tree.skipFirst(!val, position)
+
+    const j = position & (BITS_PER_PAGE - 1)
+    const i = (position - j) / BITS_PER_PAGE
+
+    if (i >= PAGES_PER_SEGMENT) return -1
+
+    const p = this.pages[i]
+
+    if (p) {
+      const index = p.findFirst(val, j)
+
+      if (index !== -1) {
+        return i * BITS_PER_PAGE + index
+      }
+    }
+
+    return -1
+  }
+
+  findLast (val, position) {
+    position = this.tree.skipLast(!val, position)
+
+    const j = position & (BITS_PER_PAGE - 1)
+    const i = (position - j) / BITS_PER_PAGE
+
+    if (i >= PAGES_PER_SEGMENT) return -1
+
+    const p = this.pages[i]
+
+    if (p) {
+      const index = p.findLast(val, j)
+
+      if (index !== -1) {
+        return i * BITS_PER_PAGE + index
+      }
+    }
+
+    return -1
+  }
+}
 
 module.exports = class RemoteBitfield {
   constructor () {
-    this.pageSize = 32768
-    this.pages = new BigSparseArray()
+    this._pages = new BigSparseArray()
+    this._segments = new BigSparseArray()
   }
 
   get (index) {
-    const j = index & (this.pageSize - 1)
-    const i = (index - j) / this.pageSize
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
 
-    const p = this.pages.get(i)
+    const p = this._pages.get(i)
 
-    return p ? bits.get(p, j) : false
+    return p ? p.get(j) : false
   }
 
   set (index, val) {
-    const j = index & (this.pageSize - 1)
-    const i = (index - j) / this.pageSize
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
 
-    const p = this.pages.get(i) || this.pages.set(i, new Uint32Array(1024))
+    let p = this._pages.get(i)
 
-    bits.set(p, j, val)
+    if (!p && val) {
+      const k = Math.floor(i / PAGES_PER_SEGMENT)
+      const s = this._segments.get(k) || this._segments.set(k, new RemoteBitfieldSegment(k))
+
+      p = this._pages.set(i, new RemoteBitfieldPage(i, new Uint32Array(WORDS_PER_PAGE), s))
+    }
+
+    if (p) p.set(j, val)
   }
 
   setRange (start, length, val) {
-    let j = start & (this.pageSize - 1)
-    let i = (start - j) / this.pageSize
+    let j = start & (BITS_PER_PAGE - 1)
+    let i = (start - j) / BITS_PER_PAGE
 
     while (length > 0) {
-      const p = this.pages.get(i) || this.pages.set(i, new Uint32Array(1024))
+      let p = this._pages.get(i)
+
+      if (!p && val) {
+        const k = Math.floor(i / PAGES_PER_SEGMENT)
+        const s = this._segments.get(k) || this._segments.set(k, new RemoteBitfieldSegment(k))
+
+        p = this._pages.set(i, new RemoteBitfieldPage(i, new Uint32Array(WORDS_PER_PAGE), s))
+      }
 
-      const end = Math.min(j + length, this.pageSize)
+      const end = Math.min(j + length, BITS_PER_PAGE)
       const range = end - j
 
-      bits.fill(p, val, j, end)
+      if (p) p.setRange(j, range, val)
 
       j = 0
       i++
       length -= range
     }
   }
+
+  findFirst (val, position) {
+    let j = position & (BITS_PER_SEGMENT - 1)
+    let i = (position - j) / BITS_PER_SEGMENT
+
+    while (i < this._segments.maxLength) {
+      const s = this._segments.get(i)
+
+      if (s) {
+        const index = s.findFirst(val, j)
+
+        if (index !== -1) {
+          return i * BITS_PER_SEGMENT + index
+        }
+      }
+
+      j = 0
+      i++
+    }
+
+    return -1
+  }
+
+  firstSet (position) {
+    return this.findFirst(true, position)
+  }
+
+  firstUnset (position) {
+    return this.findFirst(false, position)
+  }
+
+  findLast (val, position) {
+    let j = position & (BITS_PER_SEGMENT - 1)
+    let i = (position - j) / BITS_PER_SEGMENT
+
+    while (i >= 0) {
+      const s = this._segments.get(i)
+
+      if (s) {
+        const index = s.findLast(val, j)
+
+        if (index !== -1) {
+          return i * BITS_PER_SEGMENT + index
+        }
+      }
+
+      j = BITS_PER_SEGMENT - 1
+      i--
+    }
+
+    return -1
+  }
+
+  lastSet (position) {
+    return this.findLast(true, position)
+  }
+
+  lastUnset (position) {
+    return this.findLast(false, position)
+  }
+
+  insert (start, bitfield) {
+    if (start % 32 !== 0) return false
+
+    let length = bitfield.byteLength * 8
+
+    let j = start & (BITS_PER_PAGE - 1)
+    let i = (start - j) / BITS_PER_PAGE
+
+    while (length > 0) {
+      let p = this._pages.get(i)
+
+      if (!p) {
+        const k = Math.floor(i / PAGES_PER_SEGMENT)
+        const s = this._segments.get(k) || this._segments.set(k, new RemoteBitfieldSegment(k))
+
+        p = this._pages.set(i, new RemoteBitfieldPage(i, new Uint32Array(WORDS_PER_PAGE), s))
+      }
+
+      const end = Math.min(j + length, BITS_PER_PAGE)
+      const range = end - j
+
+      p.insert(j, bitfield.subarray(0, range / 32))
+
+      bitfield = bitfield.subarray(range / 32)
+
+      j = 0
+      i++
+      length -= range
+    }
+
+    return true
+  }
 }

package/lib/replicator.js

@@ -243,13 +243,15 @@ class BlockTracker {
 }
 
 class Peer {
-  constructor (replicator, protomux, channel) {
+  constructor (replicator, protomux, channel, session) {
     this.core = replicator.core
     this.replicator = replicator
     this.stream = protomux.stream
     this.protomux = protomux
     this.remotePublicKey = this.stream.remotePublicKey
 
+    this.session = session
+
     this.channel = channel
     this.channel.userData = this
 
@@ -376,6 +378,8 @@ class Peer {
   }
 
   onclose (isRemote) {
+    if (this.session) this.session.close().catch(noop)
+
     if (this.remoteOpened === false) {
       this.replicator._ifAvailable--
       this.replicator.updateAll()
@@ -570,17 +574,9 @@ class Peer {
   }
 
   onbitfield ({ start, bitfield }) {
-    // TODO: tweak this to be more generic
-
-    if (bitfield.length < 1024) {
-      const buf = b4a.from(bitfield.buffer, bitfield.byteOffset, bitfield.byteLength)
-      const bigger = b4a.concat([buf, b4a.alloc(4096 - buf.length)])
-      bitfield = new Uint32Array(bigger.buffer, bigger.byteOffset, 1024)
+    if (this.remoteBitfield.insert(start, bitfield)) {
+      this._update()
     }
-
-    this.remoteBitfield.pages.set(start / this.core.bitfield.pageSize, bitfield)
-
-    this._update()
   }
 
   onrange ({ drop, start, length }) {
@@ -1219,8 +1215,7 @@ module.exports = class Replicator {
       for (let i = 0; i < this._ranges.length; i++) {
         const r = this._ranges[i]
 
-        while ((r.end === -1 || r.start < r.end) && this.core.bitfield.get(mapIndex(r.blocks, r.start)) === true) r.start++
-        while (r.start < r.end && this.core.bitfield.get(mapIndex(r.blocks, r.end - 1)) === true) r.end--
+        clampRange(this.core, r)
 
         if (r.end !== -1 && r.start >= r.end) {
           this._resolveRangeRequest(r, i--)
@@ -1332,18 +1327,8 @@ module.exports = class Replicator {
 
     peer.protomux.cork()
 
-    let i = Math.floor(start / this.core.bitfield.pageSize)
-    const n = Math.ceil((start + length) / this.core.bitfield.pageSize)
-
-    for (; i < n; i++) {
-      const p = this.core.bitfield.pages.get(i)
-
-      if (p) {
-        peer.wireBitfield.send({
-          start: i * this.core.bitfield.pageSize,
-          bitfield: p.bitfield
-        })
-      }
+    for (const msg of this.core.bitfield.want(start, length)) {
+      peer.wireBitfield.send(msg)
     }
 
     peer.protomux.uncork()
@@ -1525,21 +1510,21 @@ module.exports = class Replicator {
     this._maybeResolveIfAvailableRanges()
   }
 
-  attachTo (protomux) {
-    const makePeer = this._makePeer.bind(this, protomux)
+  attachTo (protomux, session) {
+    const makePeer = this._makePeer.bind(this, protomux, session)
 
     protomux.pair({ protocol: 'hypercore/alpha', id: this.discoveryKey }, makePeer)
-
     this._ifAvailable++
     protomux.stream.opened.then((opened) => {
       this._ifAvailable--
       if (opened) makePeer()
+      else if (session) session.close().catch(noop)
       this._checkUpgradeIfAvailable()
     })
   }
 
-  _makePeer (protomux) {
-    if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) return false
+  _makePeer (protomux, session) {
+    if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) return onnochannel()
 
     const channel = protomux.createChannel({
       userData: null,
@@ -1563,9 +1548,9 @@ module.exports = class Replicator {
       onclose: onwireclose
     })
 
-    if (channel === null) return false
+    if (channel === null) return onnochannel()
 
-    const peer = new Peer(this, protomux, channel)
+    const peer = new Peer(this, protomux, channel, session)
     const stream = protomux.stream
 
     peer.channel.open({
@@ -1573,6 +1558,11 @@ module.exports = class Replicator {
     })
 
     return true
+
+    function onnochannel () {
+      if (session) session.close().catch(noop)
+      return false
+    }
   }
 }
 
@@ -1592,16 +1582,34 @@ function removeInflight (inf, req) {
   return true
 }
 
-function mapIndex (blocks, index) {
-  return blocks === null ? index : blocks[index]
-}
-
 function noop () {}
 
 function toLength (start, end) {
   return end === -1 ? -1 : (end < start ? 0 : end - start)
 }
 
+function clampRange (core, r) {
+  if (r.blocks === null) {
+    const start = core.bitfield.firstUnset(r.start)
+
+    if (r.end === -1) {
+      r.start = start === -1 ? core.tree.length : start
+    } else {
+      const end = core.bitfield.lastUnset(r.end - 1) + 1
+
+      if (start === -1) {
+        r.start = r.end
+      } else {
+        r.start = start
+        r.end = end
+      }
+    }
+  } else {
+    while (r.start < r.end && core.bitfield.get(r.blocks[r.start])) r.start++
+    while (r.start < r.end && core.bitfield.get(r.blocks[r.end - 1])) r.end--
+  }
+}
+
 function onwireopen (m, c) {
   return c.userData.onopen(m)
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.3.2",
+  "version": "10.4.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -36,14 +36,14 @@
     "@hyperswarm/secret-stream": "^6.0.0",
     "b4a": "^1.1.0",
     "big-sparse-array": "^1.0.2",
-    "bits-to-bytes": "^1.3.0",
     "compact-encoding": "^2.11.0",
-    "crc32-universal": "^1.0.1",
+    "crc-universal": "^1.0.2",
     "events": "^3.3.0",
     "flat-tree": "^1.9.0",
     "hypercore-crypto": "^3.2.1",
     "is-options": "^1.0.1",
     "protomux": "^3.4.0",
+    "quickbit-universal": "^2.0.3",
     "random-access-file": "^4.0.0",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",