hypercore 10.4.0 → 10.4.1

package/lib/bitfield.js

@@ -2,16 +2,28 @@ const BigSparseArray = require('big-sparse-array')
 const b4a = require('b4a')
 const quickbit = require('./compat').quickbit
 
-const BITS_PER_PAGE = 2097152
+const BITS_PER_PAGE = 32768
 const BYTES_PER_PAGE = BITS_PER_PAGE / 8
 const WORDS_PER_PAGE = BYTES_PER_PAGE / 4
+const BITS_PER_SEGMENT = 2097152
+const BYTES_PER_SEGMENT = BITS_PER_SEGMENT / 8
+const WORDS_PER_SEGMENT = BYTES_PER_SEGMENT / 4
+const INITIAL_WORDS_PER_SEGMENT = 1024
+const PAGES_PER_SEGMENT = BITS_PER_SEGMENT / BITS_PER_PAGE
+const SEGMENT_GROWTH_FACTOR = 4
 
 class BitfieldPage {
-  constructor (index, bitfield) {
+  constructor (index, segment) {
     this.dirty = false
     this.index = index
-    this.bitfield = bitfield
-    this.tree = quickbit.Index.from(this.bitfield)
+    this.bitfield = null
+    this.segment = segment
+
+    segment.add(this)
+  }
+
+  get tree () {
+    return this.segment.tree
   }
 
   get (index) {
@@ -20,7 +32,7 @@ class BitfieldPage {
 
   set (index, val) {
     if (quickbit.set(this.bitfield, index, val)) {
-      this.tree.update(index)
+      this.tree.update(this.offset * 8 + index)
     }
   }
 
@@ -30,34 +42,144 @@ class BitfieldPage {
     let i = Math.floor(start / 32)
     const n = i + Math.ceil(length / 32)
 
-    while (i < n) this.tree.update(i++ * 32)
+    while (i < n) this.tree.update(this.offset * 8 + i++ * 32)
   }
 
   findFirst (val, position) {
-    return quickbit.findFirst(this.bitfield, val, this.tree.skipFirst(!val, position))
+    return quickbit.findFirst(this.bitfield, val, position)
+  }
+
+  findLast (val, position) {
+    return quickbit.findLast(this.bitfield, val, position)
+  }
+}
+
+class BitfieldSegment {
+  constructor (index, bitfield) {
+    this.index = index
+    this.offset = index * BYTES_PER_SEGMENT
+    this.tree = quickbit.Index.from(bitfield)
+    this.pages = new Array(PAGES_PER_SEGMENT)
+  }
+
+  get bitfield () {
+    return this.tree.field
+  }
+
+  add (page) {
+    const i = page.index - this.index * PAGES_PER_SEGMENT
+    this.pages[i] = page
+
+    const start = i * WORDS_PER_PAGE
+    const end = start + WORDS_PER_PAGE
+
+    if (end >= this.bitfield.length) this.reallocate(end)
+
+    page.bitfield = this.bitfield.subarray(start, end)
+  }
+
+  reallocate (length) {
+    let target = this.bitfield.length
+    while (target < length) target *= SEGMENT_GROWTH_FACTOR
+
+    const bitfield = new Uint32Array(target)
+    bitfield.set(this.bitfield)
+
+    this.tree = quickbit.Index.from(bitfield)
+
+    for (let i = 0; i < this.pages.length; i++) {
+      const page = this.pages[i]
+      if (!page) continue
+
+      const start = i * WORDS_PER_PAGE
+      const end = start + WORDS_PER_PAGE
+
+      page.bitfield = bitfield.subarray(start, end)
+    }
+  }
+
+  findFirst (val, position) {
+    position = this.tree.skipFirst(!val, position)
+
+    const j = position & (BITS_PER_PAGE - 1)
+    const i = (position - j) / BITS_PER_PAGE
+
+    if (i >= PAGES_PER_SEGMENT) return -1
+
+    const p = this.pages[i]
+
+    if (p) {
+      const index = p.findFirst(val, j)
+
+      if (index !== -1) {
+        return i * BITS_PER_PAGE + index
+      }
+    }
+
+    return -1
   }
 
   findLast (val, position) {
-    return quickbit.findLast(this.bitfield, val, this.tree.skipLast(!val, position))
+    position = this.tree.skipLast(!val, position)
+
+    const j = position & (BITS_PER_PAGE - 1)
+    const i = (position - j) / BITS_PER_PAGE
+
+    if (i >= PAGES_PER_SEGMENT) return -1
+
+    const p = this.pages[i]
+
+    if (p) {
+      const index = p.findLast(val, j)
+
+      if (index !== -1) {
+        return i * BITS_PER_PAGE + index
+      }
+    }
+
+    return -1
   }
 }
 
 module.exports = class Bitfield {
-  constructor (storage, buf) {
+  constructor (storage, buffer) {
     this.unflushed = []
     this.storage = storage
-    this.resumed = !!(buf && buf.byteLength >= 4)
+    this.resumed = !!(buffer && buffer.byteLength >= 4)
 
     this._pages = new BigSparseArray()
+    this._segments = new BigSparseArray()
+
+    const view = this.resumed
+      ? new Uint32Array(
+        buffer.buffer,
+        buffer.byteOffset,
+        Math.floor(buffer.byteLength / 4)
+      )
+      : new Uint32Array(INITIAL_WORDS_PER_SEGMENT)
+
+    for (let i = 0; i < view.length; i += WORDS_PER_SEGMENT) {
+      let bitfield = view.subarray(i, i + (WORDS_PER_SEGMENT))
+      let length = WORDS_PER_SEGMENT
+
+      if (i === 0) {
+        length = INITIAL_WORDS_PER_SEGMENT
+        while (length < bitfield.length) length *= SEGMENT_GROWTH_FACTOR
+      }
 
-    const all = this.resumed
-      ? new Uint32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4))
-      : new Uint32Array(WORDS_PER_PAGE)
+      if (bitfield.length !== length) {
+        const copy = new Uint32Array(length)
+        copy.set(bitfield, 0)
+        bitfield = copy
+      }
+
+      const segment = new BitfieldSegment(i / (WORDS_PER_SEGMENT), bitfield)
+      this._segments.set(segment.index, segment)
 
-    for (let i = 0; i < all.length; i += WORDS_PER_PAGE) {
-      const bitfield = ensureSize(all.subarray(i, i + (WORDS_PER_PAGE)), WORDS_PER_PAGE)
-      const page = new BitfieldPage(i / (WORDS_PER_PAGE), bitfield)
-      this._pages.set(page.index, page)
+      for (let j = 0; j < bitfield.length; j += WORDS_PER_PAGE) {
+        const page = new BitfieldPage((i + j) / WORDS_PER_PAGE, segment)
+        this._pages.set(page.index, page)
+      }
     }
   }
 
@@ -77,7 +199,10 @@ module.exports = class Bitfield {
     let p = this._pages.get(i)
 
     if (!p && val) {
-      p = this._pages.set(i, new BitfieldPage(i, new Uint32Array(WORDS_PER_PAGE)))
+      const k = Math.floor(i / PAGES_PER_SEGMENT)
+      const s = this._segments.get(k) || this._segments.set(k, new BitfieldSegment(k, new Uint32Array(k === 0 ? INITIAL_WORDS_PER_SEGMENT : WORDS_PER_SEGMENT)))
+
+      p = this._pages.set(i, new BitfieldPage(i, s))
     }
 
     if (p) {
@@ -98,7 +223,10 @@ module.exports = class Bitfield {
       let p = this._pages.get(i)
 
       if (!p && val) {
-        p = this._pages.set(i, new BitfieldPage(i, new Uint32Array(WORDS_PER_PAGE)))
+        const k = Math.floor(i / PAGES_PER_SEGMENT)
+        const s = this._segments.get(k) || this._segments.set(k, new BitfieldSegment(k, new Uint32Array(k === 0 ? INITIAL_WORDS_PER_SEGMENT : WORDS_PER_SEGMENT)))
+
+        p = this._pages.set(i, new BitfieldPage(i, s))
       }
 
       const end = Math.min(j + length, BITS_PER_PAGE)
@@ -120,17 +248,17 @@ module.exports = class Bitfield {
   }
 
   findFirst (val, position) {
-    let j = position & (BITS_PER_PAGE - 1)
-    let i = (position - j) / BITS_PER_PAGE
+    let j = position & (BITS_PER_SEGMENT - 1)
+    let i = (position - j) / BITS_PER_SEGMENT
 
-    while (i < this._pages.maxLength) {
-      const p = this._pages.get(i)
+    while (i < this._segments.maxLength) {
+      const s = this._segments.get(i)
 
-      if (p) {
-        const index = p.findFirst(val, j)
+      if (s) {
+        const index = s.findFirst(val, j)
 
         if (index !== -1) {
-          return i * BITS_PER_PAGE + index
+          return i * BITS_PER_SEGMENT + index
         }
       }
 
@@ -150,21 +278,21 @@ module.exports = class Bitfield {
   }
 
   findLast (val, position) {
-    let j = position & (BITS_PER_PAGE - 1)
-    let i = (position - j) / BITS_PER_PAGE
+    let j = position & (BITS_PER_SEGMENT - 1)
+    let i = (position - j) / BITS_PER_SEGMENT
 
     while (i >= 0) {
-      const p = this._pages.get(i)
+      const s = this._segments.get(i)
 
-      if (p) {
-        const index = p.findLast(val, j)
+      if (s) {
+        const index = s.findLast(val, j)
 
         if (index !== -1) {
-          return i * BITS_PER_PAGE + index
+          return i * BITS_PER_SEGMENT + index
         }
       }
 
-      j = BITS_PER_PAGE - 1
+      j = BITS_PER_SEGMENT - 1
       i--
     }
 
@@ -180,25 +308,25 @@ module.exports = class Bitfield {
   }
 
   * want (start, length) {
-    const j = start & (BITS_PER_PAGE - 1)
-    let i = (start - j) / BITS_PER_PAGE
+    const j = start & (BITS_PER_SEGMENT - 1)
+    let i = (start - j) / BITS_PER_SEGMENT
 
     while (length > 0) {
-      const p = this._pages.get(i)
+      const s = this._segments.get(i)
 
-      if (p) {
+      if (s) {
         // We always send at least 4 KiB worth of bitfield in a want, rounding
         // to the nearest 4 KiB.
-        const end = ceilTo(clamp(length / 8, 4096, BYTES_PER_PAGE), 4096)
+        const end = ceilTo(clamp(length / 8, 4096, BYTES_PER_SEGMENT), 4096)
 
         yield {
-          start: i * BITS_PER_PAGE,
-          bitfield: p.bitfield.subarray(0, end / 4)
+          start: i * BITS_PER_SEGMENT,
+          bitfield: s.bitfield.subarray(0, end / 4)
         }
       }
 
       i++
-      length -= BITS_PER_PAGE
+      length -= BITS_PER_SEGMENT
     }
   }
 
@@ -251,12 +379,13 @@ module.exports = class Bitfield {
     })
   }
 
-  static open (storage) {
+  static open (storage, tree = null) {
     return new Promise((resolve, reject) => {
       storage.stat((err, st) => {
         if (err) return resolve(new Bitfield(storage, null))
-        const size = st.size - (st.size & 3)
+        let size = st.size - (st.size & 3)
         if (!size) return resolve(new Bitfield(storage, null))
+        if (tree) size = Math.min(size, ceilTo(tree.length / 8, 4096))
         storage.read(0, size, (err, data) => {
           if (err) return reject(err)
           resolve(new Bitfield(storage, data))
@@ -266,13 +395,6 @@ module.exports = class Bitfield {
   }
 }
 
-function ensureSize (buffer, size) {
-  if (buffer.byteLength === size) return buffer
-  const copy = new Uint32Array(size)
-  copy.set(buffer, 0)
-  return copy
-}
-
 function clamp (n, min, max) {
   return Math.min(Math.max(n, min), max)
 }

package/lib/core.js

@@ -108,7 +108,7 @@ module.exports = class Core {
     }
 
     const tree = await MerkleTree.open(treeFile, { crypto, ...header.tree })
-    const bitfield = await Bitfield.open(bitfieldFile)
+    const bitfield = await Bitfield.open(bitfieldFile, tree)
     const blocks = new BlockStore(dataFile, tree)
 
     if (overwrite) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.4.0",
+  "version": "10.4.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {