hypercore 10.38.2 → 11.0.1

package/lib/audit.js

@@ -1,14 +1,13 @@
 const hypercoreCrypto = require('hypercore-crypto')
 const flat = require('flat-tree')
-const c = require('compact-encoding')
 const b4a = require('b4a')
 
-const empty = b4a.alloc(32)
+const BitInterlude = require('./bit-interlude')
 
 // this is optimised for speed over mem atm
 // can be tweaked in the future
 
-module.exports = async function auditCore (core) {
+module.exports = async function auditCore (core, storage) {
   const corrections = {
     tree: 0,
     blocks: 0
@@ -16,8 +15,10 @@ module.exports = async function auditCore (core) {
 
   const length = core.header.tree.length
 
-  const data = await readFullStorage(core.blocks.storage)
-  const tree = await readFullStorage(core.tree.storage)
+  const bitfield = new BitInterlude()
+
+  const data = await readAllBlocks(core.storage)
+  const tree = await readAllTreeNodes(core.tree.storage)
 
   const valid = new Uint8Array(Math.ceil(tree.byteLength / 40))
   const stack = []
@@ -32,8 +33,8 @@ module.exports = async function auditCore (core) {
     if ((node.index & 1) === 0) continue
 
     const [left, right] = flat.children(node.index)
-    const leftNode = getNode(left)
-    const rightNode = getNode(right)
+    const leftNode = tree.get(left)
+    const rightNode = tree.get(right)
 
     if (!rightNode && !leftNode) continue
 
@@ -48,12 +49,8 @@ module.exports = async function auditCore (core) {
       }
     }
 
-    if (leftNode.size) clearNode(leftNode)
-    if (rightNode.size) clearNode(rightNode)
-  }
-
-  if (corrections.tree) {
-    core.tree.cache.clear()
+    if (leftNode.size) clearNode(left)
+    if (rightNode.size) clearNode(right)
   }
 
   let i = 0
@@ -73,57 +70,52 @@ module.exports = async function auditCore (core) {
       try {
         nextOffset = await core.tree.byteOffset(i * 2)
       } catch {
-        core._setBitfield(i, false)
+        storage.deleteBlock(i)
+        bitfield.set(i, false)
         corrections.blocks++
         i++
         continue
       }
     }
 
-    const node = getNode(i * 2)
-    const blk = data.subarray(nextOffset, nextOffset + node.size)
+    const node = tree.get(i * 2)
+    const blk = data.get(i)
     const hash = hypercoreCrypto.data(blk)
 
     nextOffset += blk.byteLength
 
     if (!b4a.equals(hash, node.hash)) {
-      core._setBitfield(i, false)
+      storage.deleteBlock(i)
+      bitfield.set(i, false)
       corrections.blocks++
     }
 
     i++
   }
 
-  return corrections
+  bitfield.flush(storage, core.bitfield)
 
-  function getNode (index) {
-    if (index * 40 + 40 > tree.byteLength) return null
-    const state = { start: index * 40, end: index * 40 + 40, buffer: tree }
-    const size = c.uint64.decode(state)
-    const hash = c.fixed32.decode(state)
-    if (size === 0 && hash.equals(empty)) return null
-    return { index, size, hash }
-  }
+  return corrections
 
   function clearNode (node) {
     valid[node.index] = 0
+    storage.deleteTreeNode(node.index)
+    corrections.tree++
+  }
+}
 
-    if (node.size) {
-      b4a.fill(tree, 0, node.index * 40, node.index * 40 + 40)
-      core.tree.unflushed.set(node.index, core.tree.blankNode(node.index))
-      corrections.tree++
-    }
+async function readAllBlocks (storage) {
+  const data = new Map()
+  for await (const block of storage.createBlockStream()) {
+    data.set(block.index, block.value)
   }
+  return data
 }
 
-function readFullStorage (storage) {
-  return new Promise((resolve, reject) => {
-    storage.stat((_, st) => {
-      if (!st) return resolve(b4a.alloc(0))
-      storage.read(0, st.size, (err, data) => {
-        if (err) reject(err)
-        else resolve(data)
-      })
-    })
-  })
+async function readAllTreeNodes (storage) {
+  const nodes = new Map()
+  for await (const node of storage.createTreeNodeStream()) {
+    nodes.set(node.index, node)
+  }
+  return nodes
 }

package/lib/bit-interlude.js

@@ -0,0 +1,174 @@
+const b4a = require('b4a')
+const quickbit = require('./compat').quickbit
+
+module.exports = class BitInterlude {
+  constructor () {
+    this.ranges = []
+  }
+
+  contiguousLength (from) {
+    for (const r of this.ranges) {
+      if (r.start > from) break
+      if (!r.value && r.start <= from) return r.start
+    }
+
+    // TODO: be smarter
+    while (this.get(from) === true) from++
+    return from
+  }
+
+  get (index) {
+    let start = 0
+    let end = this.ranges.length
+
+    while (start < end) {
+      const mid = (start + end) >> 1
+      const r = this.ranges[mid]
+
+      if (index < r.start) {
+        end = mid
+        continue
+      }
+
+      if (index >= r.end) {
+        if (mid === start) break
+        start = mid
+        continue
+      }
+
+      return r.value
+    }
+
+    return false
+  }
+
+  setRange (start, end, value) {
+    if (start === end) return
+
+    let r = null
+
+    for (let i = 0; i < this.ranges.length; i++) {
+      r = this.ranges[i]
+
+      // if already inside, stop
+      if (r.start <= start && end <= r.end) {
+        if (value === r.value) return
+
+        const ranges = mergeRanges(r, { start, end, value })
+        this.ranges.splice(i, 1, ...ranges)
+
+        return
+      }
+
+      // we wanna overun the interval
+      if (start > r.end) {
+        continue
+      }
+
+      // we overran but this interval is ending after us, move it back
+      if (end >= r.start && end <= r.end) {
+        r.start = r.value === value ? start : end
+        if (r.value !== value) this.ranges.splice(i, 0, { start, end, value })
+        return
+      }
+
+      // we overran but our start is contained in this interval, move start back
+      if (start >= r.start && start <= r.end) {
+        if (r.value !== value) {
+          this.ranges.splice(++i, 0, { start, end, value })
+          r.end = start
+          return
+        }
+
+        start = r.start
+      }
+
+      let remove = 0
+
+      for (let j = i; j < this.ranges.length; j++) {
+        const n = this.ranges[j]
+        if (n.start > end || n.value !== value) break
+        if (n.start <= end && n.end > end) end = n.end
+        remove++
+      }
+
+      this.ranges.splice(i, remove, { start, end, value })
+      return
+    }
+
+    if (r !== null) {
+      if (start <= r.end && end > r.end) {
+        r.end = end
+        return
+      }
+
+      // we never
+      if (r.end > start) return
+    }
+
+    this.ranges.push({ start, end, value })
+  }
+
+  flush (tx, bitfield) {
+    if (!this.ranges.length) return []
+
+    let index = this.ranges[0].start
+    const final = this.ranges[this.ranges.length - 1].end
+
+    let i = 0
+
+    while (index < final) {
+      const page = bitfield.getBitfield(index) // read only
+      const pageIndex = page ? page.index : bitfield.getPageIndex(index)
+
+      const buf = b4a.allocUnsafe(bitfield.getPageByteLength())
+
+      const view = new DataView(
+        buf.buffer,
+        buf.byteOffset,
+        buf.byteLength
+      )
+
+      if (page) {
+        for (let i = 0; i < page.bitfield.length; i++) {
+          view.setUint32(i * 4, page.bitfield[i], true)
+        }
+      }
+
+      const last = (pageIndex + 1) * (buf.byteLength << 3)
+      const offset = pageIndex * (buf.byteLength << 3)
+
+      let hasValue = false
+
+      while (i < this.ranges.length) {
+        const { start, end, value } = this.ranges[i]
+
+        if (!hasValue && value) hasValue = true
+
+        const from = start < index ? index : start
+        const to = end < last ? end : last
+
+        quickbit.fill(buf, value, from - offset, to - offset)
+
+        index = to
+
+        if (to === last) break
+
+        i++
+      }
+
+      if (page || hasValue) tx.putBitfieldPage(pageIndex, buf)
+    }
+
+    return this.ranges
+  }
+}
+
+function mergeRanges (a, b) {
+  const ranges = []
+  if (a.start < b.start) ranges.push({ start: a.start, end: b.start, value: a.value })
+  ranges.push({ start: b.start, end: b.end, value: b.value })
+  if (b.end < a.end) ranges.push({ start: b.end, end: a.end, value: a.value })
+
+  return ranges
+}

package/lib/bitfield.js

@@ -14,7 +14,6 @@ const SEGMENT_GROWTH_FACTOR = 4
 
 class BitfieldPage {
   constructor (index, segment) {
-    this.dirty = false
     this.index = index
     this.offset = index * BYTES_PER_PAGE - segment.offset
     this.bitfield = null
@@ -27,7 +26,7 @@ class BitfieldPage {
     return this.segment.tree
   }
 
-  get (index) {
+  get (index, dirty) {
     return quickbit.get(this.bitfield, index)
   }
 
@@ -37,11 +36,11 @@ class BitfieldPage {
     }
   }
 
-  setRange (start, length, val) {
-    quickbit.fill(this.bitfield, val, start, start + length)
+  setRange (start, end, val) {
+    quickbit.fill(this.bitfield, val, start, end)
 
     let i = Math.floor(start / 128)
-    const n = i + Math.ceil(length / 128)
+    const n = i + Math.ceil((end - start) / 128)
 
     while (i <= n) this.tree.update(this.offset * 8 + i++ * 128)
   }
@@ -172,10 +171,11 @@ class BitfieldSegment {
 }
 
 module.exports = class Bitfield {
-  constructor (storage, buffer) {
-    this.unflushed = []
-    this.storage = storage
-    this.resumed = !!(buffer && buffer.byteLength >= 4)
+  static BITS_PER_PAGE = BITS_PER_PAGE
+  static BYTES_PER_PAGE = BYTES_PER_PAGE
+
+  constructor (buffer) {
+    this.resumed = !!(buffer && buffer.byteLength >= 0)
 
     this._pages = new BigSparseArray()
     this._segments = new BigSparseArray()
@@ -213,6 +213,10 @@ module.exports = class Bitfield {
     }
   }
 
+  static from (bitfield) {
+    return new Bitfield(bitfield.toBuffer(bitfield._pages.maxLength * BITS_PER_PAGE))
+  }
+
   toBuffer (length) {
     const pages = Math.ceil(length / BITS_PER_PAGE)
     const buffer = b4a.allocUnsafe(pages * BYTES_PER_PAGE)
@@ -238,13 +242,29 @@ module.exports = class Bitfield {
   }
 
   getBitfield (index) {
-    const j = index & (BITS_PER_PAGE - 1)
-    const i = (index - j) / BITS_PER_PAGE
+    const i = this.getPageIndex(index)
 
     const p = this._pages.get(i)
     return p || null
   }
 
+  merge (bitfield, length) {
+    let i = 0
+
+    while (i < length) {
+      const start = bitfield.firstSet(i)
+      if (start === -1) break
+
+      i = bitfield.firstUnset(start)
+
+      if (i === -1 || i > length) i = length
+
+      this.setRange(start, i, true)
+
+      if (i >= length) break
+    }
+  }
+
   get (index) {
     const j = index & (BITS_PER_PAGE - 1)
     const i = (index - j) / BITS_PER_PAGE
@@ -254,6 +274,32 @@ module.exports = class Bitfield {
     return p ? p.get(j) : false
   }
 
+  getPageByteLength () {
+    return BYTES_PER_PAGE
+  }
+
+  getPageIndex (index) {
+    const j = index & (BITS_PER_PAGE - 1)
+    return (index - j) / BITS_PER_PAGE
+  }
+
+  getPage (index, create) {
+    const i = this.getPageIndex(index)
+
+    let p = this._pages.get(i)
+
+    if (p) return p
+
+    if (!create) return null
+
+    const k = Math.floor(i / PAGES_PER_SEGMENT)
+    const s = this._segments.get(k) || this._segments.set(k, new BitfieldSegment(k, new Uint32Array(k === 0 ? INITIAL_WORDS_PER_SEGMENT : WORDS_PER_SEGMENT)))
+
+    p = this._pages.set(i, new BitfieldPage(i, s))
+
+    return p
+  }
+
   set (index, val) {
     const j = index & (BITS_PER_PAGE - 1)
     const i = (index - j) / BITS_PER_PAGE
@@ -267,21 +313,14 @@ module.exports = class Bitfield {
       p = this._pages.set(i, new BitfieldPage(i, s))
     }
 
-    if (p) {
-      p.set(j, val)
-
-      if (!p.dirty) {
-        p.dirty = true
-        this.unflushed.push(p)
-      }
-    }
+    if (p) p.set(j, val)
   }
 
-  setRange (start, length, val) {
+  setRange (start, end, val) {
     let j = start & (BITS_PER_PAGE - 1)
     let i = (start - j) / BITS_PER_PAGE
 
-    while (length > 0) {
+    while (start < end) {
       let p = this._pages.get(i)
 
       if (!p && val) {
@@ -291,21 +330,14 @@ module.exports = class Bitfield {
         p = this._pages.set(i, new BitfieldPage(i, s))
       }
 
-      const end = Math.min(j + length, BITS_PER_PAGE)
-      const range = end - j
-
-      if (p) {
-        p.setRange(j, range, val)
+      const last = Math.min(end, BITS_PER_PAGE)
+      const range = last - j
 
-        if (!p.dirty) {
-          p.dirty = true
-          this.unflushed.push(p)
-        }
-      }
+      if (p) p.setRange(j, last, val)
 
       j = 0
       i++
-      length -= range
+      end -= range
     }
   }
 
@@ -420,68 +452,28 @@ module.exports = class Bitfield {
     }
   }
 
-  clear () {
-    return new Promise((resolve, reject) => {
-      this.storage.truncate(0, (err) => {
-        if (err) return reject(err)
-        this._pages = new BigSparseArray()
-        this.unflushed = []
-        resolve()
-      })
-    })
+  clear (tx) {
+    return tx.deleteBitfieldPageRange(0, -1)
   }
 
-  close () {
-    return new Promise((resolve, reject) => {
-      this.storage.close((err) => {
-        if (err) reject(err)
-        else resolve()
-      })
-    })
+  onupdate (ranges) {
+    for (const { start, end, value } of ranges) {
+      this.setRange(start, end, value)
+    }
   }
 
-  flush () {
-    return new Promise((resolve, reject) => {
-      if (!this.unflushed.length) return resolve()
-
-      const self = this
-      let missing = this.unflushed.length
-      let error = null
+  static async open (storage, length) {
+    if (length === 0) return new Bitfield(storage, null)
 
-      for (const page of this.unflushed) {
-        const buf = b4a.from(
-          page.bitfield.buffer,
-          page.bitfield.byteOffset,
-          page.bitfield.byteLength
-        )
+    const pages = Math.ceil(length / BITS_PER_PAGE)
+    const buffer = b4a.alloc(pages * BYTES_PER_PAGE)
+    const stream = storage.createBitfieldStream()
 
-        page.dirty = false
-        this.storage.write(page.index * BYTES_PER_PAGE, buf, done)
-      }
+    for await (const { index, page } of stream) {
+      buffer.set(page, index * BYTES_PER_PAGE)
+    }
 
-      function done (err) {
-        if (err) error = err
-        if (--missing) return
-        if (error) return reject(error)
-        self.unflushed = []
-        resolve()
-      }
-    })
-  }
-
-  static open (storage, tree = null) {
-    return new Promise((resolve, reject) => {
-      storage.stat((err, st) => {
-        if (err) return resolve(new Bitfield(storage, null))
-        let size = st.size - (st.size & 3)
-        if (!size) return resolve(new Bitfield(storage, null))
-        if (tree) size = Math.min(size, ceilTo(tree.length / 8, 4096))
-        storage.read(0, size, (err, data) => {
-          if (err) return reject(err)
-          resolve(new Bitfield(storage, data))
-        })
-      })
-    })
+    return new Bitfield(buffer)
   }
 }
 

package/lib/block-store.js

@@ -1,63 +1,25 @@
-const b4a = require('b4a')
-const { WRITE_FAILED } = require('hypercore-errors')
-
 module.exports = class BlockStore {
-  constructor (storage, tree) {
+  constructor (storage) {
     this.storage = storage
-    this.tree = tree
-  }
-
-  async get (i, tree) {
-    if (!tree) tree = this.tree
-    const [offset, size] = await tree.byteRange(2 * i)
-    return this._read(offset, size)
   }
 
-  async put (i, data, offset) {
-    return this._write(offset, data)
+  async get (rx, i) {
+    return rx.getBlock(i)
   }
 
-  putBatch (i, batch, offset) {
-    if (batch.length === 0) return Promise.resolve()
-    return this.put(i, batch.length === 1 ? batch[0] : b4a.concat(batch), offset)
+  put (tx, i, data) {
+    tx.putBlock(i, data)
   }
 
-  clear (offset = 0, length = -1) {
-    return new Promise((resolve, reject) => {
-      if (length === -1) this.storage.truncate(offset, done)
-      else this.storage.del(offset, length, done)
-
-      function done (err) {
-        if (err) reject(err)
-        else resolve()
-      }
-    })
-  }
-
-  close () {
-    return new Promise((resolve, reject) => {
-      this.storage.close((err) => {
-        if (err) reject(err)
-        else resolve()
-      })
-    })
-  }
+  putBatch (tx, i, blocks) {
+    if (blocks.length === 0) return Promise.resolve()
 
-  _read (offset, size) {
-    return new Promise((resolve, reject) => {
-      this.storage.read(offset, size, (err, data) => {
-        if (err) reject(err)
-        else resolve(data)
-      })
-    })
+    for (let j = 0; j < blocks.length; j++) {
+      tx.putBlock(i + j, blocks[j])
+    }
   }
 
-  _write (offset, data) {
-    return new Promise((resolve, reject) => {
-      this.storage.write(offset, data, (err) => {
-        if (err) reject(WRITE_FAILED(err.message))
-        else resolve(offset + data.byteLength)
-      })
-    })
+  clear (tx, start = 0, end = -1) {
+    tx.deleteBlockRange(start, end)
   }
 }