hypercore 10.0.0-alpha.8 → 10.1.0

package/lib/bitfield.js

@@ -1,6 +1,6 @@
-// TODO: needs massive improvements obvs
-
 const BigSparseArray = require('big-sparse-array')
+const b4a = require('b4a')
+const bits = require('bits-to-bytes')
 
 class FixedBitfield {
   constructor (index, bitfield) {
@@ -10,27 +10,26 @@ class FixedBitfield {
   }
 
   get (index) {
-    const j = index & 31
-    const i = (index - j) / 32
-
-    return i < this.bitfield.length && (this.bitfield[i] & (1 << j)) !== 0
+    return bits.get(this.bitfield, index)
   }
 
   set (index, val) {
-    const j = index & 31
-    const i = (index - j) / 32
-    const v = this.bitfield[i]
-
-    if (val === ((v & (1 << j)) !== 0)) return false
+    return bits.set(this.bitfield, index, val)
+  }
 
-    const u = val
-      ? v | (1 << j)
-      : v ^ (1 << j)
+  setRange (start, length, val) {
+    // Using fill instead of setRange is ~2 orders of magnitude faster, but does
+    // have the downside of not being able to tell if any bits actually changed.
+    bits.fill(this.bitfield, val, start, start + length)
+    return true
+  }
 
-    if (u === v) return false
+  firstSet (position) {
+    return bits.indexOf(this.bitfield, true, position)
+  }
 
-    this.bitfield[i] = u
-    return true
+  lastSet (position) {
+    return bits.lastIndexOf(this.bitfield, true, position)
   }
 }
 
@@ -40,9 +39,14 @@ module.exports = class Bitfield {
     this.pages = new BigSparseArray()
     this.unflushed = []
     this.storage = storage
-
-    const all = (buf && buf.byteLength >= 4)
-      ? new Uint32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4))
+    this.resumed = !!(buf && buf.byteLength >= 4)
+
+    const all = this.resumed
+      ? new Uint32Array(
+          buf.buffer,
+          buf.byteOffset,
+          Math.floor(buf.byteLength / 4)
+        )
       : new Uint32Array(1024)
 
     for (let i = 0; i < all.length; i += 1024) {
@@ -53,47 +57,106 @@ module.exports = class Bitfield {
   }
 
   get (index) {
-    const j = index & 32767
-    const i = (index - j) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i)
 
     return p ? p.get(j) : false
   }
 
   set (index, val) {
-    const j = index & 32767
-    const i = (index - j) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
 
     let p = this.pages.get(i)
 
-    if (!p) {
-      if (!val) return
+    if (!p && val) {
       p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
     }
 
-    if (!p.set(j, val) || p.dirty) return
-
-    p.dirty = true
-    this.unflushed.push(p)
+    if (p && p.set(j, val) && !p.dirty) {
+      p.dirty = true
+      this.unflushed.push(p)
+    }
   }
 
   setRange (start, length, val) {
-    for (let i = 0; i < length; i++) {
-      this.set(start + i, val)
+    let j = start & (this.pageSize - 1)
+    let i = (start - j) / this.pageSize
+
+    while (length > 0) {
+      let p = this.pages.get(i)
+
+      if (!p && val) {
+        p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
+      }
+
+      const end = Math.min(j + length, this.pageSize)
+      const range = end - j
+
+      if (p && p.setRange(j, range, val) && !p.dirty) {
+        p.dirty = true
+        this.unflushed.push(p)
+      }
+
+      j = 0
+      i++
+      length -= range
     }
   }
 
-  // Should prob be removed, when/if we re-add compression
-  page (i) {
-    const p = this.pages.get(i)
-    return p ? p.bitfield : new Uint32Array(1024)
+  firstSet (position) {
+    let j = position & (this.pageSize - 1)
+    let i = (position - j) / this.pageSize
+
+    while (i < this.pages.factor) {
+      const p = this.pages.get(i)
+
+      if (p) {
+        const index = p.firstSet(j)
+
+        if (index !== -1) {
+          return i * this.pageSize + index
+        }
+      }
+
+      j = 0
+      i++
+    }
+
+    return -1
+  }
+
+  lastSet (position) {
+    let j = position & (this.pageSize - 1)
+    let i = (position - j) / this.pageSize
+
+    while (i >= 0) {
+      const p = this.pages.get(i)
+
+      if (p) {
+        const index = p.lastSet(j)
+
+        if (index !== -1) {
+          return i * this.pageSize + index
+        }
+      }
+
+      j = this.pageSize - 1
+      i--
+    }
+
+    return -1
   }
 
   clear () {
     return new Promise((resolve, reject) => {
-      this.storage.del(0, Infinity, (err) => {
-        if (err) reject(err)
-        else resolve()
+      this.storage.truncate(0, (err) => {
+        if (err) return reject(err)
+        this.pages = new BigSparseArray()
+        this.unflushed = []
+        resolve()
       })
     })
   }
@@ -116,9 +179,14 @@ module.exports = class Bitfield {
       let error = null
 
       for (const page of this.unflushed) {
-        const b = Buffer.from(page.bitfield.buffer, page.bitfield.byteOffset, page.bitfield.byteLength)
+        const buf = b4a.from(
+          page.bitfield.buffer,
+          page.bitfield.byteOffset,
+          page.bitfield.byteLength
+        )
+
         page.dirty = false
-        this.storage.write(page.index * 4096, b, done)
+        this.storage.write(page.index * 4096, buf, done)
       }
 
       function done (err) {
@@ -147,7 +215,7 @@ module.exports = class Bitfield {
 }
 
 function ensureSize (uint32, size) {
-  if (uint32.length === size) return uint32
+  if (uint32.byteLength === size) return uint32
   const a = new Uint32Array(1024)
   a.set(uint32, 0)
   return a

package/lib/block-encryption.js

@@ -1,11 +1,12 @@
 const sodium = require('sodium-universal')
 const c = require('compact-encoding')
+const b4a = require('b4a')
 
-const nonce = Buffer.alloc(sodium.crypto_stream_NONCEBYTES)
+const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
 
 module.exports = class BlockEncryption {
   constructor (encryptionKey, hypercoreKey) {
-    const subKeys = Buffer.alloc(2 * sodium.crypto_stream_KEYBYTES)
+    const subKeys = b4a.alloc(2 * sodium.crypto_stream_KEYBYTES)
 
     this.key = encryptionKey
     this.blockKey = subKeys.subarray(0, sodium.crypto_stream_KEYBYTES)

package/lib/block-store.js

@@ -1,3 +1,5 @@
+const b4a = require('b4a')
+
 module.exports = class BlockStore {
   constructor (storage, tree) {
     this.storage = storage
@@ -15,15 +17,18 @@ module.exports = class BlockStore {
 
   putBatch (i, batch, offset) {
     if (batch.length === 0) return Promise.resolve()
-    return this.put(i, batch.length === 1 ? batch[0] : Buffer.concat(batch), offset)
+    return this.put(i, batch.length === 1 ? batch[0] : b4a.concat(batch), offset)
   }
 
-  clear () {
+  clear (offset = 0, length = -1) {
     return new Promise((resolve, reject) => {
-      this.storage.del(0, Infinity, (err) => {
+      if (length === -1) this.storage.truncate(offset, done)
+      else this.storage.del(offset, length, done)
+
+      function done (err) {
         if (err) reject(err)
         else resolve()
-      })
+      }
     })
   }
 
@@ -49,7 +54,7 @@ module.exports = class BlockStore {
     return new Promise((resolve, reject) => {
       this.storage.write(offset, data, (err) => {
         if (err) reject(err)
-        else resolve()
+        else resolve(offset + data.byteLength)
       })
     })
   }

package/lib/caps.js

@@ -0,0 +1,32 @@
+const crypto = require('hypercore-crypto')
+const sodium = require('sodium-universal')
+const b4a = require('b4a')
+const c = require('compact-encoding')
+
+// TODO: rename this to "crypto" and move everything hashing related etc in here
+// Also lets move the tree stuff from hypercore-crypto here
+
+const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER] = crypto.namespace('hypercore', 3)
+
+exports.replicate = function (isInitiator, key, handshakeHash) {
+  const out = b4a.allocUnsafe(32)
+  sodium.crypto_generichash_batch(out, [isInitiator ? REPLICATE_INITIATOR : REPLICATE_RESPONDER, key], handshakeHash)
+  return out
+}
+
+exports.treeSignable = function (hash, length, fork) {
+  const state = { start: 0, end: 80, buffer: b4a.allocUnsafe(80) }
+  c.raw.encode(state, TREE)
+  c.raw.encode(state, hash)
+  c.uint64.encode(state, length)
+  c.uint64.encode(state, fork)
+  return state.buffer
+}
+
+exports.treeSignableLegacy = function (hash, length, fork) {
+  const state = { start: 0, end: 48, buffer: b4a.allocUnsafe(48) }
+  c.raw.encode(state, hash)
+  c.uint64.encode(state, length)
+  c.uint64.encode(state, fork)
+  return state.buffer
+}