hypercore 10.0.0-alpha.7 → 10.0.0

package/lib/bitfield.js

@@ -1,6 +1,6 @@
-// TODO: needs massive improvements obvs
-
 const BigSparseArray = require('big-sparse-array')
+const b4a = require('b4a')
+const bits = require('bits-to-bytes')
 
 class FixedBitfield {
   constructor (index, bitfield) {
@@ -10,27 +10,26 @@ class FixedBitfield {
   }
 
   get (index) {
-    const j = index & 31
-    const i = (index - j) / 32
-
-    return i < this.bitfield.length && (this.bitfield[i] & (1 << j)) !== 0
+    return bits.get(this.bitfield, index)
   }
 
   set (index, val) {
-    const j = index & 31
-    const i = (index - j) / 32
-    const v = this.bitfield[i]
-
-    if (val === ((v & (1 << j)) !== 0)) return false
+    return bits.set(this.bitfield, index, val)
+  }
 
-    const u = val
-      ? v | (1 << j)
-      : v ^ (1 << j)
+  setRange (start, length, val) {
+    // Using fill instead of setRange is ~2 orders of magnitude faster, but does
+    // have the downside of not being able to tell if any bits actually changed.
+    bits.fill(this.bitfield, val, start, start + length)
+    return true
+  }
 
-    if (u === v) return false
+  firstSet (position) {
+    return bits.indexOf(this.bitfield, true, position)
+  }
 
-    this.bitfield[i] = u
-    return true
+  lastSet (position) {
+    return bits.lastIndexOf(this.bitfield, true, position)
   }
 }
 
@@ -40,9 +39,14 @@ module.exports = class Bitfield {
     this.pages = new BigSparseArray()
     this.unflushed = []
     this.storage = storage
-
-    const all = (buf && buf.byteLength >= 4)
-      ? new Uint32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4))
+    this.resumed = !!(buf && buf.byteLength >= 4)
+
+    const all = this.resumed
+      ? new Uint32Array(
+          buf.buffer,
+          buf.byteOffset,
+          Math.floor(buf.byteLength / 4)
+        )
       : new Uint32Array(1024)
 
     for (let i = 0; i < all.length; i += 1024) {
@@ -53,47 +57,106 @@ module.exports = class Bitfield {
   }
 
   get (index) {
-    const j = index & 32767
-    const i = (index - j) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i)
 
     return p ? p.get(j) : false
   }
 
   set (index, val) {
-    const j = index & 32767
-    const i = (index - j) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
 
     let p = this.pages.get(i)
 
-    if (!p) {
-      if (!val) return
+    if (!p && val) {
       p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
     }
 
-    if (!p.set(j, val) || p.dirty) return
-
-    p.dirty = true
-    this.unflushed.push(p)
+    if (p && p.set(j, val) && !p.dirty) {
+      p.dirty = true
+      this.unflushed.push(p)
+    }
   }
 
   setRange (start, length, val) {
-    for (let i = 0; i < length; i++) {
-      this.set(start + i, val)
+    let j = start & (this.pageSize - 1)
+    let i = (start - j) / this.pageSize
+
+    while (length > 0) {
+      let p = this.pages.get(i)
+
+      if (!p && val) {
+        p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
+      }
+
+      const end = Math.min(j + length, this.pageSize)
+      const range = end - j
+
+      if (p && p.setRange(j, range, val) && !p.dirty) {
+        p.dirty = true
+        this.unflushed.push(p)
+      }
+
+      j = 0
+      i++
+      length -= range
     }
   }
 
-  // Should prob be removed, when/if we re-add compression
-  page (i) {
-    const p = this.pages.get(i)
-    return p ? p.bitfield : new Uint32Array(1024)
+  firstSet (position) {
+    let j = position & (this.pageSize - 1)
+    let i = (position - j) / this.pageSize
+
+    while (i < this.pages.factor) {
+      const p = this.pages.get(i)
+
+      if (p) {
+        const index = p.firstSet(j)
+
+        if (index !== -1) {
+          return i * this.pageSize + index
+        }
+      }
+
+      j = 0
+      i++
+    }
+
+    return -1
+  }
+
+  lastSet (position) {
+    let j = position & (this.pageSize - 1)
+    let i = (position - j) / this.pageSize
+
+    while (i >= 0) {
+      const p = this.pages.get(i)
+
+      if (p) {
+        const index = p.lastSet(j)
+
+        if (index !== -1) {
+          return i * this.pageSize + index
+        }
+      }
+
+      j = this.pageSize - 1
+      i--
+    }
+
+    return -1
   }
 
   clear () {
     return new Promise((resolve, reject) => {
       this.storage.del(0, Infinity, (err) => {
-        if (err) reject(err)
-        else resolve()
+        if (err) return reject(err)
+        this.pages = new BigSparseArray()
+        this.unflushed = []
+        resolve()
       })
     })
   }
@@ -116,9 +179,14 @@ module.exports = class Bitfield {
       let error = null
 
       for (const page of this.unflushed) {
-        const b = Buffer.from(page.bitfield.buffer, page.bitfield.byteOffset, page.bitfield.byteLength)
+        const buf = b4a.from(
+          page.bitfield.buffer,
+          page.bitfield.byteOffset,
+          page.bitfield.byteLength
+        )
+
         page.dirty = false
-        this.storage.write(page.index * 4096, b, done)
+        this.storage.write(page.index * 4096, buf, done)
       }
 
       function done (err) {
@@ -147,7 +215,7 @@ module.exports = class Bitfield {
 }
 
 function ensureSize (uint32, size) {
-  if (uint32.length === size) return uint32
+  if (uint32.byteLength === size) return uint32
   const a = new Uint32Array(1024)
   a.set(uint32, 0)
   return a

package/lib/block-encryption.js

@@ -1,11 +1,12 @@
 const sodium = require('sodium-universal')
 const c = require('compact-encoding')
+const b4a = require('b4a')
 
-const nonce = Buffer.alloc(sodium.crypto_stream_NONCEBYTES)
+const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
 
 module.exports = class BlockEncryption {
   constructor (encryptionKey, hypercoreKey) {
-    const subKeys = Buffer.alloc(2 * sodium.crypto_stream_KEYBYTES)
+    const subKeys = b4a.alloc(2 * sodium.crypto_stream_KEYBYTES)
 
     this.key = encryptionKey
     this.blockKey = subKeys.subarray(0, sodium.crypto_stream_KEYBYTES)

package/lib/block-store.js

@@ -1,3 +1,5 @@
+const b4a = require('b4a')
+
 module.exports = class BlockStore {
   constructor (storage, tree) {
     this.storage = storage
@@ -15,12 +17,12 @@ module.exports = class BlockStore {
 
   putBatch (i, batch, offset) {
     if (batch.length === 0) return Promise.resolve()
-    return this.put(i, batch.length === 1 ? batch[0] : Buffer.concat(batch), offset)
+    return this.put(i, batch.length === 1 ? batch[0] : b4a.concat(batch), offset)
   }
 
-  clear () {
+  clear (offset = 0, length = Infinity) {
     return new Promise((resolve, reject) => {
-      this.storage.del(0, Infinity, (err) => {
+      this.storage.del(offset, length, (err) => {
         if (err) reject(err)
         else resolve()
       })
@@ -49,7 +51,7 @@ module.exports = class BlockStore {
     return new Promise((resolve, reject) => {
       this.storage.write(offset, data, (err) => {
         if (err) reject(err)
-        else resolve()
+        else resolve(offset + data.byteLength)
       })
     })
   }

package/lib/caps.js

@@ -0,0 +1,32 @@
+const crypto = require('hypercore-crypto')
+const sodium = require('sodium-universal')
+const b4a = require('b4a')
+const c = require('compact-encoding')
+
+// TODO: rename this to "crypto" and move everything hashing related etc in here
+// Also lets move the tree stuff from hypercore-crypto here
+
+const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER] = crypto.namespace('hypercore', 3)
+
+exports.replicate = function (isInitiator, key, handshakeHash) {
+  const out = b4a.allocUnsafe(32)
+  sodium.crypto_generichash_batch(out, [isInitiator ? REPLICATE_INITIATOR : REPLICATE_RESPONDER, key], handshakeHash)
+  return out
+}
+
+exports.treeSignable = function (hash, length, fork) {
+  const state = { start: 0, end: 80, buffer: b4a.allocUnsafe(80) }
+  c.raw.encode(state, TREE)
+  c.raw.encode(state, hash)
+  c.uint64.encode(state, length)
+  c.uint64.encode(state, fork)
+  return state.buffer
+}
+
+exports.treeSignableLegacy = function (hash, length, fork) {
+  const state = { start: 0, end: 48, buffer: b4a.allocUnsafe(48) }
+  c.raw.encode(state, hash)
+  c.uint64.encode(state, length)
+  c.uint64.encode(state, fork)
+  return state.buffer
+}

package/lib/core.js

@@ -1,13 +1,15 @@
 const hypercoreCrypto = require('hypercore-crypto')
+const b4a = require('b4a')
 const Oplog = require('./oplog')
 const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
-const { oplogHeader, oplogEntry } = require('./messages')
+const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = require('./errors')
+const m = require('./messages')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, sign, onupdate) {
+  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate) {
     this.onupdate = onupdate
     this.header = header
     this.crypto = crypto
@@ -15,7 +17,7 @@ module.exports = class Core {
     this.tree = tree
     this.blocks = blocks
     this.bitfield = bitfield
-    this.defaultSign = sign
+    this.defaultAuth = auth
     this.truncating = 0
 
     this._maxOplogSize = 65536
@@ -23,6 +25,7 @@ module.exports = class Core {
     this._verifies = null
     this._verifiesFlushed = null
     this._mutex = new Mutex()
+    this._legacy = legacy
   }
 
   static async open (storage, opts = {}) {
@@ -49,27 +52,46 @@ module.exports = class Core {
     }
   }
 
-  // TODO: we should prob have a general "auth" abstraction instead somewhere?
-  static createSigner (crypto, { publicKey, secretKey }) {
-    if (!crypto.validateKeyPair({ publicKey, secretKey })) throw new Error('Invalid key pair')
-    return signable => crypto.sign(signable, secretKey)
+  static createAuth (crypto, { publicKey, secretKey }, opts = {}) {
+    if (secretKey && !crypto.validateKeyPair({ publicKey, secretKey })) {
+      throw BAD_ARGUMENT('Invalid key pair')
+    }
+
+    const sign = opts.sign
+      ? opts.sign
+      : secretKey
+        ? (signable) => crypto.sign(signable, secretKey)
+        : undefined
+
+    return {
+      sign,
+      verify (signable, signature) {
+        return crypto.verify(signable, signature, publicKey)
+      }
+    }
   }
 
   static async resume (oplogFile, treeFile, bitfieldFile, dataFile, opts) {
-    const overwrite = opts.overwrite === true
+    let overwrite = opts.overwrite === true
+
+    const force = opts.force === true
     const createIfMissing = opts.createIfMissing !== false
     const crypto = opts.crypto || hypercoreCrypto
 
     const oplog = new Oplog(oplogFile, {
-      headerEncoding: oplogHeader,
-      entryEncoding: oplogEntry
+      headerEncoding: m.oplog.header,
+      entryEncoding: m.oplog.entry
     })
 
     let { header, entries } = await oplog.open()
 
-    if (!header || overwrite === true) {
+    if (force && opts.keyPair && header && header.signer && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
+      overwrite = true
+    }
+
+    if (!header || overwrite) {
       if (!createIfMissing) {
-        throw new Error('No hypercore is stored here')
+        throw STORAGE_EMPTY('No Hypercore is stored here')
       }
 
       header = {
@@ -84,14 +106,15 @@ module.exports = class Core {
         signer: opts.keyPair || crypto.keyPair(),
         hints: {
           reorgs: []
-        }
+        },
+        contiguousLength: 0
       }
 
       await oplog.flush(header)
     }
 
-    if (opts.keyPair && !header.signer.publicKey.equals(opts.keyPair.publicKey)) {
-      throw new Error('Another hypercore is stored here')
+    if (opts.keyPair && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
+      throw STORAGE_CONFLICT('Another Hypercore is stored here')
     }
 
     const tree = await MerkleTree.open(treeFile, { crypto, ...header.tree })
@@ -102,9 +125,18 @@ module.exports = class Core {
       await tree.clear()
       await blocks.clear()
       await bitfield.clear()
+      entries = []
+    } else if (bitfield.resumed && header.tree.length === 0) {
+      // If this was an old bitfield, reset it since it loads based on disk size atm (TODO: change that)
+      await bitfield.clear()
+    }
+
+    // compat from earlier version that do not store contig length
+    if (header.contiguousLength === 0) {
+      while (bitfield.get(header.contiguousLength)) header.contiguousLength++
     }
 
-    const sign = opts.sign || (header.signer.secretKey ? this.createSigner(crypto, header.signer) : null)
+    const auth = opts.auth || this.createAuth(crypto, header.signer)
 
     for (const e of entries) {
       if (e.userData) {
@@ -118,7 +150,8 @@ module.exports = class Core {
       }
 
       if (e.bitfield) {
-        bitfield.setRange(e.bitfield.start, e.bitfield.length)
+        bitfield.setRange(e.bitfield.start, e.bitfield.length, !e.bitfield.drop)
+        updateContig(header, e.bitfield, bitfield)
       }
 
       if (e.treeUpgrade) {
@@ -135,7 +168,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, sign, opts.onupdate || noop)
+    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop)
   }
 
   _shouldFlush () {
@@ -176,7 +209,7 @@ module.exports = class Core {
 
       for (const u of this.header.userData) {
         if (u.key !== key) continue
-        if (value && u.value.equals(value)) return
+        if (value && b4a.equals(u.value, value)) return
         empty = false
         break
       }
@@ -200,13 +233,13 @@ module.exports = class Core {
     }
   }
 
-  async truncate (length, fork, sign = this.defaultSign) {
+  async truncate (length, fork, auth = this.defaultAuth) {
     this.truncating++
     await this._mutex.lock()
 
     try {
       const batch = await this.tree.truncate(length, fork)
-      batch.signature = await sign(batch.signable())
+      batch.signature = await auth.sign(batch.signable())
       await this._truncate(batch, null)
     } finally {
       this.truncating--
@@ -214,19 +247,63 @@ module.exports = class Core {
     }
   }
 
-  async append (values, sign = this.defaultSign, hooks = {}) {
+  async clear (start, end) {
+    await this._mutex.lock()
+
+    const entry = {
+      userData: null,
+      treeNodes: null,
+      treeUpgrade: null,
+      bitfield: {
+        start,
+        length: end - start,
+        drop: true
+      }
+    }
+
+    await this.oplog.append([entry], false)
+
+    this.bitfield.setRange(start, end - start, false)
+
+    if (start < this.header.contiguousLength) {
+      this.header.contiguousLength = start
+    }
+
+    start = this.bitfield.lastSet(start) + 1
+    end = this.bitfield.firstSet(end)
+
+    if (end === -1) end = this.tree.length
+
+    try {
+      const offset = await this.tree.byteOffset(start * 2)
+      const [byteEnd, byteEndLength] = await this.tree.byteRange((end - 1) * 2)
+      const length = (byteEnd + byteEndLength) - offset
+
+      await this.blocks.clear(offset, length)
+
+      this.onupdate(0, entry.bitfield, null, null)
+
+      if (this._shouldFlush()) await this._flushOplog()
+    } finally {
+      this._mutex.unlock()
+    }
+  }
+
+  async append (values, auth = this.defaultAuth, hooks = {}) {
     await this._mutex.lock()
 
     try {
       if (hooks.preappend) await hooks.preappend(values)
 
-      if (!values.length) return this.tree.length
+      if (!values.length) {
+        return { length: this.tree.length, byteLength: this.tree.byteLength }
+      }
 
       const batch = this.tree.batch()
       for (const val of values) batch.append(val)
 
       const hash = batch.hash()
-      batch.signature = await sign(batch.signable(hash))
+      batch.signature = await auth.sign(this._legacy ? batch.signableLegacy(hash) : batch.signable(hash))
 
       const entry = {
         userData: null,
@@ -239,7 +316,8 @@ module.exports = class Core {
         }
       }
 
-      await this._appendBlocks(values)
+      const byteLength = await this._appendBlocks(values)
+
       await this.oplog.append([entry], false)
 
       this.bitfield.setRange(batch.ancestors, batch.length - batch.ancestors, true)
@@ -248,21 +326,28 @@ module.exports = class Core {
       this.header.tree.length = batch.length
       this.header.tree.rootHash = hash
       this.header.tree.signature = batch.signature
-      this.onupdate(0b01, entry.bitfield, null, null)
+
+      const status = 0b0001 | updateContig(this.header, entry.bitfield, this.bitfield)
+      this.onupdate(status, entry.bitfield, null, null)
 
       if (this._shouldFlush()) await this._flushOplog()
 
-      return batch.ancestors
+      return { length: batch.length, byteLength }
     } finally {
       this._mutex.unlock()
     }
   }
 
+  _signed (batch, hash, auth = this.defaultAuth) {
+    const signable = this._legacy ? batch.signableLegacy(hash) : batch.signable(hash)
+    return auth.verify(signable, batch.signature)
+  }
+
   async _verifyExclusive ({ batch, bitfield, value, from }) {
     // TODO: move this to tree.js
     const hash = batch.hash()
-    if (!batch.signature || !this.crypto.verify(batch.signable(hash), batch.signature, this.header.signer.publicKey)) {
-      throw new Error('Remote signature does not match')
+    if (!batch.signature || !this._signed(batch, hash)) {
+      throw INVALID_SIGNATURE('Proof contains an invalid signature')
     }
 
     await this._mutex.lock()
@@ -281,14 +366,21 @@ module.exports = class Core {
 
       await this.oplog.append([entry], false)
 
-      if (bitfield) this.bitfield.set(bitfield.start, true)
+      let status = 0b0001
+
+      if (bitfield) {
+        this.bitfield.set(bitfield.start, true)
+        status |= updateContig(this.header, bitfield, this.bitfield)
+      }
+
       batch.commit()
 
       this.header.tree.fork = batch.fork
       this.header.tree.length = batch.length
       this.header.tree.rootHash = batch.rootHash
       this.header.tree.signature = batch.signature
-      this.onupdate(0b01, bitfield, value, from)
+
+      this.onupdate(status, bitfield, value, from)
 
       if (this._shouldFlush()) await this._flushOplog()
     } finally {
@@ -335,9 +427,16 @@ module.exports = class Core {
           continue
         }
 
-        if (bitfield) this.bitfield.set(bitfield.start, true)
+        let status = 0
+
+        if (bitfield) {
+          this.bitfield.set(bitfield.start, true)
+          status = updateContig(this.header, bitfield, this.bitfield)
+        }
+
         batch.commit()
-        this.onupdate(0, bitfield, value, from)
+
+        this.onupdate(status, bitfield, value, from)
       }
 
       if (this._shouldFlush()) await this._flushOplog()
@@ -415,13 +514,15 @@ module.exports = class Core {
     addReorgHint(this.header.hints.reorgs, this.tree, batch)
     batch.commit()
 
-    const appended = batch.length > batch.ancestors
+    const contigStatus = updateContig(this.header, entry.bitfield, this.bitfield)
+    const status = ((batch.length > batch.ancestors) ? 0b0011 : 0b0010) | contigStatus
 
     this.header.tree.fork = batch.fork
     this.header.tree.length = batch.length
     this.header.tree.rootHash = batch.hash()
     this.header.tree.signature = batch.signature
-    this.onupdate(appended ? 0b11 : 0b10, entry.bitfield, null, from)
+
+    this.onupdate(status, entry.bitfield, null, from)
 
     // TODO: there is a bug in the merkle tree atm where it cannot handle unflushed
     // truncates if we append or download anything after the truncation point later on
@@ -442,6 +543,36 @@ module.exports = class Core {
   }
 }
 
+function updateContig (header, upd, bitfield) {
+  const end = upd.start + upd.length
+
+  let c = header.contiguousLength
+
+  if (upd.drop) {
+    // If we dropped a block in the current contig range, "downgrade" it
+    if (c <= end && c > upd.start) {
+      c = upd.start
+    }
+  } else {
+    if (c <= end && c >= upd.start) {
+      c = end
+      while (bitfield.get(c)) c++
+    }
+  }
+
+  if (c === header.contiguousLength) {
+    return 0b0000
+  }
+
+  if (c > header.contiguousLength) {
+    header.contiguousLength = c
+    return 0b0100
+  }
+
+  header.contiguousLength = c
+  return 0b1000
+}
+
 function addReorgHint (list, tree, batch) {
   if (tree.length === 0 || tree.fork === batch.fork) return