hypercore 10.0.0-alpha.4 → 10.0.0-alpha.40

package/lib/bitfield.js

@@ -1,6 +1,7 @@
 // TODO: needs massive improvements obvs
 
 const BigSparseArray = require('big-sparse-array')
+const b4a = require('b4a')
 
 class FixedBitfield {
   constructor (index, bitfield) {
@@ -40,8 +41,9 @@ module.exports = class Bitfield {
     this.pages = new BigSparseArray()
     this.unflushed = []
     this.storage = storage
+    this.resumed = !!(buf && buf.byteLength >= 4)
 
-    const all = (buf && buf.byteLength >= 4)
+    const all = this.resumed
       ? new Uint32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4))
       : new Uint32Array(1024)
 
@@ -92,8 +94,10 @@ module.exports = class Bitfield {
   clear () {
     return new Promise((resolve, reject) => {
       this.storage.del(0, Infinity, (err) => {
-        if (err) reject(err)
-        else resolve()
+        if (err) return reject(err)
+        this.pages = new BigSparseArray()
+        this.unflushed = []
+        resolve()
       })
     })
   }
@@ -116,9 +120,9 @@ module.exports = class Bitfield {
       let error = null
 
       for (const page of this.unflushed) {
-        const b = Buffer.from(page.bitfield.buffer, page.bitfield.byteOffset, page.bitfield.byteLength)
+        const buf = b4a.from(page.bitfield.buffer, page.bitfield.byteOffset, page.bitfield.byteLength)
         page.dirty = false
-        this.storage.write(page.index * 4096, b, done)
+        this.storage.write(page.index * 4096, buf, done)
       }
 
       function done (err) {

package/lib/block-encryption.js

@@ -0,0 +1,68 @@
+const sodium = require('sodium-universal')
+const c = require('compact-encoding')
+const b4a = require('b4a')
+
+const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
+
+module.exports = class BlockEncryption {
+  constructor (encryptionKey, hypercoreKey) {
+    const subKeys = b4a.alloc(2 * sodium.crypto_stream_KEYBYTES)
+
+    this.key = encryptionKey
+    this.blockKey = subKeys.subarray(0, sodium.crypto_stream_KEYBYTES)
+    this.blindingKey = subKeys.subarray(sodium.crypto_stream_KEYBYTES)
+    this.padding = 8
+
+    sodium.crypto_generichash(this.blockKey, encryptionKey, hypercoreKey)
+    sodium.crypto_generichash(this.blindingKey, this.blockKey)
+  }
+
+  encrypt (index, block, fork) {
+    const padding = block.subarray(0, this.padding)
+    block = block.subarray(this.padding)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: padding }, fork)
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+    // Zero out any previous padding.
+    nonce.fill(0, 8, 8 + padding.byteLength)
+
+    // Blind the fork ID, possibly risking reusing the nonce on a reorg of the
+    // Hypercore. This is fine as the blinding is best-effort and the latest
+    // fork ID shared on replication anyway.
+    sodium.crypto_stream_xor(
+      padding,
+      padding,
+      nonce,
+      this.blindingKey
+    )
+
+    nonce.set(padding, 8)
+
+    // The combination of a (blinded) fork ID and a block index is unique for a
+    // given Hypercore and is therefore a valid nonce for encrypting the block.
+    sodium.crypto_stream_xor(
+      block,
+      block,
+      nonce,
+      this.blockKey
+    )
+  }
+
+  decrypt (index, block) {
+    const padding = block.subarray(0, this.padding)
+    block = block.subarray(this.padding)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+    nonce.set(padding, 8)
+
+    // Decrypt the block using the blinded fork ID.
+    sodium.crypto_stream_xor(
+      block,
+      block,
+      nonce,
+      this.blockKey
+    )
+  }
+}

package/lib/block-store.js

@@ -1,3 +1,5 @@
+const b4a = require('b4a')
+
 module.exports = class BlockStore {
   constructor (storage, tree) {
     this.storage = storage
@@ -15,7 +17,7 @@ module.exports = class BlockStore {
 
   putBatch (i, batch, offset) {
     if (batch.length === 0) return Promise.resolve()
-    return this.put(i, batch.length === 1 ? batch[0] : Buffer.concat(batch), offset)
+    return this.put(i, batch.length === 1 ? batch[0] : b4a.concat(batch), offset)
   }
 
   clear () {

package/lib/caps.js

@@ -0,0 +1,34 @@
+const crypto = require('hypercore-crypto')
+const sodium = require('sodium-universal')
+const b4a = require('b4a')
+const c = require('compact-encoding')
+
+// TODO: rename this to "crypto" and move everything hashing related etc in here
+// Also lets move the tree stuff from hypercore-crypto here, and loose the types
+// from the hashes there - they are not needed since we lock the indexes in the tree
+// hash and just makes alignment etc harder in other languages
+
+const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER] = crypto.namespace('hypercore', 3)
+
+exports.replicate = function (isInitiator, key, handshakeHash) {
+  const out = b4a.allocUnsafe(32)
+  sodium.crypto_generichash_batch(out, [isInitiator ? REPLICATE_INITIATOR : REPLICATE_RESPONDER, key], handshakeHash)
+  return out
+}
+
+exports.treeSignable = function (hash, length, fork) {
+  const state = { start: 0, end: 80, buffer: b4a.allocUnsafe(80) }
+  c.raw.encode(state, TREE)
+  c.raw.encode(state, hash)
+  c.uint64.encode(state, length)
+  c.uint64.encode(state, fork)
+  return state.buffer
+}
+
+exports.treeSignableLegacy = function (hash, length, fork) {
+  const state = { start: 0, end: 48, buffer: b4a.allocUnsafe(48) }
+  c.raw.encode(state, hash)
+  c.uint64.encode(state, length)
+  c.uint64.encode(state, fork)
+  return state.buffer
+}

package/lib/core.js

@@ -1,13 +1,15 @@
 const hypercoreCrypto = require('hypercore-crypto')
+const b4a = require('b4a')
 const Oplog = require('./oplog')
 const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
-const { oplogHeader, oplogEntry } = require('./messages')
+const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = require('./errors')
+const m = require('./messages')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, sign, onupdate) {
+  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate) {
     this.onupdate = onupdate
     this.header = header
     this.crypto = crypto
@@ -15,7 +17,7 @@ module.exports = class Core {
     this.tree = tree
     this.blocks = blocks
     this.bitfield = bitfield
-    this.defaultSign = sign
+    this.defaultAuth = auth
     this.truncating = 0
 
     this._maxOplogSize = 65536
@@ -23,6 +25,9 @@ module.exports = class Core {
     this._verifies = null
     this._verifiesFlushed = null
     this._mutex = new Mutex()
+    this._legacy = legacy
+
+    this._updateContiguousLength(header.contiguousLength)
   }
 
   static async open (storage, opts = {}) {
@@ -49,27 +54,46 @@ module.exports = class Core {
     }
   }
 
-  // TODO: we should prob have a general "auth" abstraction instead somewhere?
-  static createSigner (crypto, { publicKey, secretKey }) {
-    if (!crypto.validateKeyPair({ publicKey, secretKey })) throw new Error('Invalid key pair')
-    return signable => crypto.sign(signable, secretKey)
+  static createAuth (crypto, { publicKey, secretKey }, opts = {}) {
+    if (secretKey && !crypto.validateKeyPair({ publicKey, secretKey })) {
+      throw BAD_ARGUMENT('Invalid key pair')
+    }
+
+    const sign = opts.sign
+      ? opts.sign
+      : secretKey
+        ? (signable) => crypto.sign(signable, secretKey)
+        : undefined
+
+    return {
+      sign,
+      verify (signable, signature) {
+        return crypto.verify(signable, signature, publicKey)
+      }
+    }
   }
 
   static async resume (oplogFile, treeFile, bitfieldFile, dataFile, opts) {
-    const overwrite = opts.overwrite === true
+    let overwrite = opts.overwrite === true
+
+    const force = opts.force === true
     const createIfMissing = opts.createIfMissing !== false
     const crypto = opts.crypto || hypercoreCrypto
 
     const oplog = new Oplog(oplogFile, {
-      headerEncoding: oplogHeader,
-      entryEncoding: oplogEntry
+      headerEncoding: m.oplog.header,
+      entryEncoding: m.oplog.entry
     })
 
     let { header, entries } = await oplog.open()
 
-    if (!header || overwrite === true) {
+    if (force && opts.keyPair && header && header.signer && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
+      overwrite = true
+    }
+
+    if (!header || overwrite) {
       if (!createIfMissing) {
-        throw new Error('No hypercore is stored here')
+        throw STORAGE_EMPTY('No Hypercore is stored here')
       }
 
       header = {
@@ -84,14 +108,15 @@ module.exports = class Core {
         signer: opts.keyPair || crypto.keyPair(),
         hints: {
           reorgs: []
-        }
+        },
+        contiguousLength: 0
       }
 
       await oplog.flush(header)
     }
 
-    if (opts.keyPair && !header.signer.publicKey.equals(opts.keyPair.publicKey)) {
-      throw new Error('Another hypercore is stored here')
+    if (opts.keyPair && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
+      throw STORAGE_CONFLICT('Another Hypercore is stored here')
     }
 
     const tree = await MerkleTree.open(treeFile, { crypto, ...header.tree })
@@ -102,9 +127,13 @@ module.exports = class Core {
       await tree.clear()
       await blocks.clear()
       await bitfield.clear()
+      entries = []
+    } else if (bitfield.resumed && header.tree.length === 0) {
+      // If this was an old bitfield, reset it since it loads based on disk size atm (TODO: change that)
+      await bitfield.clear()
     }
 
-    const sign = opts.sign || (header.signer.secretKey ? this.createSigner(crypto, header.signer) : null)
+    const auth = opts.auth || this.createAuth(crypto, header.signer)
 
     for (const e of entries) {
       if (e.userData) {
@@ -118,7 +147,7 @@ module.exports = class Core {
       }
 
       if (e.bitfield) {
-        bitfield.setRange(e.bitfield.start, e.bitfield.length)
+        bitfield.setRange(e.bitfield.start, e.bitfield.length, !e.bitfield.drop)
       }
 
       if (e.treeUpgrade) {
@@ -135,7 +164,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, sign, opts.onupdate || noop)
+    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop)
   }
 
   _shouldFlush () {
@@ -166,6 +195,16 @@ module.exports = class Core {
     await this.blocks.put(index, value, byteOffset)
   }
 
+  _updateContiguousLength (index, length = 0) {
+    if (index === this.header.contiguousLength) {
+      let i = this.header.contiguousLength + length
+
+      while (this.bitfield.get(i)) i++
+
+      this.header.contiguousLength = i
+    }
+  }
+
   async userData (key, value) {
     // TODO: each oplog append can set user data, so we should have a way
     // to just hitch a ride on one of the other ongoing appends?
@@ -176,7 +215,7 @@ module.exports = class Core {
 
       for (const u of this.header.userData) {
         if (u.key !== key) continue
-        if (value && u.value.equals(value)) return
+        if (value && b4a.equals(u.value, value)) return
         empty = false
         break
       }
@@ -200,13 +239,13 @@ module.exports = class Core {
     }
   }
 
-  async truncate (length, fork, sign = this.defaultSign) {
+  async truncate (length, fork, auth = this.defaultAuth) {
     this.truncating++
     await this._mutex.lock()
 
     try {
       const batch = await this.tree.truncate(length, fork)
-      batch.signature = await sign(batch.signable())
+      batch.signature = await auth.sign(batch.signable())
       await this._truncate(batch, null)
     } finally {
       this.truncating--
@@ -214,17 +253,19 @@ module.exports = class Core {
     }
   }
 
-  async append (values, sign = this.defaultSign) {
+  async append (values, auth = this.defaultAuth, hooks = {}) {
     await this._mutex.lock()
 
     try {
+      if (hooks.preappend) await hooks.preappend(values)
+
       if (!values.length) return this.tree.length
 
       const batch = this.tree.batch()
       for (const val of values) batch.append(val)
 
       const hash = batch.hash()
-      batch.signature = await sign(batch.signable(hash))
+      batch.signature = await auth.sign(this._legacy ? batch.signableLegacy(hash) : batch.signable(hash))
 
       const entry = {
         userData: null,
@@ -243,6 +284,7 @@ module.exports = class Core {
       this.bitfield.setRange(batch.ancestors, batch.length - batch.ancestors, true)
       batch.commit()
 
+      this.header.contiguousLength = batch.length
       this.header.tree.length = batch.length
       this.header.tree.rootHash = hash
       this.header.tree.signature = batch.signature
@@ -256,11 +298,16 @@ module.exports = class Core {
     }
   }
 
+  _signed (batch, hash, auth = this.defaultAuth) {
+    const signable = this._legacy ? batch.signableLegacy(hash) : batch.signable(hash)
+    return auth.verify(signable, batch.signature)
+  }
+
   async _verifyExclusive ({ batch, bitfield, value, from }) {
     // TODO: move this to tree.js
     const hash = batch.hash()
-    if (!batch.signature || !this.crypto.verify(batch.signable(hash), batch.signature, this.header.signer.publicKey)) {
-      throw new Error('Remote signature does not match')
+    if (!batch.signature || !this._signed(batch, hash)) {
+      throw INVALID_SIGNATURE('Proof contains an invalid signature')
     }
 
     await this._mutex.lock()
@@ -279,7 +326,11 @@ module.exports = class Core {
 
       await this.oplog.append([entry], false)
 
-      if (bitfield) this.bitfield.set(bitfield.start, true)
+      if (bitfield) {
+        this.bitfield.set(bitfield.start, true)
+        this._updateContiguousLength(bitfield.start, bitfield.length)
+      }
+
       batch.commit()
 
       this.header.tree.fork = batch.fork
@@ -333,8 +384,13 @@ module.exports = class Core {
           continue
         }
 
-        if (bitfield) this.bitfield.set(bitfield.start, true)
+        if (bitfield) {
+          this.bitfield.set(bitfield.start, true)
+          this._updateContiguousLength(bitfield.start, bitfield.length)
+        }
+
         batch.commit()
+
         this.onupdate(0, bitfield, value, from)
       }
 
@@ -415,6 +471,7 @@ module.exports = class Core {
 
     const appended = batch.length > batch.ancestors
 
+    this.header.contiguousLength = Math.min(batch.ancestors, this.header.contiguousLength)
     this.header.tree.fork = batch.fork
     this.header.tree.length = batch.length
     this.header.tree.rootHash = batch.hash()

package/lib/errors.js

@@ -0,0 +1,42 @@
+module.exports = class HypercoreError extends Error {
+  constructor (msg, code) {
+    super(msg)
+    this.code = code
+  }
+
+  static BAD_ARGUMENT (msg) {
+    return new HypercoreError(msg, 'BAD_ARGUMENT')
+  }
+
+  static STORAGE_EMPTY (msg) {
+    return new HypercoreError(msg, 'STORAGE_EMPTY')
+  }
+
+  static STORAGE_CONFLICT (msg) {
+    return new HypercoreError(msg, 'STORAGE_CONFLICT')
+  }
+
+  static INVALID_SIGNATURE (msg) {
+    return new HypercoreError(msg, 'INVALID_SIGNATURE')
+  }
+
+  static INVALID_CAPABILITY (msg) {
+    return new HypercoreError(msg, 'INVALID_CAPABILITY')
+  }
+
+  static SNAPSHOT_NOT_AVAILABLE (msg = 'Snapshot is not available') {
+    return new HypercoreError(msg, 'SNAPSHOT_NOT_AVAILABLE')
+  }
+
+  static REQUEST_CANCELLED (msg = 'Request was cancelled') {
+    return new HypercoreError(msg, 'REQUEST_CANCELLED')
+  }
+
+  static SESSION_NOT_WRITABLE (msg = 'Session is not writable') {
+    return new HypercoreError(msg, 'SESSION_NOT_WRITABLE')
+  }
+
+  static SESSION_CLOSED (msg = 'Session is closed') {
+    return new HypercoreError(msg, 'SESSION_CLOSED')
+  }
+}