hypercore 10.0.0-alpha.2 → 10.0.0-alpha.23

package/lib/block-encryption.js

@@ -0,0 +1,68 @@
+const sodium = require('sodium-universal')
+const c = require('compact-encoding')
+const b4a = require('b4a')
+
+const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
+
+module.exports = class BlockEncryption {
+  constructor (encryptionKey, hypercoreKey) {
+    const subKeys = b4a.alloc(2 * sodium.crypto_stream_KEYBYTES)
+
+    this.key = encryptionKey
+    this.blockKey = subKeys.subarray(0, sodium.crypto_stream_KEYBYTES)
+    this.blindingKey = subKeys.subarray(sodium.crypto_stream_KEYBYTES)
+    this.padding = 8
+
+    sodium.crypto_generichash(this.blockKey, encryptionKey, hypercoreKey)
+    sodium.crypto_generichash(this.blindingKey, this.blockKey)
+  }
+
+  encrypt (index, block, fork) {
+    const padding = block.subarray(0, this.padding)
+    block = block.subarray(this.padding)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: padding }, fork)
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+    // Zero out any previous padding.
+    nonce.fill(0, 8, 8 + padding.byteLength)
+
+    // Blind the fork ID, possibly risking reusing the nonce on a reorg of the
+    // Hypercore. This is fine as the blinding is best-effort and the latest
+    // fork ID shared on replication anyway.
+    sodium.crypto_stream_xor(
+      padding,
+      padding,
+      nonce,
+      this.blindingKey
+    )
+
+    nonce.set(padding, 8)
+
+    // The combination of a (blinded) fork ID and a block index is unique for a
+    // given Hypercore and is therefore a valid nonce for encrypting the block.
+    sodium.crypto_stream_xor(
+      block,
+      block,
+      nonce,
+      this.blockKey
+    )
+  }
+
+  decrypt (index, block) {
+    const padding = block.subarray(0, this.padding)
+    block = block.subarray(this.padding)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+    nonce.set(padding, 8)
+
+    // Decrypt the block using the blinded fork ID.
+    sodium.crypto_stream_xor(
+      block,
+      block,
+      nonce,
+      this.blockKey
+    )
+  }
+}

package/lib/block-store.js

@@ -1,3 +1,5 @@
+const b4a = require('b4a')
+
 module.exports = class BlockStore {
   constructor (storage, tree) {
     this.storage = storage
@@ -15,7 +17,7 @@ module.exports = class BlockStore {
 
   putBatch (i, batch, offset) {
     if (batch.length === 0) return Promise.resolve()
-    return this.put(i, batch.length === 1 ? batch[0] : Buffer.concat(batch), offset)
+    return this.put(i, batch.length === 1 ? batch[0] : b4a.concat(batch), offset)
   }
 
   clear () {

package/lib/core.js

@@ -1,4 +1,5 @@
 const hypercoreCrypto = require('hypercore-crypto')
+const b4a = require('b4a')
 const Oplog = require('./oplog')
 const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
@@ -90,7 +91,7 @@ module.exports = class Core {
       await oplog.flush(header)
     }
 
-    if (opts.keyPair && !header.signer.publicKey.equals(opts.keyPair.publicKey)) {
+    if (opts.keyPair && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
       throw new Error('Another hypercore is stored here')
     }
 
@@ -176,7 +177,7 @@ module.exports = class Core {
 
       for (const u of this.header.userData) {
         if (u.key !== key) continue
-        if (value && u.value.equals(value)) return
+        if (value && b4a.equals(u.value, value)) return
         empty = false
         break
       }
@@ -214,10 +215,12 @@ module.exports = class Core {
     }
   }
 
-  async append (values, sign = this.defaultSign) {
+  async append (values, sign = this.defaultSign, hooks = {}) {
     await this._mutex.lock()
 
     try {
+      if (hooks.preappend) await hooks.preappend(values)
+
       if (!values.length) return this.tree.length
 
       const batch = this.tree.batch()

package/lib/merkle-tree.js

@@ -1,9 +1,10 @@
 const flat = require('flat-tree')
 const crypto = require('hypercore-crypto')
-const uint64le = require('uint64le')
+const c = require('compact-encoding')
+const b4a = require('b4a')
 
-const BLANK_HASH = Buffer.alloc(32)
-const OLD_TREE = Buffer.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
+const BLANK_HASH = b4a.alloc(32)
+const OLD_TREE = b4a.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
 
 class NodeQueue {
   constructor (nodes, extra = null) {
@@ -214,7 +215,7 @@ class ReorgBatch extends MerkleTreeBatch {
     const nodes = []
     const root = verifyBlock(proof, this.tree.crypto, nodes)
 
-    if (root === null || !root.hash.equals(this.diff.hash)) return false
+    if (root === null || !b4a.equals(root.hash, this.diff.hash)) return false
 
     this.nodes.push(...nodes)
     return this._update(nodes)
@@ -232,7 +233,7 @@ class ReorgBatch extends MerkleTreeBatch {
       if (!left) break
 
       const existing = await this.tree.get(left.index, false)
-      if (!existing || !existing.hash.equals(left.hash)) {
+      if (!existing || !b4a.equals(existing.hash, left.hash)) {
         diff = left
       } else {
         diff = n.get(ite.sibling())
@@ -246,6 +247,8 @@ class ReorgBatch extends MerkleTreeBatch {
   }
 
   _updateDiffRoot (diff) {
+    if (this.want === null) return true
+
     const spans = flat.spans(diff.index)
     const start = spans[0] / 2
     const end = Math.min(this.treeLength, spans[1] / 2 + 1)
@@ -272,11 +275,15 @@ class ReorgBatch extends MerkleTreeBatch {
 }
 
 class ByteSeeker {
-  constructor (tree, bytes) {
+  constructor (tree, bytes, padding = 0) {
     this.tree = tree
     this.bytes = bytes
-    this.start = bytes >= tree.byteLength ? tree.length : 0
-    this.end = bytes < tree.byteLength ? tree.length : 0
+    this.padding = padding
+
+    const size = tree.byteLength - (tree.length * padding)
+
+    this.start = bytes >= size ? tree.length : 0
+    this.end = bytes < size ? tree.length : 0
   }
 
   nodes () {
@@ -287,12 +294,12 @@ class ByteSeeker {
     if (!bytes) return [0, 0]
 
     for (const node of this.tree.roots) { // all async ticks happen once we find the root so safe
-      if (bytes === node.size) {
-        return [flat.rightSpan(node.index) + 2, 0]
-      }
+      let size = node.size
+      if (this.padding > 0) size -= this.padding * flat.countLeaves(node.index)
 
-      if (bytes > node.size) {
-        bytes -= node.size
+      if (bytes === size) return [flat.rightSpan(node.index) + 2, 0]
+      if (bytes > size) {
+        bytes -= size
         continue
       }
 
@@ -301,9 +308,12 @@ class ByteSeeker {
       while ((ite.index & 1) !== 0) {
         const l = await this.tree.get(ite.leftChild(), false)
         if (l) {
-          if (l.size === bytes) return [ite.rightSpan() + 2, 0]
-          if (l.size > bytes) continue
-          bytes -= l.size
+          let size = l.size
+          if (this.padding > 0) size -= this.padding * ite.countLeaves()
+
+          if (size === bytes) return [ite.rightSpan() + 2, 0]
+          if (size > bytes) continue
+          bytes -= size
           ite.sibling()
         } else {
           ite.parent()
@@ -347,7 +357,7 @@ module.exports = class MerkleTree {
   }
 
   addNode (node) {
-    if (node.size === 0 && node.hash.equals(BLANK_HASH)) node = blankNode(node.index)
+    if (node.size === 0 && b4a.equals(node.hash, BLANK_HASH)) node = blankNode(node.index)
     this.unflushed.set(node.index, node)
   }
 
@@ -355,8 +365,8 @@ module.exports = class MerkleTree {
     return new MerkleTreeBatch(this)
   }
 
-  seek (bytes) {
-    return new ByteSeeker(this, bytes)
+  seek (bytes, padding) {
+    return new ByteSeeker(this, bytes, padding)
   }
 
   hash () {
@@ -371,6 +381,17 @@ module.exports = class MerkleTree {
     return this.signature !== null && this.crypto.verify(this.signable(), this.signature, key)
   }
 
+  getRoots (length) {
+    const indexes = flat.fullRoots(2 * length)
+    const roots = new Array(indexes.length)
+
+    for (let i = 0; i < indexes.length; i++) {
+      roots[i] = this.get(indexes[i], true)
+    }
+
+    return Promise.all(roots)
+  }
+
   get (index, error = true) {
     let node = this.unflushed.get(index)
 
@@ -433,17 +454,23 @@ module.exports = class MerkleTree {
     // TODO: write neighbors together etc etc
     // TODO: bench loading a full disk page and copy to that instead
     return new Promise((resolve, reject) => {
-      const slab = Buffer.allocUnsafe(40 * this.flushing.size)
+      const slab = b4a.allocUnsafe(40 * this.flushing.size)
 
       let error = null
       let missing = this.flushing.size + 1
       let offset = 0
 
       for (const node of this.flushing.values()) {
-        const b = slab.slice(offset, offset += 40)
-        uint64le.encode(node.size, b, 0)
-        node.hash.copy(b, 8)
-        this.storage.write(node.index * 40, b, done)
+        const state = {
+          start: 0,
+          end: 40,
+          buffer: slab.subarray(offset, offset += 40)
+        }
+
+        c.uint64.encode(state, node.size)
+        c.raw.encode(state, node.hash)
+
+        this.storage.write(node.index * 40, state.buffer, done)
       }
 
       done(null)
@@ -520,7 +547,7 @@ module.exports = class MerkleTree {
 
     for (const root of batch.roots) {
       const existing = await this.get(root.index, false)
-      if (existing && existing.hash.equals(root.hash)) continue
+      if (existing && b4a.equals(existing.hash, root.hash)) continue
       batch._updateDiffRoot(root)
       break
     }
@@ -548,7 +575,7 @@ module.exports = class MerkleTree {
 
     if (unverified) {
       const verified = await this.get(unverified.index)
-      if (!verified.hash.equals(unverified.hash)) {
+      if (!b4a.equals(verified.hash, unverified.hash)) {
         throw new Error('Invalid checksum at node ' + unverified.index)
       }
     }
@@ -693,7 +720,7 @@ module.exports = class MerkleTree {
     await new Promise((resolve, reject) => {
       storage.read(0, OLD_TREE.length, (err, buf) => {
         if (err) return resolve()
-        if (buf.equals(OLD_TREE)) return reject(new Error('Storage contains an incompatible merkle tree'))
+        if (b4a.equals(buf, OLD_TREE)) return reject(new Error('Storage contains an incompatible merkle tree'))
         resolve()
       })
     })
@@ -1038,10 +1065,10 @@ function getStoredNode (storage, index, error) {
         return
       }
 
-      const hash = data.slice(8)
-      const size = uint64le.decode(data, 0)
+      const hash = data.subarray(8)
+      const size = c.decode(c.uint64, data)
 
-      if (size === 0 && Buffer.compare(hash, BLANK_HASH) === 0) {
+      if (size === 0 && b4a.compare(hash, BLANK_HASH) === 0) {
         if (error) reject(new Error('Could not load node: ' + index))
         else resolve(null)
         return
@@ -1088,9 +1115,9 @@ function log2 (n) {
 }
 
 function signable (hash, length, fork) {
-  const buf = Buffer.alloc(48)
-  hash.copy(buf)
-  uint64le.encode(length, buf, 32)
-  uint64le.encode(fork, buf, 40)
-  return buf
+  const state = { start: 0, end: 48, buffer: b4a.alloc(48) }
+  c.raw.encode(state, hash)
+  c.uint64.encode(state, length)
+  c.uint64.encode(state, fork)
+  return state.buffer
 }

package/lib/oplog.js

@@ -1,4 +1,5 @@
 const cenc = require('compact-encoding')
+const b4a = require('b4a')
 const crc32 = require('crc32-universal')
 
 module.exports = class Oplog {
@@ -133,7 +134,7 @@ module.exports = class Oplog {
     return new Promise((resolve, reject) => {
       this.storage.open(err => {
         if (err && err.code !== 'ENOENT') return reject(err)
-        if (err) return resolve(Buffer.alloc(0))
+        if (err) return resolve(b4a.alloc(0))
         this.storage.stat((err, stat) => {
           if (err && err.code !== 'ENOENT') return reject(err)
           this.storage.read(0, stat.size, (err, buf) => {
@@ -151,7 +152,7 @@ module.exports = class Oplog {
     const bit = (this._headers[i] + 1) & 1
 
     this.headerEncoding.preencode(state, header)
-    state.buffer = Buffer.allocUnsafe(state.end)
+    state.buffer = b4a.allocUnsafe(state.end)
     this.headerEncoding.encode(state, header)
     this._addHeader(state, state.end - 8, bit, 0)
 
@@ -187,7 +188,7 @@ module.exports = class Oplog {
       this.entryEncoding.preencode(state, batch[i])
     }
 
-    state.buffer = Buffer.allocUnsafe(state.end)
+    state.buffer = b4a.allocUnsafe(state.end)
 
     for (let i = 0; i < batch.length; i++) {
       const start = state.start += 8 // space for header

package/lib/protocol.js

@@ -1,9 +1,20 @@
 const { uint, from: fromEncoding } = require('compact-encoding')
+const b4a = require('b4a')
 const safetyCatch = require('safety-catch')
 const codecs = require('codecs')
+const sodium = require('sodium-universal')
 
 const messages = require('./messages')
 
+const slab = b4a.alloc(96)
+const NS = slab.subarray(0, 32)
+const NS_HYPERCORE_INITIATOR = slab.subarray(32, 64)
+const NS_HYPERCORE_RESPONDER = slab.subarray(64, 96)
+
+sodium.crypto_generichash(NS, b4a.from('hypercore'))
+sodium.crypto_generichash(NS_HYPERCORE_INITIATOR, b4a.from([0]), NS)
+sodium.crypto_generichash(NS_HYPERCORE_RESPONDER, b4a.from([1]), NS)
+
 class Extension {
   constructor (protocol, type, name, handlers) {
     this.protocol = protocol
@@ -38,7 +49,7 @@ class Extension {
   _sendAlias (message, alias) {
     if (this.destroyed) return
 
-    if (this._remoteAliases) {
+    if (this.remoteSupports) {
       return this.protocol.send(this.type, this.encoding, alias, message)
     }
 
@@ -125,6 +136,7 @@ class Peer {
     this.resend = false
     this.state = state
     this.extensions = new Map()
+    this.destroyed = false
 
     this._destroyer = this._safeDestroy.bind(this)
   }
@@ -229,6 +241,7 @@ class Peer {
   }
 
   destroy (err) {
+    this.destroyed = true
     return this.protocol.unregisterPeer(this, err)
   }
 }
@@ -253,11 +266,17 @@ module.exports = class Protocol {
     this._localExtensions = 128
     this._remoteExtensions = []
     this._extensions = new Map()
+    this._keepAliveInterval = null
+    this._pendingCaps = []
 
     this._destroyer = this._safeDestroy.bind(this)
     this.noiseStream.on('data', this.onmessage.bind(this))
     this.noiseStream.on('end', this.noiseStream.end) // no half open
+    this.noiseStream.on('finish', () => {
+      this.setKeepAlive(false)
+    })
     this.noiseStream.on('close', () => {
+      this.setKeepAlive(false)
       // TODO: If the stream was destroyed with an error, we probably want to forward it here
       for (const peer of this._peers.values()) {
         peer.destroy(null)
@@ -267,6 +286,18 @@ module.exports = class Protocol {
     this._sendHandshake()
   }
 
+  setKeepAlive (enable) {
+    if (enable) {
+      if (this._keepAliveInterval) return
+      this._keepAliveInterval = setInterval(this.ping.bind(this), 5000)
+      if (this._keepAliveInterval.unref) this._keepAliveInterval.unref()
+    } else {
+      if (!this._keepAliveInterval) return
+      clearInterval(this._keepAliveInterval)
+      this._keepAliveInterval = null
+    }
+  }
+
   _sendHandshake () {
     const m = { protocolVersion: this.protocolVersion, userAgent: this.userAgent }
     const state = { start: 0, end: 0, buffer: null }
@@ -283,20 +314,20 @@ module.exports = class Protocol {
 
   registerPeer (key, discoveryKey, handlers = {}, state = null) {
     const peer = new Peer(this, this._localAliases++, key, discoveryKey, handlers, state)
-    this._peers.set(discoveryKey.toString('hex'), peer)
+    this._peers.set(b4a.toString(discoveryKey, 'hex'), peer)
     this._announceCore(peer.alias, key, discoveryKey)
     return peer
   }
 
   unregisterPeer (peer, err) {
-    this._peers.delete(peer.discoveryKey.toString('hex'))
+    this._peers.delete(b4a.toString(peer.discoveryKey, 'hex'))
 
     if (peer.remoteAlias > -1) {
       this._remoteAliases[peer.remoteAlias] = null
       peer.remoteAlias = -1
     }
 
-    peer.handlers.onunregister(this, err)
+    peer.handlers.onunregister(peer, err)
 
     if (err) this.noiseStream.destroy(err)
   }
@@ -329,6 +360,11 @@ module.exports = class Protocol {
 
     if (batch.length === 0) return
 
+    while (this._pendingCaps.length > 0) {
+      const [key, cap] = this._pendingCaps.pop()
+      hypercoreCapability(this.noiseStream.isInitiator, this.noiseStream.handshakeHash, key, cap)
+    }
+
     const state = { start: 0, end: 0, buffer: null }
     const lens = new Array(batch.length)
 
@@ -369,14 +405,24 @@ module.exports = class Protocol {
   }
 
   _announceCore (alias, key, discoveryKey) {
+    const cap = b4a.alloc(32)
+
+    if (!this.noiseStream.handshakeHash) {
+      this._pendingCaps.push([key, cap]) // encode it later...
+    } else {
+      hypercoreCapability(this.noiseStream.isInitiator, this.noiseStream.handshakeHash, key, cap)
+    }
+
     this.send(2, messages.core, -1, {
       alias: alias,
       discoveryKey: discoveryKey,
-      capability: Buffer.alloc(32) // TODO
+      capability: cap
     })
   }
 
   _decode (buffer) {
+    if (buffer.byteLength === 0) return
+
     const state = { start: 0, end: buffer.length, buffer }
 
     if (this._firstMessage === true) {
@@ -441,7 +487,7 @@ module.exports = class Protocol {
   }
 
   _oncore (m) {
-    const hex = m.discoveryKey.toString('hex')
+    const hex = b4a.toString(m.discoveryKey, 'hex')
     const peer = this._peers.get(hex)
 
     // allow one alloc
@@ -450,7 +496,11 @@ module.exports = class Protocol {
     if (m.alias === this._remoteAliases.length) this._remoteAliases.push(null)
 
     if (peer) {
-      // TODO: check cap
+      const expectedCap = hypercoreCapability(!this.noiseStream.isInitiator, this.noiseStream.handshakeHash, peer.key)
+      if (!b4a.equals(expectedCap, m.capability)) {
+        this.destroy(new Error('Remote sent an invalid capability'))
+        return
+      }
 
       if (m.alias >= this._remoteAliases.length) {
         this.destroy(new Error('Remote alias out of bounds'))
@@ -477,7 +527,7 @@ module.exports = class Protocol {
   }
 
   _onunknowncore (m) {
-    const peer = this._peers.get(m.discoveryKey.toString('hex'))
+    const peer = this._peers.get(b4a.toString(m.discoveryKey, 'hex'))
     if (!peer) return
 
     peer.resend = true
@@ -505,6 +555,11 @@ module.exports = class Protocol {
     return this.noiseStream.write(state.buffer)
   }
 
+  ping () {
+    const empty = this.noiseStream.alloc(0)
+    this.noiseStream.write(empty)
+  }
+
   destroy (err) {
     return this.noiseStream.destroy(err)
   }
@@ -520,3 +575,9 @@ function noop () {}
 function isPromise (p) {
   return !!p && typeof p.then === 'function'
 }
+
+function hypercoreCapability (initiator, handshakeHash, key, cap = b4a.alloc(32)) {
+  const ns = initiator ? NS_HYPERCORE_INITIATOR : NS_HYPERCORE_RESPONDER
+  sodium.crypto_generichash_batch(cap, [handshakeHash, key], ns)
+  return cap
+}