hypercore 10.0.0-alpha.5 → 10.0.0-alpha.9

package/README.md

@@ -63,7 +63,8 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
   createIfMissing: true, // create a new Hypercore key pair if none was present in storage
   overwrite: false, // overwrite any old Hypercore that might already exist
   valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
-  keyPair: kp // optionally pass the public key and secret key as a key pair
+  keyPair: kp, // optionally pass the public key and secret key as a key pair
+  encryptionKey: k // optionally pass an encryption key to enable block encryption
 }
 ```
 
@@ -113,6 +114,7 @@ A range can have the following properties:
 {
   start: startIndex,
   end: nonInclusiveEndIndex,
+  blocks: [index1, index2, ...],
   linear: false // download range linearly and not randomly
 }
 ```
@@ -125,6 +127,12 @@ To download the full core continously (often referred to as non sparse mode) do
 core.download({ start: 0, end: -1 })
 ```
 
+To downloaded a discrete range of blocks pass a list of indices.
+
+```js
+core.download({ blocks: [4, 9, 7] });
+```
+
 To cancel downloading a range simply destroy the range instance.
 
 ``` js
@@ -195,6 +203,10 @@ In contrast to `core.key` this key does not allow you to verify the data but can
 
 Populated after `ready` has been emitted. Will be `null` before the event.
 
+#### `core.encryptionKey`
+
+Buffer containing the optional block encryption key of this core. Will be `null` unless block encryption is enabled.
+
 #### `core.length`
 
 How many blocks of data are available on this core?
@@ -213,6 +225,10 @@ What is the current fork id of this core?
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
+#### `core.padding`
+
+How much padding is applied to each block of this core? Will be `0` unless block encryption is enabled.
+
 #### `const stream = core.replicate(isInitiatorOrReplicationStream)`
 
 Create a replication stream. You should pipe this to another Hypercore instance.

package/index.js

@@ -12,6 +12,7 @@ const fsctl = requireMaybe('fsctl') || { lock: noop, sparse: noop }
 const Replicator = require('./lib/replicator')
 const Extensions = require('./lib/extensions')
 const Core = require('./lib/core')
+const BlockEncryption = require('./lib/block-encryption')
 
 const promises = Symbol.for('hypercore.promises')
 const inspect = Symbol.for('nodejs.util.inspect.custom')
@@ -28,14 +29,17 @@ module.exports = class Hypercore extends EventEmitter {
       opts = key
       key = null
     }
+
     if (key && typeof key === 'string') {
       key = Buffer.from(key, 'hex')
     }
-    if (key && key.byteLength !== 32) {
+
+    if (!opts) opts = {}
+
+    if (!opts.crypto && key && key.byteLength !== 32) {
       throw new Error('Hypercore key should be 32 bytes')
     }
 
-    if (!opts) opts = {}
     if (!storage) storage = opts.storage
 
     this[promises] = true
@@ -44,6 +48,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.crypto = opts.crypto || hypercoreCrypto
     this.core = null
     this.replicator = null
+    this.encryption = null
     this.extensions = opts.extensions || new Extensions()
     this.cache = opts.cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (opts.cache || null)
 
@@ -61,6 +66,8 @@ module.exports = class Hypercore extends EventEmitter {
     this.closing = null
     this.opening = opts._opening || this._open(key, storage, opts)
     this.opening.catch(noop)
+
+    this._preappend = preappend.bind(this)
   }
 
   [inspect] (depth, opts) {
@@ -135,6 +142,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.discoveryKey = o.discoveryKey
     this.core = o.core
     this.replicator = o.replicator
+    this.encryption = o.encryption
     this.writable = !!this.sign
     this.autoClose = o.autoClose
   }
@@ -204,7 +212,7 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   get byteLength () {
-    return this.core === null ? 0 : this.core.tree.byteLength
+    return this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding)
   }
 
   get fork () {
@@ -215,6 +223,14 @@ module.exports = class Hypercore extends EventEmitter {
     return this.replicator === null ? [] : this.replicator.peers
   }
 
+  get encryptionKey () {
+    return this.encryption && this.encryption.key
+  }
+
+  get padding () {
+    return this.encryption === null ? 0 : this.encryption.padding
+  }
+
   ready () {
     return this.opening
   }
@@ -268,6 +284,10 @@ module.exports = class Hypercore extends EventEmitter {
     this.key = this.core.header.signer.publicKey
     this.writable = !!this.sign
 
+    if (!this.encryption && opts.encryptionKey) {
+      this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
+    }
+
     this.extensions.attach(this.replicator)
     this.opened = true
 
@@ -302,6 +322,12 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     if (value) {
+      if (this.encryption) {
+        this.encryption.decrypt(bitfield.start, value)
+      }
+
+      value = value.subarray(this.padding)
+
       for (let i = 0; i < this.sessions.length; i++) {
         this.sessions[i].emit('download', bitfield.start, value, from)
       }
@@ -340,7 +366,7 @@ module.exports = class Hypercore extends EventEmitter {
   async seek (bytes) {
     if (this.opened === false) await this.opening
 
-    const s = this.core.tree.seek(bytes)
+    const s = this.core.tree.seek(bytes, this.padding)
 
     return (await s.update()) || this.replicator.requestSeek(s)
   }
@@ -364,20 +390,42 @@ module.exports = class Hypercore extends EventEmitter {
   async _get (index, opts) {
     const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
 
-    if (this.core.bitfield.get(index)) return decode(encoding, await this.core.blocks.get(index))
-    if (opts && opts.onwait) opts.onwait(index)
+    let block
+
+    if (this.core.bitfield.get(index)) {
+      block = await this.core.blocks.get(index)
+      if (this.encryption) this.encryption.decrypt(index, block)
+    } else {
+      if (opts && opts.onwait) opts.onwait(index)
+      // Note that the _oncoreupdate handler decrypts inplace so we should not decrypt here
+      block = await this.replicator.requestBlock(index)
+    }
 
-    return decode(encoding, await this.replicator.requestBlock(index))
+    return this._decode(encoding, block)
   }
 
   download (range) {
-    const start = (range && range.start) || 0
-    const end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
     const linear = !!(range && range.linear)
 
-    // TODO: support range.blocks
+    let start
+    let end
+    let filter
+
+    if (range && range.blocks) {
+      const blocks = range.blocks instanceof Set
+        ? range.blocks
+        : new Set(range.blocks)
+
+      start = range.start || (blocks.size ? min(range.blocks) : 0)
+      end = range.end || (blocks.size ? max(range.blocks) + 1 : 0)
+
+      filter = (i) => blocks.has(i)
+    } else {
+      start = (range && range.start) || 0
+      end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
+    }
 
-    const r = Replicator.createRange(start, end, linear)
+    const r = Replicator.createRange(start, end, filter, linear)
 
     if (this.opened) this.replicator.addRange(r)
     else this.opening.then(() => this.replicator.addRange(r), noop)
@@ -410,22 +458,16 @@ module.exports = class Hypercore extends EventEmitter {
     if (this.opened === false) await this.opening
     if (this.writable === false) throw new Error('Core is not writable')
 
-    const blks = Array.isArray(blocks) ? blocks : [blocks]
-    const buffers = new Array(blks.length)
+    blocks = Array.isArray(blocks) ? blocks : [blocks]
 
-    for (let i = 0; i < blks.length; i++) {
-      const blk = blks[i]
+    const preappend = this.encryption && this._preappend
+    const buffers = new Array(blocks.length)
 
-      const buf = Buffer.isBuffer(blk)
-        ? blk
-        : this.valueEncoding
-          ? c.encode(this.valueEncoding, blk)
-          : Buffer.from(blk)
-
-      buffers[i] = buf
+    for (let i = 0; i < blocks.length; i++) {
+      buffers[i] = this._encode(this.valueEncoding, blocks[i])
     }
 
-    return await this.core.append(buffers, this.sign)
+    return await this.core.append(buffers, this.sign, { preappend })
   }
 
   registerExtension (name, handlers) {
@@ -436,14 +478,38 @@ module.exports = class Hypercore extends EventEmitter {
   onextensionupdate () {
     if (this.replicator !== null) this.replicator.broadcastOptions()
   }
-}
 
-function noop () {}
+  _encode (enc, val) {
+    const state = { start: this.padding, end: this.padding, buffer: null }
+
+    if (Buffer.isBuffer(val)) {
+      if (state.start === 0) return val
+      state.end += val.byteLength
+    } else if (enc) {
+      enc.preencode(state, val)
+    } else {
+      val = Buffer.from(val)
+      if (state.start === 0) return val
+      state.end += val.byteLength
+    }
 
-function decode (enc, buf) {
-  return enc ? c.decode(enc, buf) : buf
+    state.buffer = Buffer.allocUnsafe(state.end)
+
+    if (enc) enc.encode(state, val)
+    else state.buffer.set(val, state.start)
+
+    return state.buffer
+  }
+
+  _decode (enc, block) {
+    block = block.subarray(this.padding)
+    if (enc) return c.decode(enc, block)
+    return block
+  }
 }
 
+function noop () {}
+
 function isStream (s) {
   return typeof s === 'object' && s && typeof s.pipe === 'function'
 }
@@ -459,3 +525,25 @@ function requireMaybe (name) {
 function toHex (buf) {
   return buf && buf.toString('hex')
 }
+
+function reduce (iter, fn, acc) {
+  for (const item of iter) acc = fn(acc, item)
+  return acc
+}
+
+function min (arr) {
+  return reduce(arr, (a, b) => Math.min(a, b), Infinity)
+}
+
+function max (arr) {
+  return reduce(arr, (a, b) => Math.max(a, b), -Infinity)
+}
+
+function preappend (blocks) {
+  const offset = this.core.tree.length
+  const fork = this.core.tree.fork
+
+  for (let i = 0; i < blocks.length; i++) {
+    this.encryption.encrypt(offset + i, blocks[i], fork)
+  }
+}

package/lib/block-encryption.js

@@ -0,0 +1,67 @@
+const sodium = require('sodium-universal')
+const c = require('compact-encoding')
+
+const nonce = Buffer.alloc(sodium.crypto_stream_NONCEBYTES)
+
+module.exports = class BlockEncryption {
+  constructor (encryptionKey, hypercoreKey) {
+    const subKeys = Buffer.alloc(2 * sodium.crypto_stream_KEYBYTES)
+
+    this.key = encryptionKey
+    this.blockKey = subKeys.subarray(0, sodium.crypto_stream_KEYBYTES)
+    this.blindingKey = subKeys.subarray(sodium.crypto_stream_KEYBYTES)
+    this.padding = 8
+
+    sodium.crypto_generichash(this.blockKey, encryptionKey, hypercoreKey)
+    sodium.crypto_generichash(this.blindingKey, this.blockKey)
+  }
+
+  encrypt (index, block, fork) {
+    const padding = block.subarray(0, this.padding)
+    block = block.subarray(this.padding)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: padding }, fork)
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+    // Zero out any previous padding.
+    nonce.fill(0, 8, 8 + padding.byteLength)
+
+    // Blind the fork ID, possibly risking reusing the nonce on a reorg of the
+    // Hypercore. This is fine as the blinding is best-effort and the latest
+    // fork ID shared on replication anyway.
+    sodium.crypto_stream_xor(
+      padding,
+      padding,
+      nonce,
+      this.blindingKey
+    )
+
+    nonce.set(padding, 8)
+
+    // The combination of a (blinded) fork ID and a block index is unique for a
+    // given Hypercore and is therefore a valid nonce for encrypting the block.
+    sodium.crypto_stream_xor(
+      block,
+      block,
+      nonce,
+      this.blockKey
+    )
+  }
+
+  decrypt (index, block) {
+    const padding = block.subarray(0, this.padding)
+    block = block.subarray(this.padding)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+    nonce.set(padding, 8)
+
+    // Decrypt the block using the blinded fork ID.
+    sodium.crypto_stream_xor(
+      block,
+      block,
+      nonce,
+      this.blockKey
+    )
+  }
+}

package/lib/core.js

@@ -214,10 +214,12 @@ module.exports = class Core {
     }
   }
 
-  async append (values, sign = this.defaultSign) {
+  async append (values, sign = this.defaultSign, hooks = {}) {
     await this._mutex.lock()
 
     try {
+      if (hooks.preappend) await hooks.preappend(values)
+
       if (!values.length) return this.tree.length
 
       const batch = this.tree.batch()

package/lib/merkle-tree.js

@@ -272,11 +272,15 @@ class ReorgBatch extends MerkleTreeBatch {
 }
 
 class ByteSeeker {
-  constructor (tree, bytes) {
+  constructor (tree, bytes, padding = 0) {
     this.tree = tree
     this.bytes = bytes
-    this.start = bytes >= tree.byteLength ? tree.length : 0
-    this.end = bytes < tree.byteLength ? tree.length : 0
+    this.padding = padding
+
+    const size = tree.byteLength - (tree.length * padding)
+
+    this.start = bytes >= size ? tree.length : 0
+    this.end = bytes < size ? tree.length : 0
   }
 
   nodes () {
@@ -287,12 +291,12 @@ class ByteSeeker {
     if (!bytes) return [0, 0]
 
     for (const node of this.tree.roots) { // all async ticks happen once we find the root so safe
-      if (bytes === node.size) {
-        return [flat.rightSpan(node.index) + 2, 0]
-      }
+      let size = node.size
+      if (this.padding > 0) size -= this.padding * flat.countLeaves(node.index)
 
-      if (bytes > node.size) {
-        bytes -= node.size
+      if (bytes === size) return [flat.rightSpan(node.index) + 2, 0]
+      if (bytes > size) {
+        bytes -= size
         continue
       }
 
@@ -301,9 +305,12 @@ class ByteSeeker {
       while ((ite.index & 1) !== 0) {
         const l = await this.tree.get(ite.leftChild(), false)
         if (l) {
-          if (l.size === bytes) return [ite.rightSpan() + 2, 0]
-          if (l.size > bytes) continue
-          bytes -= l.size
+          let size = l.size
+          if (this.padding > 0) size -= this.padding * ite.countLeaves()
+
+          if (size === bytes) return [ite.rightSpan() + 2, 0]
+          if (size > bytes) continue
+          bytes -= size
           ite.sibling()
         } else {
           ite.parent()
@@ -355,8 +362,8 @@ module.exports = class MerkleTree {
     return new MerkleTreeBatch(this)
   }
 
-  seek (bytes) {
-    return new ByteSeeker(this, bytes)
+  seek (bytes, padding) {
+    return new ByteSeeker(this, bytes, padding)
   }
 
   hash () {

package/lib/replicator.js

@@ -120,9 +120,10 @@ class Seek {
 }
 
 class Range {
-  constructor (start, end, linear) {
+  constructor (start, end, filter, linear) {
     this.start = start
     this.end = end
+    this.filter = filter
     this.linear = !!linear
     this.promise = null
     this.done = false
@@ -145,7 +146,7 @@ class Range {
     }
 
     for (; this._start < this.end; this._start++) {
-      if (!bitfield.get(this._start)) return false
+      if (this.filter(this._start) && !bitfield.get(this._start)) return false
     }
 
     return true
@@ -299,10 +300,13 @@ class RequestPool {
     const end = range.end === -1 ? peer.state.length : range.end
     if (end <= range._start) return false
 
-    if (range.linear) return !!this._requestRange(peer, range._start, end, 0, 0)
+    if (range.linear) return !!this._requestRange(peer, range._start, end, 0, 0, range.filter)
 
     const r = range._start + Math.floor(Math.random() * (end - range._start))
-    return !!(this._requestRange(peer, r, end, 0, 0) || this._requestRange(peer, range._start, r, 0, 0))
+    return !!(
+      this._requestRange(peer, r, end, 0, 0, range.filter) ||
+      this._requestRange(peer, range._start, r, 0, 0, range.filter)
+    )
   }
 
   _updateUpgrade (peer) {
@@ -331,7 +335,7 @@ class RequestPool {
     this.pendingUpgrade = null
   }
 
-  _requestRange (peer, start, end, seek, nodes) {
+  _requestRange (peer, start, end, seek, nodes, filter = tautology) {
     const remote = peer.state.bitfield
     const local = this.core.bitfield
 
@@ -339,7 +343,7 @@ class RequestPool {
     if (end === -1) end = peer.state.length
 
     for (let i = start; i < end; i++) {
-      if (!remote.get(i) || local.get(i)) continue
+      if (!filter(i) || !remote.get(i) || local.get(i)) continue
       // TODO: if this was a NO_VALUE request, retry if no blocks can be found elsewhere
       if (this.requests.has(i)) continue
 
@@ -677,8 +681,18 @@ module.exports = class Replicator {
     return promise
   }
 
-  static createRange (start, end, linear) {
-    return new Range(start, end, linear)
+  static createRange (start, end, filter, linear) {
+    // createRange(start, end)
+    if (filter === undefined) {
+      filter = tautology
+
+    // createRange(start, end, linear)
+    } else if (typeof filter === 'boolean') {
+      linear = filter
+      filter = tautology
+    }
+
+    return new Range(start, end, filter, linear)
   }
 
   addRange (range) {
@@ -826,6 +840,10 @@ function pages (core) {
 
 function noop () {}
 
+function tautology () {
+  return true
+}
+
 function log2 (n) {
   let res = 1
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.5",
+  "version": "10.0.0-alpha.9",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -31,14 +31,15 @@
     "@hyperswarm/secret-stream": "^5.0.0",
     "big-sparse-array": "^1.0.2",
     "codecs": "^2.2.0",
-    "compact-encoding": "^2.0.0",
+    "compact-encoding": "^2.5.0",
     "crc32-universal": "^1.0.1",
-    "flat-tree": "^1.7.0",
+    "flat-tree": "^1.9.0",
     "hypercore-crypto": "^2.1.1",
     "is-options": "^1.0.1",
     "random-access-file": "^2.1.4",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",
+    "sodium-universal": "^3.0.4",
     "uint64le": "^1.0.0",
     "xache": "^1.0.0"
   },

package/test/basic.js

@@ -76,3 +76,15 @@ test('storage options', async function (t) {
   t.alike(await core.get(0), Buffer.from('hello'))
   t.end()
 })
+
+test(
+  'allow publicKeys with different byteLength that 32, if opts.crypto were passed',
+  function (t) {
+    const key = Buffer.alloc(33).fill('a')
+
+    const core = new Hypercore(ram, key, { crypto: {} })
+
+    t.is(core.key, key)
+    t.pass('creating a core with more than 32 byteLength key did not throw')
+  }
+)

package/test/encryption.js

@@ -0,0 +1,121 @@
+const test = require('brittle')
+const RAM = require('random-access-memory')
+const Hypercore = require('..')
+const { create, replicate } = require('./helpers')
+
+const encryptionKey = Buffer.alloc(32, 'hello world')
+
+test('encrypted append and get', async function (t) {
+  const a = await create({ encryptionKey })
+
+  t.alike(a.encryptionKey, encryptionKey)
+
+  await a.append(['hello'])
+
+  t.is(a.byteLength, 5)
+  t.is(a.core.tree.byteLength, 5 + a.padding)
+
+  const unencrypted = await a.get(0)
+  t.alike(unencrypted, Buffer.from('hello'))
+
+  const encrypted = await a.core.blocks.get(0)
+  t.absent(encrypted.includes('hello'))
+})
+
+test('encrypted seek', async function (t) {
+  const a = await create({ encryptionKey })
+
+  await a.append(['hello', 'world', '!'])
+
+  t.alike(await a.seek(0), [0, 0])
+  t.alike(await a.seek(4), [0, 4])
+  t.alike(await a.seek(5), [1, 0])
+  t.alike(await a.seek(6), [1, 1])
+  t.alike(await a.seek(6), [1, 1])
+  t.alike(await a.seek(9), [1, 4])
+  t.alike(await a.seek(10), [2, 0])
+  t.alike(await a.seek(11), [3, 0])
+})
+
+test('encrypted replication', async function (t) {
+  const a = await create({ encryptionKey })
+
+  await a.append(['a', 'b', 'c', 'd', 'e'])
+
+  t.test('with encryption key', async function (t) {
+    t.plan(10)
+
+    const b = await create(a.key, { encryptionKey })
+
+    b.on('download', (i, block) => {
+      t.alike(block, Buffer.from([i + /* a */ 0x61]))
+    })
+
+    replicate(a, b, t)
+
+    const r = b.download({ start: 0, end: a.length })
+    await r.downloaded()
+
+    for (let i = 0; i < 5; i++) {
+      t.alike(await b.get(i), await a.get(i))
+    }
+  })
+
+  t.test('without encryption key', async function (t) {
+    const b = await create(a.key)
+
+    replicate(a, b, t)
+
+    const r = b.download({ start: 0, end: a.length })
+    await r.downloaded()
+
+    for (let i = 0; i < 5; i++) {
+      t.unlike(await b.get(i), await a.get(i))
+      t.alike(await b.get(i), await a.core.blocks.get(i))
+    }
+  })
+})
+
+test('encrypted session', async function (t) {
+  const a = await create({ encryptionKey })
+
+  await a.append(['hello'])
+
+  const s = a.session()
+
+  t.alike(a.encryptionKey, s.encryptionKey)
+  t.alike(await s.get(0), Buffer.from('hello'))
+
+  await s.append(['world'])
+
+  const unencrypted = await s.get(1)
+  t.alike(unencrypted, Buffer.from('world'))
+  t.alike(await a.get(1), unencrypted)
+
+  const encrypted = await s.core.blocks.get(1)
+  t.absent(encrypted.includes('world'))
+  t.alike(await a.core.blocks.get(1), encrypted)
+})
+
+test('encrypted session before ready core', async function (t) {
+  const a = new Hypercore(RAM, { encryptionKey })
+  const s = a.session()
+
+  await a.ready()
+
+  t.alike(a.encryptionKey, s.encryptionKey)
+
+  await a.append(['hello'])
+  t.alike(await s.get(0), Buffer.from('hello'))
+})
+
+test('replicating encrypted block returns expected data', async function (t) {
+  const a = await create({ encryptionKey })
+  const b = await create(a.key, { encryptionKey })
+
+  replicate(a, b, t)
+
+  await a.append('hej')
+  const blk = await b.get(0)
+  t.alike(blk, Buffer.from('hej'), 'freshly replicated block is decrypted')
+})

package/test/replicate.js

@@ -331,3 +331,42 @@ test('destroying a stream and re-replicating works', async function (t) {
 
   t.is(blocks.length, 33, 'downloaded 33 blocks')
 })
+
+test('replicate discrete range', async function (t) {
+  const a = await create()
+
+  await a.append(['a', 'b', 'c', 'd', 'e'])
+
+  const b = await create(a.key)
+
+  let d = 0
+  b.on('download', () => d++)
+
+  replicate(a, b, t)
+
+  const r = b.download({ blocks: [0, 2, 3] })
+  await r.downloaded()
+
+  t.is(d, 3)
+  t.alike(await b.get(0), Buffer.from('a'))
+  t.alike(await b.get(2), Buffer.from('c'))
+  t.alike(await b.get(3), Buffer.from('d'))
+})
+
+test('replicate discrete empty range', async function (t) {
+  const a = await create()
+
+  await a.append(['a', 'b', 'c', 'd', 'e'])
+
+  const b = await create(a.key)
+
+  let d = 0
+  b.on('download', () => d++)
+
+  replicate(a, b, t)
+
+  const r = b.download({ blocks: [] })
+  await r.downloaded()
+
+  t.is(d, 0)
+})