hypercore 10.0.0-alpha.0 → 10.0.0-alpha.12

package/index.js

@@ -3,6 +3,8 @@ const raf = require('random-access-file')
 const isOptions = require('is-options')
 const hypercoreCrypto = require('hypercore-crypto')
 const c = require('compact-encoding')
+const b4a = require('b4a')
+const Xache = require('xache')
 const NoiseSecretStream = require('@hyperswarm/secret-stream')
 const codecs = require('codecs')
 
@@ -11,6 +13,8 @@ const fsctl = requireMaybe('fsctl') || { lock: noop, sparse: noop }
 const Replicator = require('./lib/replicator')
 const Extensions = require('./lib/extensions')
 const Core = require('./lib/core')
+const BlockEncryption = require('./lib/block-encryption')
+const { ReadStream } = require('./lib/streams')
 
 const promises = Symbol.for('hypercore.promises')
 const inspect = Symbol.for('nodejs.util.inspect.custom')
@@ -27,14 +31,17 @@ module.exports = class Hypercore extends EventEmitter {
       opts = key
       key = null
     }
+
     if (key && typeof key === 'string') {
-      key = Buffer.from(key, 'hex')
+      key = b4a.from(key, 'hex')
     }
-    if (key && key.byteLength !== 32) {
+
+    if (!opts) opts = {}
+
+    if (!opts.crypto && key && key.byteLength !== 32) {
       throw new Error('Hypercore key should be 32 bytes')
     }
 
-    if (!opts) opts = {}
     if (!storage) storage = opts.storage
 
     this[promises] = true
@@ -43,7 +50,9 @@ module.exports = class Hypercore extends EventEmitter {
     this.crypto = opts.crypto || hypercoreCrypto
     this.core = null
     this.replicator = null
+    this.encryption = null
     this.extensions = opts.extensions || new Extensions()
+    this.cache = opts.cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (opts.cache || null)
 
     this.valueEncoding = null
     this.key = key || null
@@ -59,6 +68,8 @@ module.exports = class Hypercore extends EventEmitter {
     this.closing = null
     this.opening = opts._opening || this._open(key, storage, opts)
     this.opening.catch(noop)
+
+    this._preappend = preappend.bind(this)
   }
 
   [inspect] (depth, opts) {
@@ -84,12 +95,14 @@ module.exports = class Hypercore extends EventEmitter {
     return noiseStream.rawStream
   }
 
-  static defaultStorage (storage) {
+  static defaultStorage (storage, opts = {}) {
     if (typeof storage !== 'string') return storage
     const directory = storage
+    const toLock = opts.lock || 'oplog'
     return function createFile (name) {
-      const lock = name === 'oplog' ? fsctl.lock : null
-      const sparse = name !== 'oplog' ? fsctl.sparse : null
+      const locked = name === toLock || name.endsWith('/' + toLock)
+      const lock = locked ? fsctl.lock : null
+      const sparse = locked ? null : null // fsctl.sparse, disable sparse on windows - seems to fail for some people. TODO: investigate
       return raf(name, { directory, lock, sparse })
     }
   }
@@ -131,6 +144,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.discoveryKey = o.discoveryKey
     this.core = o.core
     this.replicator = o.replicator
+    this.encryption = o.encryption
     this.writable = !!this.sign
     this.autoClose = o.autoClose
   }
@@ -200,7 +214,7 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   get byteLength () {
-    return this.core === null ? 0 : this.core.tree.byteLength
+    return this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding)
   }
 
   get fork () {
@@ -211,6 +225,14 @@ module.exports = class Hypercore extends EventEmitter {
     return this.replicator === null ? [] : this.replicator.peers
   }
 
+  get encryptionKey () {
+    return this.encryption && this.encryption.key
+  }
+
+  get padding () {
+    return this.encryption === null ? 0 : this.encryption.padding
+  }
+
   ready () {
     return this.opening
   }
@@ -264,6 +286,10 @@ module.exports = class Hypercore extends EventEmitter {
     this.key = this.core.header.signer.publicKey
     this.writable = !!this.sign
 
+    if (!this.encryption && opts.encryptionKey) {
+      this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
+    }
+
     this.extensions.attach(this.replicator)
     this.opened = true
 
@@ -279,8 +305,13 @@ module.exports = class Hypercore extends EventEmitter {
   _oncoreupdate (status, bitfield, value, from) {
     if (status !== 0) {
       for (let i = 0; i < this.sessions.length; i++) {
-        if ((status & 0b10) !== 0) this.sessions[i].emit('truncate', this.core.tree.fork)
-        if ((status & 0b01) !== 0) this.sessions[i].emit('append')
+        if ((status & 0b10) !== 0) {
+          if (this.cache) this.cache.clear()
+          this.sessions[i].emit('truncate', this.core.tree.fork)
+        }
+        if ((status & 0b01) !== 0) {
+          this.sessions[i].emit('append')
+        }
       }
 
       this.replicator.broadcastInfo()
@@ -293,8 +324,10 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     if (value) {
+      const byteLength = value.byteLength - this.padding
+
       for (let i = 0; i < this.sessions.length; i++) {
-        this.sessions[i].emit('download', bitfield.start, value, from)
+        this.sessions[i].emit('download', bitfield.start, byteLength, from)
       }
     }
   }
@@ -331,7 +364,7 @@ module.exports = class Hypercore extends EventEmitter {
   async seek (bytes) {
     if (this.opened === false) await this.opening
 
-    const s = this.core.tree.seek(bytes)
+    const s = this.core.tree.seek(bytes, this.padding)
 
     return (await s.update()) || this.replicator.requestSeek(s)
   }
@@ -344,25 +377,59 @@ module.exports = class Hypercore extends EventEmitter {
 
   async get (index, opts) {
     if (this.opened === false) await this.opening
+    const c = this.cache && this.cache.get(index)
+    if (c) return c
+    const fork = this.core.tree.fork
+    const b = await this._get(index, opts)
+    if (this.cache && fork === this.core.tree.fork && b) this.cache.set(index, b)
+    return b
+  }
+
+  async _get (index, opts) {
     const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
 
-    if (this.core.bitfield.get(index)) return decode(encoding, await this.core.blocks.get(index))
-    if (opts && opts.onwait) opts.onwait(index)
+    let block
+
+    if (this.core.bitfield.get(index)) {
+      block = await this.core.blocks.get(index)
+    } else {
+      if (opts && opts.onwait) opts.onwait(index)
+      block = await this.replicator.requestBlock(index)
+    }
+
+    if (this.encryption) this.encryption.decrypt(index, block)
+    return this._decode(encoding, block)
+  }
 
-    return decode(encoding, await this.replicator.requestBlock(index))
+  createReadStream (opts) {
+    return new ReadStream(this, opts)
   }
 
   download (range) {
-    const start = (range && range.start) || 0
-    const end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
     const linear = !!(range && range.linear)
 
-    // TODO: support range.blocks
+    let start
+    let end
+    let filter
+
+    if (range && range.blocks) {
+      const blocks = range.blocks instanceof Set
+        ? range.blocks
+        : new Set(range.blocks)
+
+      start = range.start || (blocks.size ? min(range.blocks) : 0)
+      end = range.end || (blocks.size ? max(range.blocks) + 1 : 0)
+
+      filter = (i) => blocks.has(i)
+    } else {
+      start = (range && range.start) || 0
+      end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
+    }
 
-    const r = Replicator.createRange(start, end, linear)
+    const r = Replicator.createRange(start, end, filter, linear)
 
     if (this.opened) this.replicator.addRange(r)
-    else this.opening.then(() => this.replicator.addRange(r))
+    else this.opening.then(() => this.replicator.addRange(r), noop)
 
     return r
   }
@@ -392,22 +459,16 @@ module.exports = class Hypercore extends EventEmitter {
     if (this.opened === false) await this.opening
     if (this.writable === false) throw new Error('Core is not writable')
 
-    const blks = Array.isArray(blocks) ? blocks : [blocks]
-    const buffers = new Array(blks.length)
-
-    for (let i = 0; i < blks.length; i++) {
-      const blk = blks[i]
+    blocks = Array.isArray(blocks) ? blocks : [blocks]
 
-      const buf = Buffer.isBuffer(blk)
-        ? blk
-        : this.valueEncoding
-          ? c.encode(this.valueEncoding, blk)
-          : Buffer.from(blk)
+    const preappend = this.encryption && this._preappend
+    const buffers = new Array(blocks.length)
 
-      buffers[i] = buf
+    for (let i = 0; i < blocks.length; i++) {
+      buffers[i] = this._encode(this.valueEncoding, blocks[i])
     }
 
-    return await this.core.append(buffers, this.sign)
+    return await this.core.append(buffers, this.sign, { preappend })
   }
 
   registerExtension (name, handlers) {
@@ -418,14 +479,38 @@ module.exports = class Hypercore extends EventEmitter {
   onextensionupdate () {
     if (this.replicator !== null) this.replicator.broadcastOptions()
   }
-}
 
-function noop () {}
+  _encode (enc, val) {
+    const state = { start: this.padding, end: this.padding, buffer: null }
+
+    if (b4a.isBuffer(val)) {
+      if (state.start === 0) return val
+      state.end += val.byteLength
+    } else if (enc) {
+      enc.preencode(state, val)
+    } else {
+      val = b4a.from(val)
+      if (state.start === 0) return val
+      state.end += val.byteLength
+    }
+
+    state.buffer = b4a.allocUnsafe(state.end)
+
+    if (enc) enc.encode(state, val)
+    else state.buffer.set(val, state.start)
 
-function decode (enc, buf) {
-  return enc ? c.decode(enc, buf) : buf
+    return state.buffer
+  }
+
+  _decode (enc, block) {
+    block = block.subarray(this.padding)
+    if (enc) return c.decode(enc, block)
+    return block
+  }
 }
 
+function noop () {}
+
 function isStream (s) {
   return typeof s === 'object' && s && typeof s.pipe === 'function'
 }
@@ -439,5 +524,27 @@ function requireMaybe (name) {
 }
 
 function toHex (buf) {
-  return buf && buf.toString('hex')
+  return buf && b4a.toString(buf, 'hex')
+}
+
+function reduce (iter, fn, acc) {
+  for (const item of iter) acc = fn(acc, item)
+  return acc
+}
+
+function min (arr) {
+  return reduce(arr, (a, b) => Math.min(a, b), Infinity)
+}
+
+function max (arr) {
+  return reduce(arr, (a, b) => Math.max(a, b), -Infinity)
+}
+
+function preappend (blocks) {
+  const offset = this.core.tree.length
+  const fork = this.core.tree.fork
+
+  for (let i = 0; i < blocks.length; i++) {
+    this.encryption.encrypt(offset + i, blocks[i], fork)
+  }
 }

package/lib/bitfield.js

@@ -1,6 +1,7 @@
 // TODO: needs massive improvements obvs
 
 const BigSparseArray = require('big-sparse-array')
+const b4a = require('b4a')
 
 class FixedBitfield {
   constructor (index, bitfield) {
@@ -116,9 +117,9 @@ module.exports = class Bitfield {
       let error = null
 
       for (const page of this.unflushed) {
-        const b = Buffer.from(page.bitfield.buffer, page.bitfield.byteOffset, page.bitfield.byteLength)
+        const buf = b4a.from(page.bitfield.buffer, page.bitfield.byteOffset, page.bitfield.byteLength)
         page.dirty = false
-        this.storage.write(page.index * 4096, b, done)
+        this.storage.write(page.index * 4096, buf, done)
       }
 
       function done (err) {

package/lib/block-encryption.js

@@ -0,0 +1,68 @@
+const sodium = require('sodium-universal')
+const c = require('compact-encoding')
+const b4a = require('b4a')
+
+const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
+
+module.exports = class BlockEncryption {
+  constructor (encryptionKey, hypercoreKey) {
+    const subKeys = b4a.alloc(2 * sodium.crypto_stream_KEYBYTES)
+
+    this.key = encryptionKey
+    this.blockKey = subKeys.subarray(0, sodium.crypto_stream_KEYBYTES)
+    this.blindingKey = subKeys.subarray(sodium.crypto_stream_KEYBYTES)
+    this.padding = 8
+
+    sodium.crypto_generichash(this.blockKey, encryptionKey, hypercoreKey)
+    sodium.crypto_generichash(this.blindingKey, this.blockKey)
+  }
+
+  encrypt (index, block, fork) {
+    const padding = block.subarray(0, this.padding)
+    block = block.subarray(this.padding)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: padding }, fork)
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+    // Zero out any previous padding.
+    nonce.fill(0, 8, 8 + padding.byteLength)
+
+    // Blind the fork ID, possibly risking reusing the nonce on a reorg of the
+    // Hypercore. This is fine as the blinding is best-effort and the latest
+    // fork ID shared on replication anyway.
+    sodium.crypto_stream_xor(
+      padding,
+      padding,
+      nonce,
+      this.blindingKey
+    )
+
+    nonce.set(padding, 8)
+
+    // The combination of a (blinded) fork ID and a block index is unique for a
+    // given Hypercore and is therefore a valid nonce for encrypting the block.
+    sodium.crypto_stream_xor(
+      block,
+      block,
+      nonce,
+      this.blockKey
+    )
+  }
+
+  decrypt (index, block) {
+    const padding = block.subarray(0, this.padding)
+    block = block.subarray(this.padding)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+    nonce.set(padding, 8)
+
+    // Decrypt the block using the blinded fork ID.
+    sodium.crypto_stream_xor(
+      block,
+      block,
+      nonce,
+      this.blockKey
+    )
+  }
+}

package/lib/block-store.js

@@ -1,3 +1,5 @@
+const b4a = require('b4a')
+
 module.exports = class BlockStore {
   constructor (storage, tree) {
     this.storage = storage
@@ -15,7 +17,7 @@ module.exports = class BlockStore {
 
   putBatch (i, batch, offset) {
     if (batch.length === 0) return Promise.resolve()
-    return this.put(i, batch.length === 1 ? batch[0] : Buffer.concat(batch), offset)
+    return this.put(i, batch.length === 1 ? batch[0] : b4a.concat(batch), offset)
   }
 
   clear () {

package/lib/core.js

@@ -1,4 +1,3 @@
-const fsctl = requireMaybe('fsctl')
 const hypercoreCrypto = require('hypercore-crypto')
 const Oplog = require('./oplog')
 const Mutex = require('./mutex')
@@ -7,8 +6,6 @@ const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
 const { oplogHeader, oplogEntry } = require('./messages')
 
-const lock = fsctl ? fsctl.lock : null
-
 module.exports = class Core {
   constructor (header, crypto, oplog, tree, blocks, bitfield, sign, onupdate) {
     this.onupdate = onupdate
@@ -29,7 +26,7 @@ module.exports = class Core {
   }
 
   static async open (storage, opts = {}) {
-    const oplogFile = storage('oplog', { lock })
+    const oplogFile = storage('oplog')
     const treeFile = storage('tree')
     const bitfieldFile = storage('bitfield')
     const dataFile = storage('data')
@@ -217,10 +214,12 @@ module.exports = class Core {
     }
   }
 
-  async append (values, sign = this.defaultSign) {
+  async append (values, sign = this.defaultSign, hooks = {}) {
     await this._mutex.lock()
 
     try {
+      if (hooks.preappend) await hooks.preappend(values)
+
       if (!values.length) return this.tree.length
 
       const batch = this.tree.batch()
@@ -467,11 +466,3 @@ function updateUserData (list, key, value) {
 }
 
 function noop () {}
-
-function requireMaybe (name) {
-  try {
-    return require(name)
-  } catch {
-    return null
-  }
-}

package/lib/merkle-tree.js

@@ -1,9 +1,10 @@
 const flat = require('flat-tree')
 const crypto = require('hypercore-crypto')
-const uint64le = require('uint64le')
+const c = require('compact-encoding')
+const b4a = require('b4a')
 
-const BLANK_HASH = Buffer.alloc(32)
-const OLD_TREE = Buffer.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
+const BLANK_HASH = b4a.alloc(32)
+const OLD_TREE = b4a.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
 
 class NodeQueue {
   constructor (nodes, extra = null) {
@@ -272,11 +273,15 @@ class ReorgBatch extends MerkleTreeBatch {
 }
 
 class ByteSeeker {
-  constructor (tree, bytes) {
+  constructor (tree, bytes, padding = 0) {
     this.tree = tree
     this.bytes = bytes
-    this.start = bytes >= tree.byteLength ? tree.length : 0
-    this.end = bytes < tree.byteLength ? tree.length : 0
+    this.padding = padding
+
+    const size = tree.byteLength - (tree.length * padding)
+
+    this.start = bytes >= size ? tree.length : 0
+    this.end = bytes < size ? tree.length : 0
   }
 
   nodes () {
@@ -287,12 +292,12 @@ class ByteSeeker {
     if (!bytes) return [0, 0]
 
     for (const node of this.tree.roots) { // all async ticks happen once we find the root so safe
-      if (bytes === node.size) {
-        return [flat.rightSpan(node.index) + 2, 0]
-      }
+      let size = node.size
+      if (this.padding > 0) size -= this.padding * flat.countLeaves(node.index)
 
-      if (bytes > node.size) {
-        bytes -= node.size
+      if (bytes === size) return [flat.rightSpan(node.index) + 2, 0]
+      if (bytes > size) {
+        bytes -= size
         continue
       }
 
@@ -301,9 +306,12 @@ class ByteSeeker {
       while ((ite.index & 1) !== 0) {
         const l = await this.tree.get(ite.leftChild(), false)
         if (l) {
-          if (l.size === bytes) return [ite.rightSpan() + 2, 0]
-          if (l.size > bytes) continue
-          bytes -= l.size
+          let size = l.size
+          if (this.padding > 0) size -= this.padding * ite.countLeaves()
+
+          if (size === bytes) return [ite.rightSpan() + 2, 0]
+          if (size > bytes) continue
+          bytes -= size
           ite.sibling()
         } else {
           ite.parent()
@@ -355,8 +363,8 @@ module.exports = class MerkleTree {
     return new MerkleTreeBatch(this)
   }
 
-  seek (bytes) {
-    return new ByteSeeker(this, bytes)
+  seek (bytes, padding) {
+    return new ByteSeeker(this, bytes, padding)
   }
 
   hash () {
@@ -433,17 +441,23 @@ module.exports = class MerkleTree {
     // TODO: write neighbors together etc etc
     // TODO: bench loading a full disk page and copy to that instead
     return new Promise((resolve, reject) => {
-      const slab = Buffer.allocUnsafe(40 * this.flushing.size)
+      const slab = b4a.allocUnsafe(40 * this.flushing.size)
 
       let error = null
       let missing = this.flushing.size + 1
       let offset = 0
 
       for (const node of this.flushing.values()) {
-        const b = slab.slice(offset, offset += 40)
-        uint64le.encode(node.size, b, 0)
-        node.hash.copy(b, 8)
-        this.storage.write(node.index * 40, b, done)
+        const state = {
+          start: 0,
+          end: 40,
+          buffer: slab.subarray(offset, offset += 40)
+        }
+
+        c.uint64.encode(state, node.size)
+        c.raw.encode(state, node.hash)
+
+        this.storage.write(node.index * 40, state.buffer, done)
       }
 
       done(null)
@@ -1038,10 +1052,10 @@ function getStoredNode (storage, index, error) {
         return
       }
 
-      const hash = data.slice(8)
-      const size = uint64le.decode(data, 0)
+      const hash = data.subarray(8)
+      const size = c.decode(c.uint64, data)
 
-      if (size === 0 && Buffer.compare(hash, BLANK_HASH) === 0) {
+      if (size === 0 && b4a.compare(hash, BLANK_HASH) === 0) {
         if (error) reject(new Error('Could not load node: ' + index))
         else resolve(null)
         return
@@ -1088,9 +1102,9 @@ function log2 (n) {
 }
 
 function signable (hash, length, fork) {
-  const buf = Buffer.alloc(48)
-  hash.copy(buf)
-  uint64le.encode(length, buf, 32)
-  uint64le.encode(fork, buf, 40)
-  return buf
+  const state = { start: 0, end: 48, buffer: b4a.alloc(48) }
+  c.raw.encode(state, hash)
+  c.uint64.encode(state, length)
+  c.uint64.encode(state, fork)
+  return state.buffer
 }

package/lib/oplog.js

@@ -1,4 +1,5 @@
 const cenc = require('compact-encoding')
+const b4a = require('b4a')
 const crc32 = require('crc32-universal')
 
 module.exports = class Oplog {
@@ -133,7 +134,7 @@ module.exports = class Oplog {
     return new Promise((resolve, reject) => {
       this.storage.open(err => {
         if (err && err.code !== 'ENOENT') return reject(err)
-        if (err) return resolve(Buffer.alloc(0))
+        if (err) return resolve(b4a.alloc(0))
         this.storage.stat((err, stat) => {
           if (err && err.code !== 'ENOENT') return reject(err)
           this.storage.read(0, stat.size, (err, buf) => {
@@ -151,7 +152,7 @@ module.exports = class Oplog {
     const bit = (this._headers[i] + 1) & 1
 
     this.headerEncoding.preencode(state, header)
-    state.buffer = Buffer.allocUnsafe(state.end)
+    state.buffer = b4a.allocUnsafe(state.end)
     this.headerEncoding.encode(state, header)
     this._addHeader(state, state.end - 8, bit, 0)
 
@@ -187,7 +188,7 @@ module.exports = class Oplog {
       this.entryEncoding.preencode(state, batch[i])
     }
 
-    state.buffer = Buffer.allocUnsafe(state.end)
+    state.buffer = b4a.allocUnsafe(state.end)
 
     for (let i = 0; i < batch.length; i++) {
       const start = state.start += 8 // space for header