hypercore 11.2.0 → 11.4.0

package/README.md

@@ -68,6 +68,8 @@ You can also set valueEncoding to any [compact-encoding](https://github.com/comp
 
 valueEncodings will be applied to individual blocks, even if you append batches. If you want to control encoding at the batch-level, you can use the `encodeBatch` option, which is a function that takes a batch and returns a binary-encoded batch. If you provide a custom valueEncoding, it will not be applied prior to `encodeBatch`.
 
+The user may provide a custom encryption module as `opts.encryption`, which should satisfy the [HypercoreEncryption](https://github.com/holepunchto/hypercore-encryption) interface.
+
 #### `const { length, byteLength } = await core.append(block)`
 
 Append a block of data (or an array of blocks) to the core.

package/index.js

@@ -3,10 +3,8 @@ const isOptions = require('is-options')
 const crypto = require('hypercore-crypto')
 const CoreStorage = require('hypercore-storage')
 const c = require('compact-encoding')
-const sodium = require('sodium-universal')
 const b4a = require('b4a')
 const NoiseSecretStream = require('@hyperswarm/secret-stream')
-const HypercoreEncryption = require('hypercore-encryption')
 const Protomux = require('protomux')
 const id = require('hypercore-id-encoding')
 const safetyCatch = require('safety-catch')
@@ -15,6 +13,7 @@ const unslab = require('unslab')
 const Core = require('./lib/core')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
+const DefaultEncryption = require('./lib/default-encryption')
 const caps = require('./lib/caps')
 const { manifestHash, createManifest } = require('./lib/verifier')
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
@@ -140,6 +139,8 @@ class Hypercore extends EventEmitter {
 
   static MAX_SUGGESTED_BLOCK_SIZE = MAX_SUGGESTED_BLOCK_SIZE
 
+  static DefaultEncryption = DefaultEncryption
+
   static key (manifest, { compat, version, namespace } = {}) {
     if (b4a.isBuffer(manifest)) manifest = { version, signers: [{ publicKey: manifest, namespace }] }
     return compat ? manifest.signers[0].publicKey : manifestHash(createManifest(manifest))
@@ -150,7 +151,7 @@ class Hypercore extends EventEmitter {
   }
 
   static blockEncryptionKey (key, encryptionKey) {
-    return HypercoreEncryption.blockEncryptionKey(key, encryptionKey)
+    return DefaultEncryption.blockEncryptionKey(key, encryptionKey)
   }
 
   static getProtocolMuxer (stream) {
@@ -235,7 +236,7 @@ class Hypercore extends EventEmitter {
   }
 
   setEncryptionKey (encryptionKey, opts) {
-    const encryption = this._getLegacyEncryption(encryptionKey, !!(opts && opts.block))
+    const encryption = this._getEncryptionProvider(encryptionKey, !!(opts && opts.block))
     return this.setEncryption(encryption, opts)
   }
 
@@ -247,8 +248,8 @@ class Hypercore extends EventEmitter {
       return
     }
 
-    if (!HypercoreEncryption.isHypercoreEncryption(encryption)) {
-      throw new Error('Expected hypercore encryption provider')
+    if (!isEncryptionProvider(encryption)) {
+      throw new Error('Provider does not satisfy HypercoreEncryption interface')
     }
 
     this.encryption = encryption
@@ -328,13 +329,12 @@ class Hypercore extends EventEmitter {
 
     if (this.keyPair === null) this.keyPair = opts.keyPair || this.core.header.keyPair
 
-    if (!this.core.encryption) {
-      const e = getEncryptionOption(opts)
-
-      if (HypercoreEncryption.isHypercoreEncryption(e)) {
+    const e = getEncryptionOption(opts)
+    if (!this.core.encryption && e) {
+      if (isEncryptionProvider(e)) {
         this.core.encryption = e
-      } else if (e) {
-        this.core.encryption = this._getLegacyEncryption(e.key, e.block)
+      } else {
+        this.core.encryption = this._getEncryptionProvider(e.key, e.block)
       }
     }
 
@@ -592,12 +592,16 @@ class Hypercore extends EventEmitter {
     return this.state.fork
   }
 
-  get peers () {
-    return this.opened === false ? [] : this.core.replicator.peers
+  get padding () {
+    if (this.encryption && this.key && this.manifest) {
+      return this.encryption.padding(this.core, this.length)
+    }
+
+    return 0
   }
 
-  get padding () {
-    return this.encryption === null ? 0 : this.encryption.padding
+  get peers () {
+    return this.opened === false ? [] : this.core.replicator.peers
   }
 
   get globalCache () {
@@ -700,6 +704,18 @@ class Hypercore extends EventEmitter {
     if (this.opened === false) await this.opening
     if (!isValidIndex(bytes)) throw ASSERTION('seek is invalid')
 
+    const activeRequests = (opts && opts.activeRequests) || this.activeRequests
+
+    if (this.encryption && !this.core.manifest) {
+      const req = this.replicator.addUpgrade(activeRequests)
+      try {
+        await req.promise
+      } catch (err) {
+        if (isSessionMoved(err)) return this.seek(bytes, opts)
+        throw err
+      }
+    }
+
     const s = MerkleTree.seek(this.state, bytes, this.padding)
 
     const offset = await s.update()
@@ -709,7 +725,6 @@ class Hypercore extends EventEmitter {
 
     if (!this._shouldWait(opts, this.wait)) return null
 
-    const activeRequests = (opts && opts.activeRequests) || this.activeRequests
     const req = this.core.replicator.addSeek(activeRequests, s)
 
     const timeout = opts && opts.timeout !== undefined ? opts.timeout : this.timeout
@@ -772,12 +787,10 @@ class Hypercore extends EventEmitter {
       // Copy the block as it might be shared with other sessions.
       block = b4a.from(block)
 
-      if (this.encryption.compat !== this.core.compat) this._updateEncryption()
-
-      await this.encryption.decrypt(index, block)
+      await this.encryption.decrypt(index, block, this.core)
     }
 
-    return this._decode(encoding, block)
+    return this._decode(encoding, block, index)
   }
 
   async clear (start, end = start + 1, opts) {
@@ -923,7 +936,6 @@ class Hypercore extends EventEmitter {
     blocks = Array.isArray(blocks) ? blocks : [blocks]
 
     const preappend = this.encryption && this._preappend
-    if (preappend) await this.encryption.ready()
 
     const buffers = this.encodeBatch !== null ? this.encodeBatch(blocks) : new Array(blocks.length)
 
@@ -1042,8 +1054,8 @@ class Hypercore extends EventEmitter {
     return state.buffer
   }
 
-  _decode (enc, block) {
-    if (this.padding) block = block.subarray(this.padding)
+  _decode (enc, block, index) {
+    if (this.encryption) block = block.subarray(this.encryption.padding(this.core, index))
     try {
       if (enc) return c.decode(enc, block)
     } catch {
@@ -1052,23 +1064,9 @@ class Hypercore extends EventEmitter {
     return block
   }
 
-  _updateEncryption () {
-    const e = this.encryption
-    if (HypercoreEncryption.isHypercoreEncryption(e)) return
-
-    this.encryption = this._getLegacyEncryption(e.key, e.block)
-
-    if (e === this.core.encryption) this.core.encryption = this.encryption
-  }
-
-  _getLegacyEncryption (encryptionKey, block) {
+  _getEncryptionProvider (encryptionKey, block) {
     if (!encryptionKey) return null
-
-    const blockKey = block
-      ? encryptionKey
-      : getLegacyBlockKey(this.key, encryptionKey, this.core.compat)
-
-    return HypercoreEncryption.createLegacyProvider(blockKey)
+    return new DefaultEncryption(encryptionKey, this.key, { block, compat: this.core.compat })
   }
 }
 
@@ -1086,10 +1084,8 @@ async function preappend (blocks) {
   const offset = this.state.length
   const fork = this.state.encryptionFork
 
-  if (this.encryption.compat !== this.core.compat) this._updateEncryption()
-
   for (let i = 0; i < blocks.length; i++) {
-    await this.encryption.encrypt(offset + i, blocks[i], fork)
+    await this.encryption.encrypt(offset + i, blocks[i], fork, this.core)
   }
 }
 
@@ -1157,11 +1153,10 @@ function getEncryptionOption (opts) {
   return b4a.isBuffer(opts.encryption) ? { key: opts.encryption } : opts.encryption
 }
 
-function getLegacyBlockKey (hypercoreKey, encryptionKey, compat) {
-  const key = b4a.alloc(HypercoreEncryption.KEYBYTES)
-
-  if (compat) sodium.crypto_generichash_batch(key, [encryptionKey], hypercoreKey)
-  else sodium.crypto_generichash_batch(key, [caps.LEGACY_BLOCK_ENCRYPTION, hypercoreKey, encryptionKey])
+function isEncryptionProvider (e) {
+  return e && isFunction(e.padding) && isFunction(e.encrypt) && isFunction(e.decrypt)
+}
 
-  return key
+function isFunction (fn) {
+  return !!fn && typeof fn === 'function'
 }

package/lib/caps.js

@@ -12,12 +12,12 @@ const [
   REPLICATE_RESPONDER,
   MANIFEST,
   DEFAULT_NAMESPACE,
-  LEGACY_BLOCK_ENCRYPTION
+  DEFAULT_ENCRYPTION
 ] = crypto.namespace('hypercore', 6)
 
 exports.MANIFEST = MANIFEST
 exports.DEFAULT_NAMESPACE = DEFAULT_NAMESPACE
-exports.LEGACY_BLOCK_ENCRYPTION = LEGACY_BLOCK_ENCRYPTION
+exports.DEFAULT_ENCRYPTION = DEFAULT_ENCRYPTION
 
 exports.replicate = function (isInitiator, key, handshakeHash) {
   const out = b4a.allocUnsafe(32)

package/lib/default-encryption.js

@@ -0,0 +1,116 @@
+const sodium = require('sodium-universal')
+const c = require('compact-encoding')
+const b4a = require('b4a')
+const { DEFAULT_ENCRYPTION } = require('./caps')
+
+const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
+
+module.exports = class DefaultEncryption {
+  static PADDING = 8
+
+  constructor (encryptionKey, hypercoreKey, { block = false, compat = true } = {}) {
+    this.key = encryptionKey
+    this.compat = compat
+
+    const keys = DefaultEncryption.deriveKeys(encryptionKey, hypercoreKey, block, compat)
+
+    this.blockKey = keys.block
+    this.blindingKey = keys.blinding
+  }
+
+  static deriveKeys (encryptionKey, hypercoreKey, block, compat) {
+    const subKeys = b4a.alloc(2 * sodium.crypto_stream_KEYBYTES)
+
+    const blockKey = block ? encryptionKey : subKeys.subarray(0, sodium.crypto_stream_KEYBYTES)
+    const blindingKey = subKeys.subarray(sodium.crypto_stream_KEYBYTES)
+
+    if (!block) {
+      if (compat) sodium.crypto_generichash_batch(blockKey, [encryptionKey], hypercoreKey)
+      else sodium.crypto_generichash_batch(blockKey, [DEFAULT_ENCRYPTION, hypercoreKey, encryptionKey])
+    }
+
+    sodium.crypto_generichash(blindingKey, blockKey)
+
+    return {
+      blinding: blindingKey,
+      block: blockKey
+    }
+  }
+
+  static blockEncryptionKey (hypercoreKey, encryptionKey) {
+    const blockKey = b4a.alloc(sodium.crypto_stream_KEYBYTES)
+    sodium.crypto_generichash_batch(blockKey, [DEFAULT_ENCRYPTION, hypercoreKey, encryptionKey])
+    return blockKey
+  }
+
+  static encrypt (index, block, fork, blockKey, blindingKey) {
+    const padding = block.subarray(0, DefaultEncryption.PADDING)
+    block = block.subarray(DefaultEncryption.PADDING)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: padding }, fork)
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+    // Zero out any previous padding.
+    nonce.fill(0, 8, 8 + padding.byteLength)
+
+    // Blind the fork ID, possibly risking reusing the nonce on a reorg of the
+    // Hypercore. This is fine as the blinding is best-effort and the latest
+    // fork ID shared on replication anyway.
+    sodium.crypto_stream_xor(
+      padding,
+      padding,
+      nonce,
+      blindingKey
+    )
+
+    nonce.set(padding, 8)
+
+    // The combination of a (blinded) fork ID and a block index is unique for a
+    // given Hypercore and is therefore a valid nonce for encrypting the block.
+    sodium.crypto_stream_xor(
+      block,
+      block,
+      nonce,
+      blockKey
+    )
+  }
+
+  static decrypt (index, block, blockKey) {
+    const padding = block.subarray(0, DefaultEncryption.PADDING)
+    block = block.subarray(DefaultEncryption.PADDING)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+    nonce.set(padding, 8)
+
+    // Decrypt the block using the blinded fork ID.
+    sodium.crypto_stream_xor(
+      block,
+      block,
+      nonce,
+      blockKey
+    )
+  }
+
+  encrypt (index, block, fork, core) {
+    if (core.compat !== this.compat) this._reload(core)
+    return DefaultEncryption.encrypt(index, block, fork, this.blockKey, this.blindingKey)
+  }
+
+  decrypt (index, block, core) {
+    if (core.compat !== this.compat) this._reload(core)
+    return DefaultEncryption.decrypt(index, block, this.blockKey)
+  }
+
+  padding () {
+    return DefaultEncryption.PADDING
+  }
+
+  _reload (core) {
+    const block = b4a.equals(this.key, this.blockKey)
+    const keys = DefaultEncryption.deriveKeys(this.key, core.key, { block, compat: core.compat })
+
+    this.blockKey = keys.blockKey
+    this.blindingKey = keys.blindingKey
+  }
+}

package/lib/messages.js

@@ -9,6 +9,7 @@ const EMPTY = b4a.alloc(0)
 const MANIFEST_PATCH = 0b00000001
 const MANIFEST_PROLOGUE = 0b00000010
 const MANIFEST_LINKED = 0b00000100
+const MANIFEST_USER_DATA = 0b00001000
 
 const hashes = {
   preencode (state, m) {
@@ -141,7 +142,8 @@ const manifestv0 = {
           hash: c.fixed32.decode(state),
           length: 0
         },
-        linked: null
+        linked: null,
+        userData: null
       }
     }
 
@@ -153,7 +155,8 @@ const manifestv0 = {
         quorum: 1,
         signers: [signer.decode(state)],
         prologue: null,
-        linked: null
+        linked: null,
+        userData: null
       }
     }
 
@@ -166,7 +169,8 @@ const manifestv0 = {
       quorum: c.uint.decode(state),
       signers: signerArray.decode(state),
       prologue: null,
-      linked: null
+      linked: null,
+      userData: null
     }
   }
 }
@@ -176,6 +180,7 @@ const fixed32Array = c.array(c.fixed32)
 const manifest = exports.manifest = {
   preencode (state, m) {
     state.end++ // version
+
     if (m.version === 0) return manifestv0.preencode(state, m)
 
     state.end++ // flags
@@ -183,34 +188,35 @@ const manifest = exports.manifest = {
 
     c.uint.preencode(state, m.quorum)
     signerArray.preencode(state, m.signers)
-    if (m.prologue) prologue.preencode(state, m.prologue)
 
-    if (m.linked) {
-      fixed32Array.preencode(state, m.linked)
-    }
+    if (m.prologue) prologue.preencode(state, m.prologue)
+    if (m.linked) fixed32Array.preencode(state, m.linked)
+    if (m.userData) c.buffer.preencode(state, m.userData)
   },
   encode (state, m) {
     c.uint.encode(state, m.version)
+
     if (m.version === 0) return manifestv0.encode(state, m)
 
     let flags = 0
     if (m.allowPatch) flags |= MANIFEST_PATCH
     if (m.prologue) flags |= MANIFEST_PROLOGUE
     if (m.linked) flags |= MANIFEST_LINKED
+    if (m.userData) flags |= MANIFEST_USER_DATA
 
     c.uint.encode(state, flags)
     hashes.encode(state, m.hash)
 
     c.uint.encode(state, m.quorum)
     signerArray.encode(state, m.signers)
-    if (m.prologue) prologue.encode(state, m.prologue)
 
-    if (m.linked) {
-      fixed32Array.encode(state, m.linked)
-    }
+    if (m.prologue) prologue.encode(state, m.prologue)
+    if (m.linked) fixed32Array.encode(state, m.linked)
+    if (m.userData) c.buffer.encode(state, m.userData)
   },
   decode (state) {
     const version = c.uint.decode(state)
+
     if (version === 0) return manifestv0.decode(state)
     if (version > 2) throw new Error('Unknown version: ' + version)
 
@@ -222,6 +228,7 @@ const manifest = exports.manifest = {
     const hasPatch = (flags & MANIFEST_PATCH) !== 0
     const hasPrologue = (flags & MANIFEST_PROLOGUE) !== 0
     const hasLinked = (flags & MANIFEST_LINKED) !== 0
+    const hasUserData = (flags & MANIFEST_USER_DATA) !== 0
 
     return {
       version,
@@ -230,7 +237,8 @@ const manifest = exports.manifest = {
       quorum,
       signers,
       prologue: hasPrologue ? prologue.decode(state) : null,
-      linked: hasLinked ? fixed32Array.decode(state) : null
+      linked: hasLinked ? fixed32Array.decode(state) : null,
+      userData: hasUserData ? c.buffer.decode(state) : null
     }
   }
 }
@@ -936,7 +944,9 @@ oplog.header = {
             namespace: DEFAULT_NAMESPACE,
             publicKey: old.signer.publicKey
           }],
-          prologue: null
+          prologue: null,
+          linked: null,
+          userData: null
         },
         keyPair: old.signer.secretKey ? old.signer : null,
         userData: old.userData,

package/lib/verifier.js

@@ -183,7 +183,8 @@ module.exports = class Verifier {
         publicKey
       }],
       prologue: null,
-      linked: null
+      linked: null,
+      userData: null
     }
   }
 
@@ -196,13 +197,14 @@ module.exports = class Verifier {
     if (!inp) return null
 
     const manifest = {
-      version: getManifestVersion(inp), // only v2 if linked are present
+      version: getManifestVersion(inp), // defaults to v1
       hash: 'blake2b',
       allowPatch: !!inp.allowPatch,
       quorum: defaultQuorum(inp),
       signers: inp.signers ? inp.signers.map(parseSigner) : [],
       prologue: null,
-      linked: null
+      linked: null,
+      userData: inp.userData || null
     }
 
     if (inp.hash && inp.hash !== 'blake2b') throw BAD_ARGUMENT('Only Blake2b hashes are supported')
@@ -216,8 +218,12 @@ module.exports = class Verifier {
       manifest.prologue.hash = unslab(manifest.prologue.hash)
     }
 
+    if (manifest.userData !== null && manifest.version < 2) {
+      throw BAD_ARGUMENT('Invalid field: userData')
+    }
+
     if (inp.linked && inp.linked.length) {
-      if (manifest.version < 2) throw BAD_ARGUMENT('Invalid field')
+      if (manifest.version < 2) throw BAD_ARGUMENT('Invalid field: linked')
 
       for (const key of inp.linked) {
         if (!(b4a.isBuffer(key) && key.byteLength === 32)) {
@@ -328,5 +334,6 @@ function proofToVersion1 (proof) {
 function getManifestVersion (inp) {
   if (typeof inp.version === 'number') return inp.version
   if (inp.linked && inp.linked.length) return 2
+  if (inp.userData !== null) return 2
   return 1
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.2.0",
+  "version": "11.4.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -50,7 +50,6 @@
     "fast-fifo": "^1.3.0",
     "flat-tree": "^1.9.0",
     "hypercore-crypto": "^3.2.1",
-    "hypercore-encryption": "^1.0.0",
     "hypercore-errors": "^1.2.0",
     "hypercore-id-encoding": "^1.2.0",
     "hypercore-storage": "^1.0.0",