hypercore 11.0.47 → 11.1.0

package/README.md

@@ -425,10 +425,6 @@ In contrast to `core.key` this key does not allow you to verify the data but can
 
 Populated after `ready` has been emitted. Will be `null` before the event.
 
-#### `core.encryptionKey`
-
-Buffer containing the optional block encryption key of this core. Will be `null` unless block encryption is enabled.
-
 #### `core.length`
 
 How many blocks of data are available on this core.

package/index.js

@@ -3,15 +3,16 @@ const isOptions = require('is-options')
 const crypto = require('hypercore-crypto')
 const CoreStorage = require('hypercore-storage')
 const c = require('compact-encoding')
+const sodium = require('sodium-universal')
 const b4a = require('b4a')
 const NoiseSecretStream = require('@hyperswarm/secret-stream')
+const HypercoreEncryption = require('hypercore-encryption')
 const Protomux = require('protomux')
 const id = require('hypercore-id-encoding')
 const safetyCatch = require('safety-catch')
 const unslab = require('unslab')
 
 const Core = require('./lib/core')
-const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
 const caps = require('./lib/caps')
@@ -149,7 +150,7 @@ class Hypercore extends EventEmitter {
   }
 
   static blockEncryptionKey (key, encryptionKey) {
-    return BlockEncryption.blockEncryptionKey(key, encryptionKey)
+    return HypercoreEncryption.blockEncryptionKey(key, encryptionKey)
   }
 
   static getProtocolMuxer (stream) {
@@ -233,10 +234,24 @@ class Hypercore extends EventEmitter {
     return s
   }
 
-  async setEncryptionKey (encryptionKey, opts) {
+  setEncryptionKey (encryptionKey, opts) {
+    const encryption = this._getLegacyEncryption(encryptionKey, !!(opts && opts.block))
+    return this.setEncryption(encryption, opts)
+  }
+
+  async setEncryption (encryption, opts) {
     if (!this.opened) await this.opening
-    if (this.core.unencrypted) return
-    this.encryption = encryptionKey ? new BlockEncryption(encryptionKey, this.key, { compat: this.core.compat, ...opts }) : null
+
+    if (encryption === null) {
+      this.encryption = encryption
+      return
+    }
+
+    if (!HypercoreEncryption.isHypercoreEncryption(encryption)) {
+      throw new Error('Expected hypercore encryption provider')
+    }
+
+    this.encryption = encryption
     if (!this.core.encryption) this.core.encryption = this.encryption
   }
 
@@ -290,7 +305,7 @@ class Hypercore extends EventEmitter {
       if (this.state !== null) this.state.removeSession(this)
 
       this.closed = true
-      this.emit('close', this.core.hasSession() === false)
+      this.emit('close')
       throw err
     }
 
@@ -313,9 +328,14 @@ class Hypercore extends EventEmitter {
 
     if (this.keyPair === null) this.keyPair = opts.keyPair || this.core.header.keyPair
 
-    if (!this.core.encryption && !this.core.unencrypted) {
+    if (!this.core.encryption) {
       const e = getEncryptionOption(opts)
-      if (e) this.core.encryption = new BlockEncryption(e.key, this.key, { compat: this.core.compat, ...e })
+
+      if (HypercoreEncryption.isHypercoreEncryption(e)) {
+        this.core.encryption = e
+      } else if (e) {
+        this.core.encryption = this._getLegacyEncryption(e.key, e.block)
+      }
     }
 
     const parent = opts.parent || null
@@ -476,14 +496,14 @@ class Hypercore extends EventEmitter {
     if (this.core.hasSession()) {
       // emit "fake" close as this is a session
       this.closed = true
-      this.emit('close', false)
+      this.emit('close')
       return
     }
 
     if (this.core.autoClose) await this.core.close()
 
     this.closed = true
-    this.emit('close', true)
+    this.emit('close')
   }
 
   async commit (session, opts) {
@@ -576,10 +596,6 @@ class Hypercore extends EventEmitter {
     return this.opened === false ? [] : this.core.replicator.peers
   }
 
-  get encryptionKey () {
-    return this.encryption && this.encryption.key
-  }
-
   get padding () {
     return this.encryption === null ? 0 : this.encryption.padding
   }
@@ -757,8 +773,8 @@ class Hypercore extends EventEmitter {
       block = b4a.from(block)
 
       if (this.encryption.compat !== this.core.compat) this._updateEncryption()
-      if (this.core.unencrypted) this.encryption = null
-      else this.encryption.decrypt(index, block)
+
+      await this.encryption.decrypt(index, block)
     }
 
     return this._decode(encoding, block)
@@ -906,7 +922,8 @@ class Hypercore extends EventEmitter {
 
     blocks = Array.isArray(blocks) ? blocks : [blocks]
 
-    const preappend = this.core.unencrypted ? null : (this.encryption && this._preappend)
+    const preappend = this.encryption && this._preappend
+    if (preappend) await this.encryption.ready()
 
     const buffers = this.encodeBatch !== null ? this.encodeBatch(blocks) : new Array(blocks.length)
 
@@ -1037,9 +1054,22 @@ class Hypercore extends EventEmitter {
 
   _updateEncryption () {
     const e = this.encryption
-    this.encryption = new BlockEncryption(e.key, this.key, { compat: this.core.compat, block: b4a.equals(e.blockKey, e.key) })
+    if (HypercoreEncryption.isHypercoreEncryption(e)) return
+
+    this.encryption = this._getLegacyEncryption(e.key, e.block)
+
     if (e === this.core.encryption) this.core.encryption = this.encryption
   }
+
+  _getLegacyEncryption (encryptionKey, block) {
+    if (!encryptionKey) return null
+
+    const blockKey = block
+      ? encryptionKey
+      : getLegacyBlockKey(this.key, encryptionKey, this.core.compat)
+
+    return HypercoreEncryption.createLegacyProvider(blockKey)
+  }
 }
 
 module.exports = Hypercore
@@ -1052,14 +1082,14 @@ function toHex (buf) {
   return buf && b4a.toString(buf, 'hex')
 }
 
-function preappend (blocks) {
+async function preappend (blocks) {
   const offset = this.state.length
   const fork = this.state.encryptionFork
 
   if (this.encryption.compat !== this.core.compat) this._updateEncryption()
 
   for (let i = 0; i < blocks.length; i++) {
-    this.encryption.encrypt(offset + i, blocks[i], fork)
+    await this.encryption.encrypt(offset + i, blocks[i], fork)
   }
 }
 
@@ -1126,3 +1156,12 @@ function getEncryptionOption (opts) {
   if (!opts.encryption) return null
   return b4a.isBuffer(opts.encryption) ? { key: opts.encryption } : opts.encryption
 }
+
+function getLegacyBlockKey (hypercoreKey, encryptionKey, compat) {
+  const key = b4a.alloc(HypercoreEncryption.KEYBYTES)
+
+  if (compat) sodium.crypto_generichash_batch(key, [encryptionKey], hypercoreKey)
+  else sodium.crypto_generichash_batch(key, [caps.LEGACY_BLOCK_ENCRYPTION, hypercoreKey, encryptionKey])
+
+  return key
+}

package/lib/caps.js

@@ -12,12 +12,12 @@ const [
   REPLICATE_RESPONDER,
   MANIFEST,
   DEFAULT_NAMESPACE,
-  BLOCK_ENCRYPTION
+  LEGACY_BLOCK_ENCRYPTION
 ] = crypto.namespace('hypercore', 6)
 
 exports.MANIFEST = MANIFEST
 exports.DEFAULT_NAMESPACE = DEFAULT_NAMESPACE
-exports.BLOCK_ENCRYPTION = BLOCK_ENCRYPTION
+exports.LEGACY_BLOCK_ENCRYPTION = LEGACY_BLOCK_ENCRYPTION
 
 exports.replicate = function (isInitiator, key, handshakeHash) {
   const out = b4a.allocUnsafe(32)

package/lib/core.js

@@ -39,7 +39,6 @@ module.exports = class Core {
     this.verifier = null
     this.truncating = 0
     this.updating = false
-    this.unencrypted = false
     this.skipBitfield = null
     this.globalCache = opts.globalCache || null
     this.autoClose = opts.autoClose !== false
@@ -271,7 +270,6 @@ module.exports = class Core {
     // to unslab
     if (header.manifest) {
       header.manifest = Verifier.createManifest(header.manifest)
-      this.unencrypted = header.manifest.unencrypted
     }
 
     const verifier = header.manifest ? new Verifier(header.key, header.manifest, { crypto, legacy }) : null
@@ -326,7 +324,6 @@ module.exports = class Core {
     if (verifier.prologue) this.state.prologue = Object.assign({}, verifier.prologue)
 
     this.manifest = this.header.manifest = manifest
-    this.unencrypted = this.manifest.unencrypted
 
     tx.setAuth({
       key: this.header.key,

package/lib/messages.js

@@ -6,6 +6,10 @@ const unslab = require('unslab')
 
 const EMPTY = b4a.alloc(0)
 
+const MANIFEST_PATCH = 0b00000001
+const MANIFEST_PROLOGUE = 0b00000010
+const MANIFEST_LINKED = 0b00000100
+
 const hashes = {
   preencode (state, m) {
     state.end++ // small uint
@@ -137,7 +141,7 @@ const manifestv0 = {
           hash: c.fixed32.decode(state),
           length: 0
         },
-        unencrypted: false
+        linked: []
       }
     }
 
@@ -149,7 +153,7 @@ const manifestv0 = {
         quorum: 1,
         signers: [signer.decode(state)],
         prologue: null,
-        unencrypted: false
+        linked: []
       }
     }
 
@@ -162,11 +166,13 @@ const manifestv0 = {
       quorum: c.uint.decode(state),
       signers: signerArray.decode(state),
       prologue: null,
-      unencrypted: false
+      linked: []
     }
   }
 }
 
+const fixed32Array = c.array(c.fixed32)
+
 const manifest = exports.manifest = {
   preencode (state, m) {
     state.end++ // version
@@ -178,37 +184,53 @@ const manifest = exports.manifest = {
     c.uint.preencode(state, m.quorum)
     signerArray.preencode(state, m.signers)
     if (m.prologue) prologue.preencode(state, m.prologue)
+
+    if (m.linked) {
+      fixed32Array.preencode(state, m.linked)
+    }
   },
   encode (state, m) {
     c.uint.encode(state, m.version)
     if (m.version === 0) return manifestv0.encode(state, m)
 
-    c.uint.encode(state, (m.allowPatch ? 1 : 0) | (m.prologue ? 2 : 0) | (m.unencrypted ? 4 : 0))
+    let flags = 0
+    if (m.allowPatch) flags |= MANIFEST_PATCH
+    if (m.prologue) flags |= MANIFEST_PROLOGUE
+    if (m.linked) flags |= MANIFEST_LINKED
+
+    c.uint.encode(state, flags)
     hashes.encode(state, m.hash)
 
     c.uint.encode(state, m.quorum)
     signerArray.encode(state, m.signers)
     if (m.prologue) prologue.encode(state, m.prologue)
+
+    if (m.linked) {
+      fixed32Array.encode(state, m.linked)
+    }
   },
   decode (state) {
-    const v = c.uint.decode(state)
-    if (v === 0) return manifestv0.decode(state)
-    if (v !== 1) throw new Error('Unknown version: ' + v)
+    const version = c.uint.decode(state)
+    if (version === 0) return manifestv0.decode(state)
+    if (version > 2) throw new Error('Unknown version: ' + version)
 
     const flags = c.uint.decode(state)
     const hash = hashes.decode(state)
     const quorum = c.uint.decode(state)
     const signers = signerArray.decode(state)
-    const unencrypted = (flags & 4) !== 0
+
+    const hasPatch = (flags & MANIFEST_PATCH) !== 0
+    const hasPrologue = (flags & MANIFEST_PROLOGUE) !== 0
+    const hasLinked = (flags & MANIFEST_LINKED) !== 0
 
     return {
-      version: 1,
+      version,
       hash,
-      allowPatch: (flags & 1) !== 0,
+      allowPatch: hasPatch,
       quorum,
       signers,
-      prologue: (flags & 2) === 0 ? null : prologue.decode(state),
-      unencrypted
+      prologue: hasPrologue ? prologue.decode(state) : null,
+      linked: hasLinked ? fixed32Array.decode(state) : null
     }
   }
 }

package/lib/replicator.js

@@ -1520,6 +1520,7 @@ module.exports = class Replicator {
     }
 
     this._downloadingTimer = setTimeout(setDownloadingLater, this._notDownloadingLinger, this, downloading)
+    if (this._downloadingTimer.unref) this._downloadingTimer.unref()
   }
 
   setDownloadingNow (downloading) {

package/lib/session-state.js

@@ -36,6 +36,7 @@ module.exports = class SessionState {
     this.signature = treeInfo.signature || null
 
     this.snapshotCompatLength = this.isSnapshot() ? Math.min(this.length, this.core.state.length) : -1
+    this.lastTruncation = null
 
     this.active = 0
 
@@ -230,10 +231,13 @@ module.exports = class SessionState {
     this._activeTx = null
 
     try {
-      return await tx.flush()
+      if (!(await tx.flush())) return false
     } finally {
       this._clearActiveBatch()
     }
+
+    this.lastTruncation = null
+    return true
   }
 
   _precommit () {
@@ -246,6 +250,7 @@ module.exports = class SessionState {
     try {
       const bitfield = this._pendingBitfield
       this._pendingBitfield = null
+      this.lastTruncation = null
       await this.parent._oncommit(this, bitfield)
     } finally {
       this.commiting = false
@@ -576,6 +581,8 @@ module.exports = class SessionState {
   ontruncate (tree, to, from, flushed) {
     const bitfield = { start: to, length: from - to, drop: true }
 
+    this.lastTruncation = { from, to }
+
     if (!flushed) this._updateBitfield(bitfield)
     else if (this.isDefault()) this.core.ontruncate(tree, bitfield)
 

package/lib/verifier.js

@@ -129,7 +129,7 @@ module.exports = class Verifier {
   }
 
   verify (batch, signature) {
-    if (this.version !== 1) {
+    if (this.version === 0) {
       return this._verifyCompat(batch, signature)
     }
 
@@ -182,7 +182,8 @@ module.exports = class Verifier {
         namespace: caps.DEFAULT_NAMESPACE,
         publicKey
       }],
-      prologue: null
+      prologue: null,
+      linked: null
     }
   }
 
@@ -195,13 +196,13 @@ module.exports = class Verifier {
     if (!inp) return null
 
     const manifest = {
-      version: typeof inp.version === 'number' ? inp.version : 1,
+      version: getManifestVersion(inp), // only v2 if linked are present
       hash: 'blake2b',
       allowPatch: !!inp.allowPatch,
       quorum: defaultQuorum(inp),
       signers: inp.signers ? inp.signers.map(parseSigner) : [],
       prologue: null,
-      unencrypted: !!inp.unencrypted
+      linked: null
     }
 
     if (inp.hash && inp.hash !== 'blake2b') throw BAD_ARGUMENT('Only Blake2b hashes are supported')
@@ -215,6 +216,18 @@ module.exports = class Verifier {
       manifest.prologue.hash = unslab(manifest.prologue.hash)
     }
 
+    if (inp.linked && inp.linked.length) {
+      if (manifest.version < 2) throw BAD_ARGUMENT('Invalid field')
+
+      for (const key of inp.linked) {
+        if (!(b4a.isBuffer(key) && key.byteLength === 32)) {
+          throw BAD_ARGUMENT('Invalid key')
+        }
+      }
+
+      manifest.linked = inp.linked
+    }
+
     return manifest
   }
 
@@ -311,3 +324,9 @@ function proofToVersion1 (proof) {
     patch: proof.patch ? proof.patch.length : 0
   }
 }
+
+function getManifestVersion (inp) {
+  if (typeof inp.version === 'number') return inp.version
+  if (inp.linked && inp.linked.length) return 2
+  return 1
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.0.47",
+  "version": "11.1.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -49,6 +49,7 @@
     "fast-fifo": "^1.3.0",
     "flat-tree": "^1.9.0",
     "hypercore-crypto": "^3.2.1",
+    "hypercore-encryption": "^1.0.0",
     "hypercore-errors": "^1.2.0",
     "hypercore-id-encoding": "^1.2.0",
     "hypercore-storage": "^1.0.0",

package/lib/block-encryption.js

@@ -1,80 +0,0 @@
-const sodium = require('sodium-universal')
-const c = require('compact-encoding')
-const b4a = require('b4a')
-const { BLOCK_ENCRYPTION } = require('./caps')
-
-const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
-
-module.exports = class BlockEncryption {
-  constructor (encryptionKey, hypercoreKey, { block = false, compat = true } = {}) {
-    const subKeys = b4a.alloc(2 * sodium.crypto_stream_KEYBYTES)
-
-    this.key = encryptionKey
-    this.blockKey = block ? encryptionKey : subKeys.subarray(0, sodium.crypto_stream_KEYBYTES)
-    this.blindingKey = subKeys.subarray(sodium.crypto_stream_KEYBYTES)
-    this.padding = 8
-    this.compat = compat
-
-    if (!block) {
-      if (compat) sodium.crypto_generichash_batch(this.blockKey, [encryptionKey], hypercoreKey)
-      else sodium.crypto_generichash_batch(this.blockKey, [BLOCK_ENCRYPTION, hypercoreKey, encryptionKey])
-    }
-
-    sodium.crypto_generichash(this.blindingKey, this.blockKey)
-  }
-
-  static blockEncryptionKey (hypercoreKey, encryptionKey) {
-    const blockKey = b4a.alloc(sodium.crypto_stream_KEYBYTES)
-    sodium.crypto_generichash_batch(blockKey, [BLOCK_ENCRYPTION, hypercoreKey, encryptionKey])
-    return blockKey
-  }
-
-  encrypt (index, block, fork) {
-    const padding = block.subarray(0, this.padding)
-    block = block.subarray(this.padding)
-
-    c.uint64.encode({ start: 0, end: 8, buffer: padding }, fork)
-    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
-
-    // Zero out any previous padding.
-    nonce.fill(0, 8, 8 + padding.byteLength)
-
-    // Blind the fork ID, possibly risking reusing the nonce on a reorg of the
-    // Hypercore. This is fine as the blinding is best-effort and the latest
-    // fork ID shared on replication anyway.
-    sodium.crypto_stream_xor(
-      padding,
-      padding,
-      nonce,
-      this.blindingKey
-    )
-
-    nonce.set(padding, 8)
-
-    // The combination of a (blinded) fork ID and a block index is unique for a
-    // given Hypercore and is therefore a valid nonce for encrypting the block.
-    sodium.crypto_stream_xor(
-      block,
-      block,
-      nonce,
-      this.blockKey
-    )
-  }
-
-  decrypt (index, block) {
-    const padding = block.subarray(0, this.padding)
-    block = block.subarray(this.padding)
-
-    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
-
-    nonce.set(padding, 8)
-
-    // Decrypt the block using the blinded fork ID.
-    sodium.crypto_stream_xor(
-      block,
-      block,
-      nonce,
-      this.blockKey
-    )
-  }
-}