hypercore 10.0.0-alpha.5 → 10.0.0-alpha.50

package/index.js

@@ -3,15 +3,19 @@ const raf = require('random-access-file')
 const isOptions = require('is-options')
 const hypercoreCrypto = require('hypercore-crypto')
 const c = require('compact-encoding')
+const b4a = require('b4a')
 const Xache = require('xache')
 const NoiseSecretStream = require('@hyperswarm/secret-stream')
+const Protomux = require('protomux')
 const codecs = require('codecs')
 
 const fsctl = requireMaybe('fsctl') || { lock: noop, sparse: noop }
 
 const Replicator = require('./lib/replicator')
-const Extensions = require('./lib/extensions')
 const Core = require('./lib/core')
+const BlockEncryption = require('./lib/block-encryption')
+const { ReadStream, WriteStream } = require('./lib/streams')
+const { BAD_ARGUMENT, SESSION_CLOSED, SESSION_NOT_WRITABLE, SNAPSHOT_NOT_AVAILABLE } = require('./lib/errors')
 
 const promises = Symbol.for('hypercore.promises')
 const inspect = Symbol.for('nodejs.util.inspect.custom')
@@ -28,14 +32,17 @@ module.exports = class Hypercore extends EventEmitter {
       opts = key
       key = null
     }
+
     if (key && typeof key === 'string') {
-      key = Buffer.from(key, 'hex')
-    }
-    if (key && key.byteLength !== 32) {
-      throw new Error('Hypercore key should be 32 bytes')
+      key = b4a.from(key, 'hex')
     }
 
     if (!opts) opts = {}
+
+    if (!opts.crypto && key && key.byteLength !== 32) {
+      throw BAD_ARGUMENT('Hypercore key should be 32 bytes')
+    }
+
     if (!storage) storage = opts.storage
 
     this[promises] = true
@@ -44,23 +51,34 @@ module.exports = class Hypercore extends EventEmitter {
     this.crypto = opts.crypto || hypercoreCrypto
     this.core = null
     this.replicator = null
-    this.extensions = opts.extensions || new Extensions()
+    this.encryption = null
+    this.extensions = new Map()
     this.cache = opts.cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (opts.cache || null)
 
     this.valueEncoding = null
+    this.encodeBatch = null
+    this.activeRequests = []
+
     this.key = key || null
-    this.discoveryKey = null
+    this.keyPair = null
     this.readable = true
     this.writable = false
     this.opened = false
     this.closed = false
+    this.snapshotted = !!opts.snapshot
+    this.sparse = opts.sparse !== false
     this.sessions = opts._sessions || [this]
-    this.sign = opts.sign || null
+    this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
+    this.onwait = opts.onwait || null
 
     this.closing = null
-    this.opening = opts._opening || this._open(key, storage, opts)
+    this.opening = this._openSession(key, storage, opts)
     this.opening.catch(noop)
+
+    this._preappend = preappend.bind(this)
+    this._snapshot = null
+    this._findingPeers = 0
   }
 
   [inspect] (depth, opts) {
@@ -69,21 +87,78 @@ module.exports = class Hypercore extends EventEmitter {
       while (indent.length < opts.indentationLvl) indent += ' '
     }
 
+    let peers = ''
+    const min = Math.min(this.peers.length, 5)
+
+    for (let i = 0; i < min; i++) {
+      const peer = this.peers[i]
+
+      peers += indent + '    Peer(\n'
+      peers += indent + '      remotePublicKey: ' + opts.stylize(toHex(peer.remotePublicKey), 'string') + '\n'
+      peers += indent + '      remoteLength: ' + opts.stylize(peer.remoteLength, 'number') + '\n'
+      peers += indent + '      remoteFork: ' + opts.stylize(peer.remoteFork, 'number') + '\n'
+      peers += indent + '      remoteCanUpgrade: ' + opts.stylize(peer.remoteCanUpgrade, 'boolean') + '\n'
+      peers += indent + '    )' + '\n'
+    }
+
+    if (this.peers.length > 5) {
+      peers += indent + '  ... and ' + (this.peers.length - 5) + ' more\n'
+    }
+
+    if (peers) peers = '[\n' + peers + indent + '  ]'
+    else peers = '[ ' + opts.stylize(0, 'number') + ' ]'
+
     return this.constructor.name + '(\n' +
-      indent + '  key: ' + opts.stylize((toHex(this.key)), 'string') + '\n' +
+      indent + '  key: ' + opts.stylize(toHex(this.key), 'string') + '\n' +
       indent + '  discoveryKey: ' + opts.stylize(toHex(this.discoveryKey), 'string') + '\n' +
       indent + '  opened: ' + opts.stylize(this.opened, 'boolean') + '\n' +
+      indent + '  closed: ' + opts.stylize(this.closed, 'boolean') + '\n' +
+      indent + '  snapshotted: ' + opts.stylize(this.snapshotted, 'boolean') + '\n' +
+      indent + '  sparse: ' + opts.stylize(this.sparse, 'boolean') + '\n' +
       indent + '  writable: ' + opts.stylize(this.writable, 'boolean') + '\n' +
-      indent + '  sessions: ' + opts.stylize(this.sessions.length, 'number') + '\n' +
-      indent + '  peers: [ ' + opts.stylize(this.peers.length, 'number') + ' ]\n' +
       indent + '  length: ' + opts.stylize(this.length, 'number') + '\n' +
       indent + '  byteLength: ' + opts.stylize(this.byteLength, 'number') + '\n' +
+      indent + '  fork: ' + opts.stylize(this.fork, 'number') + '\n' +
+      indent + '  sessions: [ ' + opts.stylize(this.sessions.length, 'number') + ' ]\n' +
+      indent + '  activeRequests: [ ' + opts.stylize(this.activeRequests.length, 'number') + ' ]\n' +
+      indent + '  peers: ' + peers + '\n' +
       indent + ')'
   }
 
-  static createProtocolStream (isInitiator, opts) {
-    const noiseStream = new NoiseSecretStream(isInitiator, null, opts)
-    return noiseStream.rawStream
+  static getProtocolMuxer (stream) {
+    return stream.noiseStream.userData
+  }
+
+  static createProtocolStream (isInitiator, opts = {}) {
+    let outerStream = Protomux.isProtomux(isInitiator)
+      ? isInitiator.stream
+      : isStream(isInitiator)
+        ? isInitiator
+        : opts.stream
+
+    let noiseStream = null
+
+    if (outerStream) {
+      noiseStream = outerStream.noiseStream
+    } else {
+      noiseStream = new NoiseSecretStream(isInitiator, null, opts)
+      outerStream = noiseStream.rawStream
+    }
+    if (!noiseStream) throw BAD_ARGUMENT('Invalid stream')
+
+    if (!noiseStream.userData) {
+      const protocol = new Protomux(noiseStream)
+
+      if (opts.ondiscoverykey) {
+        protocol.pair({ protocol: 'hypercore/alpha' }, opts.ondiscoverykey)
+      }
+      if (opts.keepAlive !== false) {
+        noiseStream.setKeepAlive(5000)
+      }
+      noiseStream.userData = protocol
+    }
+
+    return outerStream
   }
 
   static defaultStorage (storage, opts = {}) {
@@ -98,45 +173,187 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
+  snapshot (opts) {
+    return this.session({ ...opts, snapshot: true })
+  }
+
   session (opts = {}) {
     if (this.closing) {
       // This makes the closing logic alot easier. If this turns out to be a problem
       // in practive, open an issue and we'll try to make a solution for it.
-      throw new Error('Cannot make sessions on a closing core')
+      throw SESSION_CLOSED('Cannot make sessions on a closing core')
     }
 
+    const sparse = opts.sparse === false ? false : this.sparse
+    const onwait = opts.onwait === undefined ? this.onwait : opts.onwait
     const Clz = opts.class || Hypercore
-    const keyPair = opts.keyPair && opts.keyPair.secretKey && { ...opts.keyPair }
-
-    // This only works if the hypercore was fully loaded,
-    // but we only do this to validate the keypair to help catch bugs so yolo
-    if (this.key && keyPair) keyPair.publicKey = this.key
-
     const s = new Clz(this.storage, this.key, {
       ...opts,
-      sign: opts.sign || (keyPair && keyPair.secretKey && Core.createSigner(this.crypto, keyPair)) || this.sign,
-      valueEncoding: this.valueEncoding,
-      extensions: this.extensions,
+      sparse,
+      onwait,
       _opening: this.opening,
       _sessions: this.sessions
     })
 
-    s._initSession(this)
+    s._passCapabilities(this)
+
+    // Configure the cache unless explicitly disabled.
+    if (opts.cache !== false) {
+      s.cache = opts.cache === true || !opts.cache ? this.cache : opts.cache
+    }
+
+    ensureEncryption(s, opts)
+
     this.sessions.push(s)
 
     return s
   }
 
-  _initSession (o) {
-    if (!this.sign) this.sign = o.sign
+  _passCapabilities (o) {
+    if (!this.auth) this.auth = o.auth
+
     this.crypto = o.crypto
-    this.opened = o.opened
     this.key = o.key
-    this.discoveryKey = o.discoveryKey
     this.core = o.core
     this.replicator = o.replicator
-    this.writable = !!this.sign
+    this.encryption = o.encryption
+    this.writable = !!(this.auth && this.auth.sign)
     this.autoClose = o.autoClose
+
+    if (this.snapshotted && this.core && !this._snapshot) this._updateSnapshot()
+  }
+
+  async _openFromExisting (from, opts) {
+    await from.opening
+
+    this._passCapabilities(from)
+    this.sessions = from.sessions
+    this.storage = from.storage
+    this.replicator.findingPeers += this._findingPeers
+
+    ensureEncryption(this, opts)
+
+    this.sessions.push(this)
+  }
+
+  async _openSession (key, storage, opts) {
+    const isFirst = !opts._opening
+
+    if (!isFirst) await opts._opening
+    if (opts.preload) opts = { ...opts, ...(await opts.preload()) }
+
+    const keyPair = (key && opts.keyPair)
+      ? { ...opts.keyPair, publicKey: key }
+      : key
+        ? { publicKey: key, secretKey: null }
+        : opts.keyPair
+
+    // This only works if the hypercore was fully loaded,
+    // but we only do this to validate the keypair to help catch bugs so yolo
+    if (this.key && keyPair) keyPair.publicKey = this.key
+
+    if (opts.auth) {
+      this.auth = opts.auth
+    } else if (opts.sign) {
+      this.auth = Core.createAuth(this.crypto, keyPair, opts)
+    } else if (keyPair && keyPair.secretKey) {
+      this.auth = Core.createAuth(this.crypto, keyPair)
+    }
+
+    if (isFirst) {
+      await this._openCapabilities(keyPair, storage, opts)
+      // Only the root session should pass capabilities to other sessions.
+      for (let i = 0; i < this.sessions.length; i++) {
+        const s = this.sessions[i]
+        if (s !== this) s._passCapabilities(this)
+      }
+    }
+
+    if (!this.auth) this.auth = this.core.defaultAuth
+    this.writable = !!this.auth.sign
+
+    if (opts.valueEncoding) {
+      this.valueEncoding = c.from(codecs(opts.valueEncoding))
+    }
+    if (opts.encodeBatch) {
+      this.encodeBatch = opts.encodeBatch
+    }
+
+    // Start continous replication if not in sparse mode.
+    if (!this.sparse) this.download({ start: 0, end: -1 })
+
+    // This is a hidden option that's only used by Corestore.
+    // It's required so that corestore can load a name from userData before 'ready' is emitted.
+    if (opts._preready) await opts._preready(this)
+
+    this.opened = true
+    this.emit('ready')
+  }
+
+  async _openCapabilities (keyPair, storage, opts) {
+    if (opts.from) return this._openFromExisting(opts.from, opts)
+
+    this.storage = Hypercore.defaultStorage(opts.storage || storage)
+
+    this.core = await Core.open(this.storage, {
+      force: opts.force,
+      createIfMissing: opts.createIfMissing,
+      overwrite: opts.overwrite,
+      keyPair,
+      crypto: this.crypto,
+      legacy: opts.legacy,
+      auth: opts.auth,
+      onupdate: this._oncoreupdate.bind(this),
+      oncontigupdate: this._oncorecontigupdate.bind(this)
+    })
+
+    if (opts.userData) {
+      for (const [key, value] of Object.entries(opts.userData)) {
+        await this.core.userData(key, value)
+      }
+    }
+
+    this.key = this.core.header.signer.publicKey
+    this.keyPair = this.core.header.signer
+
+    this.replicator = new Replicator(this.core, this.key, {
+      eagerUpdate: true,
+      allowFork: opts.allowFork !== false,
+      onpeerupdate: this._onpeerupdate.bind(this),
+      onupload: this._onupload.bind(this)
+    })
+
+    this.replicator.findingPeers += this._findingPeers
+
+    if (!this.encryption && opts.encryptionKey) {
+      this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
+    }
+  }
+
+  _getSnapshot () {
+    if (this.sparse) {
+      return {
+        length: this.core.tree.length,
+        byteLength: this.core.tree.byteLength,
+        fork: this.core.tree.fork,
+        compatLength: this.core.tree.length
+      }
+    }
+
+    return {
+      length: this.core.header.contiguousLength,
+      byteLength: 0,
+      fork: this.core.tree.fork,
+      compatLength: this.core.header.contiguousLength
+    }
+  }
+
+  _updateSnapshot () {
+    const prev = this._snapshot
+    const next = this._snapshot = this._getSnapshot()
+
+    if (!prev) return true
+    return prev.length !== next.length || prev.fork !== next.fork
   }
 
   close () {
@@ -155,6 +372,20 @@ module.exports = class Hypercore extends EventEmitter {
     this.readable = false
     this.writable = false
     this.closed = true
+    this.opened = false
+
+    const gc = []
+    for (const ext of this.extensions.values()) {
+      if (ext.session === this) gc.push(ext)
+    }
+    for (const ext of gc) ext.destroy()
+
+    if (this.replicator !== null) {
+      this.replicator.findingPeers -= this._findingPeers
+      this.replicator.clearRequests(this.activeRequests)
+    }
+
+    this._findingPeers = 0
 
     if (this.sessions.length) {
       // if this is the last session and we are auto closing, trigger that first to enforce error handling
@@ -170,41 +401,60 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   replicate (isInitiator, opts = {}) {
-    let outerStream = isStream(isInitiator)
-      ? isInitiator
-      : opts.stream
-    let noiseStream = null
+    // Only limitation here is that ondiscoverykey doesn't work atm when passing a muxer directly,
+    // because it doesn't really make a lot of sense.
+    if (Protomux.isProtomux(isInitiator)) return this._attachToMuxer(isInitiator, opts)
 
-    if (outerStream) {
-      noiseStream = outerStream.noiseStream
-    } else {
-      outerStream = Hypercore.createProtocolStream(isInitiator, opts)
-      noiseStream = outerStream.noiseStream
-    }
-    if (!noiseStream) throw new Error('Invalid stream passed to replicate')
+    const protocolStream = Hypercore.createProtocolStream(isInitiator, opts)
+    const noiseStream = protocolStream.noiseStream
+    const protocol = noiseStream.userData
 
-    if (!noiseStream.userData) {
-      const protocol = Replicator.createProtocol(noiseStream)
-      noiseStream.userData = protocol
-      noiseStream.on('error', noop) // All noise errors already propagate through outerStream
+    this._attachToMuxer(protocol, opts)
+
+    return protocolStream
+  }
+
+  _attachToMuxer (mux, opts) {
+    // If the user wants to, we can make this replication run in a session
+    // that way the core wont close "under them" during replication
+    if (opts.session) {
+      const s = this.session()
+      mux.stream.on('close', () => s.close().catch(noop))
     }
 
-    const protocol = noiseStream.userData
     if (this.opened) {
-      this.replicator.joinProtocol(protocol, this.key, this.discoveryKey)
+      this.replicator.attachTo(mux)
     } else {
-      this.opening.then(() => this.replicator.joinProtocol(protocol, this.key, this.discoveryKey), protocol.destroy.bind(protocol))
+      this.opening.then(() => this.replicator.attachTo(mux), mux.destroy.bind(mux))
     }
 
-    return outerStream
+    return mux
+  }
+
+  get discoveryKey () {
+    return this.replicator === null ? null : this.replicator.discoveryKey
   }
 
   get length () {
-    return this.core === null ? 0 : this.core.tree.length
+    if (this._snapshot) return this._snapshot.length
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousLength
+    return this.core.tree.length
   }
 
   get byteLength () {
-    return this.core === null ? 0 : this.core.tree.byteLength
+    if (this._snapshot) return this._snapshot.byteLength
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousByteLength
+    return this.core.tree.byteLength - (this.core.tree.length * this.padding)
+  }
+
+  get contiguousLength () {
+    return this.core === null ? 0 : this.core.header.contiguousLength
+  }
+
+  get contiguousByteLength () {
+    return 0
   }
 
   get fork () {
@@ -215,105 +465,88 @@ module.exports = class Hypercore extends EventEmitter {
     return this.replicator === null ? [] : this.replicator.peers
   }
 
-  ready () {
-    return this.opening
+  get encryptionKey () {
+    return this.encryption && this.encryption.key
   }
 
-  async _open (key, storage, opts) {
-    if (opts.preload) opts = { ...opts, ...(await opts.preload()) }
-
-    this.valueEncoding = opts.valueEncoding ? c.from(codecs(opts.valueEncoding)) : null
-
-    const keyPair = (key && opts.keyPair)
-      ? { ...opts.keyPair, publicKey: key }
-      : key
-        ? { publicKey: key, secretKey: null }
-        : opts.keyPair
-
-    if (opts.from) {
-      const from = opts.from
-      await from.opening
-      for (const [name, ext] of this.extensions) from.extensions.register(name, null, ext)
-      this._initSession(from)
-      this.extensions = from.extensions
-      this.sessions = from.sessions
-      this.storage = from.storage
-      if (!this.sign) this.sign = opts.sign || ((keyPair && keyPair.secretKey) ? Core.createSigner(this.crypto, keyPair) : null)
-      this.writable = !!this.sign
-      this.sessions.push(this)
-      return
-    }
-
-    if (!this.storage) this.storage = Hypercore.defaultStorage(opts.storage || storage)
-
-    this.core = await Core.open(this.storage, {
-      keyPair,
-      crypto: this.crypto,
-      onupdate: this._oncoreupdate.bind(this)
-    })
-
-    if (opts.userData) {
-      for (const [key, value] of Object.entries(opts.userData)) {
-        await this.core.userData(key, value)
-      }
-    }
-
-    this.replicator = new Replicator(this.core, {
-      onupdate: this._onpeerupdate.bind(this)
-    })
-
-    if (!this.sign) this.sign = opts.sign || this.core.defaultSign
-
-    this.discoveryKey = this.crypto.discoveryKey(this.core.header.signer.publicKey)
-    this.key = this.core.header.signer.publicKey
-    this.writable = !!this.sign
+  get padding () {
+    return this.encryption === null ? 0 : this.encryption.padding
+  }
 
-    this.extensions.attach(this.replicator)
-    this.opened = true
+  ready () {
+    return this.opening
+  }
 
-    if (opts.postload) await opts.postload(this)
+  _onupload (index, value, from) {
+    const byteLength = value.byteLength - this.padding
 
     for (let i = 0; i < this.sessions.length; i++) {
-      const s = this.sessions[i]
-      if (s !== this) s._initSession(this)
-      s.emit('ready')
+      this.sessions[i].emit('upload', index, byteLength, from)
     }
   }
 
   _oncoreupdate (status, bitfield, value, from) {
     if (status !== 0) {
+      const truncated = (status & 0b10) !== 0
+      const appended = (status & 0b01) !== 0
+
+      if (truncated) {
+        this.replicator.ontruncate(bitfield.start)
+      }
+
       for (let i = 0; i < this.sessions.length; i++) {
-        if ((status & 0b10) !== 0) {
-          if (this.cache) this.cache.clear()
-          this.sessions[i].emit('truncate', this.core.tree.fork)
-        }
-        if ((status & 0b01) !== 0) {
-          this.sessions[i].emit('append')
+        const s = this.sessions[i]
+
+        if (truncated) {
+          if (s.cache) s.cache.clear()
+          s.emit('truncate', bitfield.start, this.core.tree.fork)
+          // If snapshotted, make sure to update our compat so we can fail gets
+          if (s._snapshot && bitfield.start < s._snapshot.compatLength) s._snapshot.compatLength = bitfield.start
         }
+
+        // For sparse sessions, immediately emit appends. Non-sparse sessions
+        // are handled separately and only emit appends when their contiguous
+        // length is updated.
+        if (appended && s.sparse) s.emit('append')
       }
 
-      this.replicator.broadcastInfo()
+      this.replicator.onupgrade()
     }
 
-    if (bitfield && !bitfield.drop) { // TODO: support drop!
-      for (let i = 0; i < bitfield.length; i++) {
-        this.replicator.broadcastBlock(bitfield.start + i)
-      }
+    if (bitfield) {
+      this.replicator.onhave(bitfield.start, bitfield.length, bitfield.drop)
     }
 
     if (value) {
+      const byteLength = value.byteLength - this.padding
+
       for (let i = 0; i < this.sessions.length; i++) {
-        this.sessions[i].emit('download', bitfield.start, value, from)
+        this.sessions[i].emit('download', bitfield.start, byteLength, from)
       }
     }
   }
 
+  _oncorecontigupdate () {
+    // For non-sparse sessions, emit appends only when the contiguous length is
+    // updated.
+    for (let i = 0; i < this.sessions.length; i++) {
+      const s = this.sessions[i]
+
+      if (!s.sparse) s.emit('append')
+    }
+  }
+
   _onpeerupdate (added, peer) {
-    if (added) this.extensions.update(peer)
     const name = added ? 'peer-add' : 'peer-remove'
 
     for (let i = 0; i < this.sessions.length; i++) {
       this.sessions[i].emit(name, peer)
+
+      if (added) {
+        for (const ext of this.sessions[i].extensions.values()) {
+          peer.extensions.set(ext.name, ext)
+        }
+      }
     }
   }
 
@@ -330,19 +563,67 @@ module.exports = class Hypercore extends EventEmitter {
     return null
   }
 
-  async update () {
+  findingPeers () {
+    this._findingPeers++
+    if (this.replicator !== null && !this.closing) this.replicator.findingPeers++
+
+    let once = true
+
+    return () => {
+      if (this.closing || !once) return
+      once = false
+      this._findingPeers--
+      if (this.replicator !== null && --this.replicator.findingPeers === 0) {
+        this.replicator.updateAll()
+      }
+    }
+  }
+
+  async update (opts) {
     if (this.opened === false) await this.opening
+    if (this.closing !== null) return false
+
     // TODO: add an option where a writer can bootstrap it's state from the network also
-    if (this.writable) return false
-    return this.replicator.requestUpgrade()
+    if (this.writable) {
+      if (!this.snapshotted) return false
+      return this._updateSnapshot()
+    }
+
+    const activeRequests = (opts && opts.activeRequests) || this.activeRequests
+    const req = this.replicator.addUpgrade(activeRequests)
+
+    let upgraded = await req.promise
+
+    if (!this.sparse) {
+      // Download all available blocks in non-sparse mode
+      const start = this.length
+      const end = this.core.tree.length
+      const contig = this.contiguousLength
+
+      await this.download({ start, end, ifAvailable: true }).downloaded()
+
+      if (!upgraded) upgraded = this.contiguousLength !== contig
+    }
+
+    if (!upgraded) return false
+    if (this.snapshotted) return this._updateSnapshot()
+    return true
   }
 
-  async seek (bytes) {
+  async seek (bytes, opts) {
     if (this.opened === false) await this.opening
 
-    const s = this.core.tree.seek(bytes)
+    const s = this.core.tree.seek(bytes, this.padding)
+
+    const offset = await s.update()
+    if (offset) return offset
+
+    if (this.closing !== null) throw SESSION_CLOSED()
 
-    return (await s.update()) || this.replicator.requestSeek(s)
+    const activeRequests = (opts && opts.activeRequests) || this.activeRequests
+    const req = this.replicator.addSeek(activeRequests, s)
+
+    return req.promise
   }
 
   async has (index) {
@@ -353,41 +634,93 @@ module.exports = class Hypercore extends EventEmitter {
 
   async get (index, opts) {
     if (this.opened === false) await this.opening
-    const c = this.cache && this.cache.get(index)
-    if (c) return c
-    const fork = this.core.tree.fork
-    const b = await this._get(index, opts)
-    if (this.cache && fork === this.core.tree.fork && b) this.cache.set(index, b)
-    return b
+    if (this.closing !== null) throw SESSION_CLOSED()
+    if (this._snapshot !== null && index >= this._snapshot.compatLength) throw SNAPSHOT_NOT_AVAILABLE()
+
+    const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
+
+    let req = this.cache && this.cache.get(index)
+    if (!req) req = this._get(index, opts)
+
+    let block = await req
+    if (!block) return null
+
+    if (this.encryption) {
+      // Copy the block as it might be shared with other sessions.
+      block = b4a.from(block)
+
+      this.encryption.decrypt(index, block)
+    }
+
+    return this._decode(encoding, block)
   }
 
   async _get (index, opts) {
-    const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
+    let req
 
-    if (this.core.bitfield.get(index)) return decode(encoding, await this.core.blocks.get(index))
-    if (opts && opts.onwait) opts.onwait(index)
+    if (this.core.bitfield.get(index)) {
+      req = this.core.blocks.get(index)
 
-    return decode(encoding, await this.replicator.requestBlock(index))
+      if (this.cache) this.cache.set(index, req)
+    } else {
+      if (opts && opts.wait === false) return null
+      if (opts && opts.onwait) opts.onwait(index, this)
+      else if (this.onwait) this.onwait(index, this)
+
+      const activeRequests = (opts && opts.activeRequests) || this.activeRequests
+
+      req = this._cacheOnResolve(
+        index,
+        this.replicator
+          .addBlock(activeRequests, index)
+          .promise,
+        this.core.tree.fork
+      )
+    }
+
+    return req
   }
 
-  download (range) {
-    const start = (range && range.start) || 0
-    const end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
-    const linear = !!(range && range.linear)
+  async _cacheOnResolve (index, req, fork) {
+    const block = await req
 
-    // TODO: support range.blocks
+    if (this.cache && fork === this.core.tree.fork) {
+      this.cache.set(index, Promise.resolve(block))
+    }
 
-    const r = Replicator.createRange(start, end, linear)
+    return block
+  }
 
-    if (this.opened) this.replicator.addRange(r)
-    else this.opening.then(() => this.replicator.addRange(r), noop)
+  createReadStream (opts) {
+    return new ReadStream(this, opts)
+  }
 
-    return r
+  createWriteStream (opts) {
+    return new WriteStream(this, opts)
   }
 
-  // TODO: get rid of this / deprecate it?
-  cancel (request) {
-    // Do nothing for now
+  download (range) {
+    const reqP = this._download(range)
+
+    // do not crash in the background...
+    reqP.catch(noop)
+
+    // TODO: turn this into an actual object...
+    return {
+      async downloaded () {
+        const req = await reqP
+        return req.promise
+      },
+      destroy () {
+        reqP.then(req => req.context && req.context.detach(req), noop)
+      }
+    }
+  }
+
+  async _download (range) {
+    if (this.opened === false) await this.opening
+    const activeRequests = (range && range.activeRequests) || this.activeRequests
+    return this.replicator.addRange(activeRequests, range)
   }
 
   // TODO: get rid of this / deprecate it?
@@ -395,12 +728,17 @@ module.exports = class Hypercore extends EventEmitter {
     range.destroy(null)
   }
 
+  // TODO: get rid of this / deprecate it?
+  cancel (request) {
+    // Do nothing for now
+  }
+
   async truncate (newLength = 0, fork = -1) {
     if (this.opened === false) await this.opening
-    if (this.writable === false) throw new Error('Core is not writable')
+    if (this.writable === false) throw SESSION_NOT_WRITABLE()
 
     if (fork === -1) fork = this.core.tree.fork + 1
-    await this.core.truncate(newLength, fork, this.sign)
+    await this.core.truncate(newLength, fork, this.auth)
 
     // TODO: Should propagate from an event triggered by the oplog
     this.replicator.updateAll()
@@ -408,42 +746,108 @@ module.exports = class Hypercore extends EventEmitter {
 
   async append (blocks) {
     if (this.opened === false) await this.opening
-    if (this.writable === false) throw new Error('Core is not writable')
+    if (this.writable === false) throw SESSION_NOT_WRITABLE()
 
-    const blks = Array.isArray(blocks) ? blocks : [blocks]
-    const buffers = new Array(blks.length)
+    blocks = Array.isArray(blocks) ? blocks : [blocks]
 
-    for (let i = 0; i < blks.length; i++) {
-      const blk = blks[i]
+    const preappend = this.encryption && this._preappend
 
-      const buf = Buffer.isBuffer(blk)
-        ? blk
-        : this.valueEncoding
-          ? c.encode(this.valueEncoding, blk)
-          : Buffer.from(blk)
+    const buffers = this.encodeBatch !== null ? this.encodeBatch(blocks) : new Array(blocks.length)
 
-      buffers[i] = buf
+    if (this.encodeBatch === null) {
+      for (let i = 0; i < blocks.length; i++) {
+        buffers[i] = this._encode(this.valueEncoding, blocks[i])
+      }
     }
 
-    return await this.core.append(buffers, this.sign)
+    return await this.core.append(buffers, this.auth, { preappend })
   }
 
-  registerExtension (name, handlers) {
-    return this.extensions.register(name, handlers)
+  async treeHash (length) {
+    if (length === undefined) {
+      await this.ready()
+      length = this.core.length
+    }
+
+    const roots = await this.core.tree.getRoots(length)
+    return this.crypto.tree(roots)
   }
 
-  // called by the extensions
-  onextensionupdate () {
-    if (this.replicator !== null) this.replicator.broadcastOptions()
+  registerExtension (name, handlers = {}) {
+    if (this.extensions.has(name)) {
+      const ext = this.extensions.get(name)
+      ext.handlers = handlers
+      ext.encoding = c.from(codecs(handlers.encoding) || c.buffer)
+      ext.session = this
+      return ext
+    }
+
+    const ext = {
+      name,
+      handlers,
+      encoding: c.from(codecs(handlers.encoding) || c.buffer),
+      session: this,
+      send (message, peer) {
+        const buffer = c.encode(this.encoding, message)
+        peer.extension(name, buffer)
+      },
+      broadcast (message) {
+        const buffer = c.encode(this.encoding, message)
+        for (const peer of this.session.peers) {
+          peer.extension(name, buffer)
+        }
+      },
+      destroy () {
+        for (const peer of this.session.peers) {
+          if (peer.extensions.get(name) === ext) peer.extensions.delete(name)
+        }
+        this.session.extensions.delete(name)
+      },
+      _onmessage (state, peer) {
+        const m = this.encoding.decode(state)
+        if (this.handlers.onmessage) this.handlers.onmessage(m, peer)
+      }
+    }
+
+    this.extensions.set(name, ext)
+    for (const peer of this.peers) {
+      peer.extensions.set(name, ext)
+    }
+
+    return ext
   }
-}
 
-function noop () {}
+  _encode (enc, val) {
+    const state = { start: this.padding, end: this.padding, buffer: null }
 
-function decode (enc, buf) {
-  return enc ? c.decode(enc, buf) : buf
+    if (b4a.isBuffer(val)) {
+      if (state.start === 0) return val
+      state.end += val.byteLength
+    } else if (enc) {
+      enc.preencode(state, val)
+    } else {
+      val = b4a.from(val)
+      if (state.start === 0) return val
+      state.end += val.byteLength
+    }
+
+    state.buffer = b4a.allocUnsafe(state.end)
+
+    if (enc) enc.encode(state, val)
+    else state.buffer.set(val, state.start)
+
+    return state.buffer
+  }
+
+  _decode (enc, block) {
+    if (this.padding) block = block.subarray(this.padding)
+    if (enc) return c.decode(enc, block)
+    return block
+  }
 }
 
+function noop () {}
+
 function isStream (s) {
   return typeof s === 'object' && s && typeof s.pipe === 'function'
 }
@@ -457,5 +861,22 @@ function requireMaybe (name) {
 }
 
 function toHex (buf) {
-  return buf && buf.toString('hex')
+  return buf && b4a.toString(buf, 'hex')
+}
+
+function preappend (blocks) {
+  const offset = this.core.tree.length
+  const fork = this.core.tree.fork
+
+  for (let i = 0; i < blocks.length; i++) {
+    this.encryption.encrypt(offset + i, blocks[i], fork)
+  }
+}
+
+function ensureEncryption (core, opts) {
+  if (!opts.encryptionKey) return
+  // Only override the block encryption if its either not already set or if
+  // the caller provided a different key.
+  if (core.encryption && b4a.equals(core.encryption.key, opts.encryptionKey)) return
+  core.encryption = new BlockEncryption(opts.encryptionKey, core.key)
 }