hypercore 10.0.0-alpha.8 → 10.1.0

package/index.js

@@ -1,18 +1,20 @@
 const { EventEmitter } = require('events')
-const raf = require('random-access-file')
+const RAF = require('random-access-file')
 const isOptions = require('is-options')
 const hypercoreCrypto = require('hypercore-crypto')
 const c = require('compact-encoding')
+const b4a = require('b4a')
 const Xache = require('xache')
 const NoiseSecretStream = require('@hyperswarm/secret-stream')
-const codecs = require('codecs')
-
-const fsctl = requireMaybe('fsctl') || { lock: noop, sparse: noop }
+const Protomux = require('protomux')
 
 const Replicator = require('./lib/replicator')
-const Extensions = require('./lib/extensions')
 const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
+const Info = require('./lib/info')
+const Download = require('./lib/download')
+const { ReadStream, WriteStream } = require('./lib/streams')
+const { BAD_ARGUMENT, SESSION_CLOSED, SESSION_NOT_WRITABLE, SNAPSHOT_NOT_AVAILABLE } = require('./lib/errors')
 
 const promises = Symbol.for('hypercore.promises')
 const inspect = Symbol.for('nodejs.util.inspect.custom')
@@ -31,13 +33,13 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     if (key && typeof key === 'string') {
-      key = Buffer.from(key, 'hex')
+      key = b4a.from(key, 'hex')
     }
 
     if (!opts) opts = {}
 
     if (!opts.crypto && key && key.byteLength !== 32) {
-      throw new Error('Hypercore key should be 32 bytes')
+      throw BAD_ARGUMENT('Hypercore key should be 32 bytes')
     }
 
     if (!storage) storage = opts.storage
@@ -49,25 +51,34 @@ module.exports = class Hypercore extends EventEmitter {
     this.core = null
     this.replicator = null
     this.encryption = null
-    this.extensions = opts.extensions || new Extensions()
+    this.extensions = new Map()
     this.cache = opts.cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (opts.cache || null)
 
     this.valueEncoding = null
+    this.encodeBatch = null
+    this.activeRequests = []
+
     this.key = key || null
-    this.discoveryKey = null
+    this.keyPair = null
     this.readable = true
     this.writable = false
     this.opened = false
     this.closed = false
+    this.snapshotted = !!opts.snapshot
+    this.sparse = opts.sparse !== false
     this.sessions = opts._sessions || [this]
-    this.sign = opts.sign || null
+    this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
+    this.onwait = opts.onwait || null
+    this.wait = opts.wait !== false
 
     this.closing = null
-    this.opening = opts._opening || this._open(key, storage, opts)
+    this.opening = this._openSession(key, storage, opts)
     this.opening.catch(noop)
 
     this._preappend = preappend.bind(this)
+    this._snapshot = null
+    this._findingPeers = 0
   }
 
   [inspect] (depth, opts) {
@@ -76,75 +87,284 @@ module.exports = class Hypercore extends EventEmitter {
       while (indent.length < opts.indentationLvl) indent += ' '
     }
 
+    let peers = ''
+    const min = Math.min(this.peers.length, 5)
+
+    for (let i = 0; i < min; i++) {
+      const peer = this.peers[i]
+
+      peers += indent + '    Peer(\n'
+      peers += indent + '      remotePublicKey: ' + opts.stylize(toHex(peer.remotePublicKey), 'string') + '\n'
+      peers += indent + '      remoteLength: ' + opts.stylize(peer.remoteLength, 'number') + '\n'
+      peers += indent + '      remoteFork: ' + opts.stylize(peer.remoteFork, 'number') + '\n'
+      peers += indent + '      remoteCanUpgrade: ' + opts.stylize(peer.remoteCanUpgrade, 'boolean') + '\n'
+      peers += indent + '    )' + '\n'
+    }
+
+    if (this.peers.length > 5) {
+      peers += indent + '  ... and ' + (this.peers.length - 5) + ' more\n'
+    }
+
+    if (peers) peers = '[\n' + peers + indent + '  ]'
+    else peers = '[ ' + opts.stylize(0, 'number') + ' ]'
+
     return this.constructor.name + '(\n' +
-      indent + '  key: ' + opts.stylize((toHex(this.key)), 'string') + '\n' +
+      indent + '  key: ' + opts.stylize(toHex(this.key), 'string') + '\n' +
       indent + '  discoveryKey: ' + opts.stylize(toHex(this.discoveryKey), 'string') + '\n' +
       indent + '  opened: ' + opts.stylize(this.opened, 'boolean') + '\n' +
+      indent + '  closed: ' + opts.stylize(this.closed, 'boolean') + '\n' +
+      indent + '  snapshotted: ' + opts.stylize(this.snapshotted, 'boolean') + '\n' +
+      indent + '  sparse: ' + opts.stylize(this.sparse, 'boolean') + '\n' +
       indent + '  writable: ' + opts.stylize(this.writable, 'boolean') + '\n' +
-      indent + '  sessions: ' + opts.stylize(this.sessions.length, 'number') + '\n' +
-      indent + '  peers: [ ' + opts.stylize(this.peers.length, 'number') + ' ]\n' +
       indent + '  length: ' + opts.stylize(this.length, 'number') + '\n' +
-      indent + '  byteLength: ' + opts.stylize(this.byteLength, 'number') + '\n' +
+      indent + '  fork: ' + opts.stylize(this.fork, 'number') + '\n' +
+      indent + '  sessions: [ ' + opts.stylize(this.sessions.length, 'number') + ' ]\n' +
+      indent + '  activeRequests: [ ' + opts.stylize(this.activeRequests.length, 'number') + ' ]\n' +
+      indent + '  peers: ' + peers + '\n' +
       indent + ')'
   }
 
-  static createProtocolStream (isInitiator, opts) {
-    const noiseStream = new NoiseSecretStream(isInitiator, null, opts)
-    return noiseStream.rawStream
+  static getProtocolMuxer (stream) {
+    return stream.noiseStream.userData
+  }
+
+  static createProtocolStream (isInitiator, opts = {}) {
+    let outerStream = Protomux.isProtomux(isInitiator)
+      ? isInitiator.stream
+      : isStream(isInitiator)
+        ? isInitiator
+        : opts.stream
+
+    let noiseStream = null
+
+    if (outerStream) {
+      noiseStream = outerStream.noiseStream
+    } else {
+      noiseStream = new NoiseSecretStream(isInitiator, null, opts)
+      outerStream = noiseStream.rawStream
+    }
+    if (!noiseStream) throw BAD_ARGUMENT('Invalid stream')
+
+    if (!noiseStream.userData) {
+      const protocol = new Protomux(noiseStream)
+
+      if (opts.ondiscoverykey) {
+        protocol.pair({ protocol: 'hypercore/alpha' }, opts.ondiscoverykey)
+      }
+      if (opts.keepAlive !== false) {
+        noiseStream.setKeepAlive(5000)
+      }
+      noiseStream.userData = protocol
+    }
+
+    return outerStream
   }
 
   static defaultStorage (storage, opts = {}) {
-    if (typeof storage !== 'string') return storage
+    if (typeof storage !== 'string') {
+      if (!isRandomAccessClass(storage)) return storage
+      const Cls = storage // just to satisfy standard...
+      return name => new Cls(name)
+    }
+
     const directory = storage
     const toLock = opts.lock || 'oplog'
-    return function createFile (name) {
-      const locked = name === toLock || name.endsWith('/' + toLock)
-      const lock = locked ? fsctl.lock : null
-      const sparse = locked ? null : null // fsctl.sparse, disable sparse on windows - seems to fail for some people. TODO: investigate
-      return raf(name, { directory, lock, sparse })
+
+    return createFile
+
+    function createFile (name) {
+      const lock = isFile(name, toLock)
+      const sparse = isFile(name, 'data') || isFile(name, 'bitfield') || isFile(name, 'tree')
+      return new RAF(name, { directory, lock, sparse })
     }
+
+    function isFile (name, n) {
+      return name === n || name.endsWith('/' + n)
+    }
+  }
+
+  snapshot (opts) {
+    return this.session({ ...opts, snapshot: true })
   }
 
   session (opts = {}) {
     if (this.closing) {
       // This makes the closing logic alot easier. If this turns out to be a problem
       // in practive, open an issue and we'll try to make a solution for it.
-      throw new Error('Cannot make sessions on a closing core')
+      throw SESSION_CLOSED('Cannot make sessions on a closing core')
     }
 
+    const sparse = opts.sparse === false ? false : this.sparse
+    const wait = opts.wait === false ? false : this.wait
+    const onwait = opts.onwait === undefined ? this.onwait : opts.onwait
     const Clz = opts.class || Hypercore
-    const keyPair = opts.keyPair && opts.keyPair.secretKey && { ...opts.keyPair }
-
-    // This only works if the hypercore was fully loaded,
-    // but we only do this to validate the keypair to help catch bugs so yolo
-    if (this.key && keyPair) keyPair.publicKey = this.key
-
     const s = new Clz(this.storage, this.key, {
       ...opts,
-      sign: opts.sign || (keyPair && keyPair.secretKey && Core.createSigner(this.crypto, keyPair)) || this.sign,
-      valueEncoding: this.valueEncoding,
-      extensions: this.extensions,
+      sparse,
+      wait,
+      onwait,
       _opening: this.opening,
       _sessions: this.sessions
     })
 
-    s._initSession(this)
+    s._passCapabilities(this)
+
+    // Configure the cache unless explicitly disabled.
+    if (opts.cache !== false) {
+      s.cache = opts.cache === true || !opts.cache ? this.cache : opts.cache
+    }
+
+    ensureEncryption(s, opts)
+
     this.sessions.push(s)
 
     return s
   }
 
-  _initSession (o) {
-    if (!this.sign) this.sign = o.sign
+  _passCapabilities (o) {
+    if (!this.auth) this.auth = o.auth
+
     this.crypto = o.crypto
-    this.opened = o.opened
     this.key = o.key
-    this.discoveryKey = o.discoveryKey
     this.core = o.core
     this.replicator = o.replicator
     this.encryption = o.encryption
-    this.writable = !!this.sign
+    this.writable = !!(this.auth && this.auth.sign)
     this.autoClose = o.autoClose
+
+    if (this.snapshotted && this.core && !this._snapshot) this._updateSnapshot()
+  }
+
+  async _openFromExisting (from, opts) {
+    await from.opening
+
+    this._passCapabilities(from)
+    this.sessions = from.sessions
+    this.storage = from.storage
+    this.replicator.findingPeers += this._findingPeers
+
+    ensureEncryption(this, opts)
+
+    this.sessions.push(this)
+  }
+
+  async _openSession (key, storage, opts) {
+    const isFirst = !opts._opening
+
+    if (!isFirst) await opts._opening
+    if (opts.preload) opts = { ...opts, ...(await opts.preload()) }
+
+    const keyPair = (key && opts.keyPair)
+      ? { ...opts.keyPair, publicKey: key }
+      : key
+        ? { publicKey: key, secretKey: null }
+        : opts.keyPair
+
+    // This only works if the hypercore was fully loaded,
+    // but we only do this to validate the keypair to help catch bugs so yolo
+    if (this.key && keyPair) keyPair.publicKey = this.key
+
+    if (opts.auth) {
+      this.auth = opts.auth
+    } else if (opts.sign) {
+      this.auth = Core.createAuth(this.crypto, keyPair, opts)
+    } else if (keyPair && keyPair.secretKey) {
+      this.auth = Core.createAuth(this.crypto, keyPair)
+    }
+
+    if (isFirst) {
+      await this._openCapabilities(keyPair, storage, opts)
+      // Only the root session should pass capabilities to other sessions.
+      for (let i = 0; i < this.sessions.length; i++) {
+        const s = this.sessions[i]
+        if (s !== this) s._passCapabilities(this)
+      }
+    }
+
+    if (!this.auth) this.auth = this.core.defaultAuth
+    this.writable = !!this.auth.sign
+
+    if (opts.valueEncoding) {
+      this.valueEncoding = c.from(opts.valueEncoding)
+    }
+    if (opts.encodeBatch) {
+      this.encodeBatch = opts.encodeBatch
+    }
+
+    // Start continous replication if not in sparse mode.
+    if (!this.sparse) this.download({ start: 0, end: -1 })
+
+    // This is a hidden option that's only used by Corestore.
+    // It's required so that corestore can load a name from userData before 'ready' is emitted.
+    if (opts._preready) await opts._preready(this)
+
+    this.opened = true
+    this.emit('ready')
+  }
+
+  async _openCapabilities (keyPair, storage, opts) {
+    if (opts.from) return this._openFromExisting(opts.from, opts)
+
+    this.storage = Hypercore.defaultStorage(opts.storage || storage)
+
+    this.core = await Core.open(this.storage, {
+      force: opts.force,
+      createIfMissing: opts.createIfMissing,
+      overwrite: opts.overwrite,
+      keyPair,
+      crypto: this.crypto,
+      legacy: opts.legacy,
+      auth: opts.auth,
+      onupdate: this._oncoreupdate.bind(this)
+    })
+
+    if (opts.userData) {
+      for (const [key, value] of Object.entries(opts.userData)) {
+        await this.core.userData(key, value)
+      }
+    }
+
+    this.key = this.core.header.signer.publicKey
+    this.keyPair = this.core.header.signer
+
+    this.replicator = new Replicator(this.core, this.key, {
+      eagerUpdate: true,
+      allowFork: opts.allowFork !== false,
+      onpeerupdate: this._onpeerupdate.bind(this),
+      onupload: this._onupload.bind(this)
+    })
+
+    this.replicator.findingPeers += this._findingPeers
+
+    if (!this.encryption && opts.encryptionKey) {
+      this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
+    }
+  }
+
+  _getSnapshot () {
+    if (this.sparse) {
+      return {
+        length: this.core.tree.length,
+        byteLength: this.core.tree.byteLength,
+        fork: this.core.tree.fork,
+        compatLength: this.core.tree.length
+      }
+    }
+
+    return {
+      length: this.core.header.contiguousLength,
+      byteLength: 0,
+      fork: this.core.tree.fork,
+      compatLength: this.core.header.contiguousLength
+    }
+  }
+
+  _updateSnapshot () {
+    const prev = this._snapshot
+    const next = this._snapshot = this._getSnapshot()
+
+    if (!prev) return true
+    return prev.length !== next.length || prev.fork !== next.fork
   }
 
   close () {
@@ -163,6 +383,20 @@ module.exports = class Hypercore extends EventEmitter {
     this.readable = false
     this.writable = false
     this.closed = true
+    this.opened = false
+
+    const gc = []
+    for (const ext of this.extensions.values()) {
+      if (ext.session === this) gc.push(ext)
+    }
+    for (const ext of gc) ext.destroy()
+
+    if (this.replicator !== null) {
+      this.replicator.findingPeers -= this._findingPeers
+      this.replicator.clearRequests(this.activeRequests)
+    }
+
+    this._findingPeers = 0
 
     if (this.sessions.length) {
       // if this is the last session and we are auto closing, trigger that first to enforce error handling
@@ -178,41 +412,63 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   replicate (isInitiator, opts = {}) {
-    let outerStream = isStream(isInitiator)
-      ? isInitiator
-      : opts.stream
-    let noiseStream = null
+    // Only limitation here is that ondiscoverykey doesn't work atm when passing a muxer directly,
+    // because it doesn't really make a lot of sense.
+    if (Protomux.isProtomux(isInitiator)) return this._attachToMuxer(isInitiator, opts)
 
-    if (outerStream) {
-      noiseStream = outerStream.noiseStream
-    } else {
-      outerStream = Hypercore.createProtocolStream(isInitiator, opts)
-      noiseStream = outerStream.noiseStream
-    }
-    if (!noiseStream) throw new Error('Invalid stream passed to replicate')
+    const protocolStream = Hypercore.createProtocolStream(isInitiator, opts)
+    const noiseStream = protocolStream.noiseStream
+    const protocol = noiseStream.userData
 
-    if (!noiseStream.userData) {
-      const protocol = Replicator.createProtocol(noiseStream)
-      noiseStream.userData = protocol
-      noiseStream.on('error', noop) // All noise errors already propagate through outerStream
+    this._attachToMuxer(protocol, opts)
+
+    return protocolStream
+  }
+
+  _attachToMuxer (mux, opts) {
+    // If the user wants to, we can make this replication run in a session
+    // that way the core wont close "under them" during replication
+    if (opts.session) {
+      const s = this.session()
+      mux.stream.on('close', () => s.close().catch(noop))
     }
 
-    const protocol = noiseStream.userData
     if (this.opened) {
-      this.replicator.joinProtocol(protocol, this.key, this.discoveryKey)
+      this.replicator.attachTo(mux)
     } else {
-      this.opening.then(() => this.replicator.joinProtocol(protocol, this.key, this.discoveryKey), protocol.destroy.bind(protocol))
+      this.opening.then(() => this.replicator.attachTo(mux), mux.destroy.bind(mux))
     }
 
-    return outerStream
+    return mux
+  }
+
+  get discoveryKey () {
+    return this.replicator === null ? null : this.replicator.discoveryKey
   }
 
   get length () {
-    return this.core === null ? 0 : this.core.tree.length
+    if (this._snapshot) return this._snapshot.length
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousLength
+    return this.core.tree.length
   }
 
+  /**
+   * Deprecated. Use `const { byteLength } = await core.info()`.
+   */
   get byteLength () {
-    return this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding)
+    if (this._snapshot) return this._snapshot.byteLength
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousByteLength
+    return this.core.tree.byteLength - (this.core.tree.length * this.padding)
+  }
+
+  get contiguousLength () {
+    return this.core === null ? 0 : this.core.header.contiguousLength
+  }
+
+  get contiguousByteLength () {
+    return 0
   }
 
   get fork () {
@@ -235,111 +491,86 @@ module.exports = class Hypercore extends EventEmitter {
     return this.opening
   }
 
-  async _open (key, storage, opts) {
-    if (opts.preload) opts = { ...opts, ...(await opts.preload()) }
-
-    this.valueEncoding = opts.valueEncoding ? c.from(codecs(opts.valueEncoding)) : null
-
-    const keyPair = (key && opts.keyPair)
-      ? { ...opts.keyPair, publicKey: key }
-      : key
-        ? { publicKey: key, secretKey: null }
-        : opts.keyPair
+  _onupload (index, value, from) {
+    const byteLength = value.byteLength - this.padding
 
-    if (opts.from) {
-      const from = opts.from
-      await from.opening
-      for (const [name, ext] of this.extensions) from.extensions.register(name, null, ext)
-      this._initSession(from)
-      this.extensions = from.extensions
-      this.sessions = from.sessions
-      this.storage = from.storage
-      if (!this.sign) this.sign = opts.sign || ((keyPair && keyPair.secretKey) ? Core.createSigner(this.crypto, keyPair) : null)
-      this.writable = !!this.sign
-      this.sessions.push(this)
-      return
+    for (let i = 0; i < this.sessions.length; i++) {
+      this.sessions[i].emit('upload', index, byteLength, from)
     }
+  }
 
-    if (!this.storage) this.storage = Hypercore.defaultStorage(opts.storage || storage)
+  _oncoreupdate (status, bitfield, value, from) {
+    if (status !== 0) {
+      const truncatedNonSparse = (status & 0b1000) !== 0
+      const appendedNonSparse = (status & 0b0100) !== 0
+      const truncated = (status & 0b0010) !== 0
+      const appended = (status & 0b0001) !== 0
 
-    this.core = await Core.open(this.storage, {
-      keyPair,
-      crypto: this.crypto,
-      onupdate: this._oncoreupdate.bind(this)
-    })
+      if (truncated) {
+        this.replicator.ontruncate(bitfield.start)
+      }
 
-    if (opts.userData) {
-      for (const [key, value] of Object.entries(opts.userData)) {
-        await this.core.userData(key, value)
+      if ((status & 0b0011) !== 0) {
+        this.replicator.onupgrade()
       }
-    }
 
-    this.replicator = new Replicator(this.core, {
-      onupdate: this._onpeerupdate.bind(this)
-    })
+      for (let i = 0; i < this.sessions.length; i++) {
+        const s = this.sessions[i]
 
-    if (!this.sign) this.sign = opts.sign || this.core.defaultSign
+        if (truncated) {
+          if (s.cache) s.cache.clear()
 
-    this.discoveryKey = this.crypto.discoveryKey(this.core.header.signer.publicKey)
-    this.key = this.core.header.signer.publicKey
-    this.writable = !!this.sign
+          // If snapshotted, make sure to update our compat so we can fail gets
+          if (s._snapshot && bitfield.start < s._snapshot.compatLength) s._snapshot.compatLength = bitfield.start
+        }
 
-    if (!this.encryption && opts.encryptionKey) {
-      this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
-    }
+        if (s.sparse ? truncated : truncatedNonSparse) {
+          s.emit('truncate', bitfield.start, this.core.tree.fork)
+        }
 
-    this.extensions.attach(this.replicator)
-    this.opened = true
+        // For sparse sessions, immediately emit appends. If non-sparse, emit if contig length has updated
+        if (s.sparse ? appended : appendedNonSparse) {
+          s.emit('append')
+        }
+      }
 
-    if (opts.postload) await opts.postload(this)
+      const contig = this.core.header.contiguousLength
 
-    for (let i = 0; i < this.sessions.length; i++) {
-      const s = this.sessions[i]
-      if (s !== this) s._initSession(this)
-      s.emit('ready')
-    }
-  }
+      // When the contig length catches up, broadcast the non-sparse length to peers
+      if (appendedNonSparse && contig === this.core.tree.length) {
+        for (const peer of this.peers) {
+          if (peer.broadcastedNonSparse) continue
 
-  _oncoreupdate (status, bitfield, value, from) {
-    if (status !== 0) {
-      for (let i = 0; i < this.sessions.length; i++) {
-        if ((status & 0b10) !== 0) {
-          if (this.cache) this.cache.clear()
-          this.sessions[i].emit('truncate', this.core.tree.fork)
-        }
-        if ((status & 0b01) !== 0) {
-          this.sessions[i].emit('append')
+          peer.broadcastRange(0, contig)
+          peer.broadcastedNonSparse = true
         }
       }
-
-      this.replicator.broadcastInfo()
     }
 
-    if (bitfield && !bitfield.drop) { // TODO: support drop!
-      for (let i = 0; i < bitfield.length; i++) {
-        this.replicator.broadcastBlock(bitfield.start + i)
-      }
+    if (bitfield) {
+      this.replicator.onhave(bitfield.start, bitfield.length, bitfield.drop)
     }
 
     if (value) {
-      if (this.encryption) {
-        this.encryption.decrypt(bitfield.start, value)
-      }
-
-      value = value.subarray(this.padding)
+      const byteLength = value.byteLength - this.padding
 
       for (let i = 0; i < this.sessions.length; i++) {
-        this.sessions[i].emit('download', bitfield.start, value, from)
+        this.sessions[i].emit('download', bitfield.start, byteLength, from)
       }
     }
   }
 
   _onpeerupdate (added, peer) {
-    if (added) this.extensions.update(peer)
     const name = added ? 'peer-add' : 'peer-remove'
 
     for (let i = 0; i < this.sessions.length; i++) {
       this.sessions[i].emit(name, peer)
+
+      if (added) {
+        for (const ext of this.sessions[i].extensions.values()) {
+          peer.extensions.set(ext.name, ext)
+        }
+      }
     }
   }
 
@@ -356,19 +587,72 @@ module.exports = class Hypercore extends EventEmitter {
     return null
   }
 
-  async update () {
+  findingPeers () {
+    this._findingPeers++
+    if (this.replicator !== null && !this.closing) this.replicator.findingPeers++
+
+    let once = true
+
+    return () => {
+      if (this.closing || !once) return
+      once = false
+      this._findingPeers--
+      if (this.replicator !== null && --this.replicator.findingPeers === 0) {
+        this.replicator.updateAll()
+      }
+    }
+  }
+
+  async info () {
     if (this.opened === false) await this.opening
-    // TODO: add an option where a writer can bootstrap it's state from the network also
-    if (this.writable) return false
-    return this.replicator.requestUpgrade()
+
+    return Info.from(this.core, this.padding, this._snapshot)
+  }
+
+  async update (opts) {
+    if (this.opened === false) await this.opening
+    if (this.closing !== null) return false
+
+    if (this.writable && (!opts || opts.force !== true)) {
+      if (!this.snapshotted) return false
+      return this._updateSnapshot()
+    }
+
+    const activeRequests = (opts && opts.activeRequests) || this.activeRequests
+    const req = this.replicator.addUpgrade(activeRequests)
+
+    let upgraded = await req.promise
+
+    if (!this.sparse) {
+      // Download all available blocks in non-sparse mode
+      const start = this.length
+      const end = this.core.tree.length
+      const contig = this.contiguousLength
+
+      await this.download({ start, end, ifAvailable: true }).done()
+
+      if (!upgraded) upgraded = this.contiguousLength !== contig
+    }
+
+    if (!upgraded) return false
+    if (this.snapshotted) return this._updateSnapshot()
+    return true
   }
 
-  async seek (bytes) {
+  async seek (bytes, opts) {
     if (this.opened === false) await this.opening
 
     const s = this.core.tree.seek(bytes, this.padding)
 
-    return (await s.update()) || this.replicator.requestSeek(s)
+    const offset = await s.update()
+    if (offset) return offset
+
+    if (this.closing !== null) throw SESSION_CLOSED()
+
+    const activeRequests = (opts && opts.activeRequests) || this.activeRequests
+    const req = this.replicator.addSeek(activeRequests, s)
+
+    return req.promise
   }
 
   async has (index) {
@@ -379,62 +663,97 @@ module.exports = class Hypercore extends EventEmitter {
 
   async get (index, opts) {
     if (this.opened === false) await this.opening
-    const c = this.cache && this.cache.get(index)
-    if (c) return c
-    const fork = this.core.tree.fork
-    const b = await this._get(index, opts)
-    if (this.cache && fork === this.core.tree.fork && b) this.cache.set(index, b)
-    return b
+    if (this.closing !== null) throw SESSION_CLOSED()
+    if (this._snapshot !== null && index >= this._snapshot.compatLength) throw SNAPSHOT_NOT_AVAILABLE()
+
+    const encoding = (opts && opts.valueEncoding && c.from(opts.valueEncoding)) || this.valueEncoding
+
+    let req = this.cache && this.cache.get(index)
+    if (!req) req = this._get(index, opts)
+
+    let block = await req
+    if (!block) return null
+
+    if (this.encryption) {
+      // Copy the block as it might be shared with other sessions.
+      block = b4a.from(block)
+
+      this.encryption.decrypt(index, block)
+    }
+
+    return this._decode(encoding, block)
   }
 
-  async _get (index, opts) {
-    const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
+  async clear (start, end = start + 1, opts) {
+    if (this.opened === false) await this.opening
+    if (this.closing !== null) throw SESSION_CLOSED()
+
+    if (typeof end === 'object') {
+      opts = end
+      end = start + 1
+    }
+
+    if (start >= end) return
+
+    await this.core.clear(start, end)
+  }
 
+  async _get (index, opts) {
     let block
 
     if (this.core.bitfield.get(index)) {
-      block = await this.core.blocks.get(index)
+      block = this.core.blocks.get(index)
+
+      if (this.cache) this.cache.set(index, block)
     } else {
-      if (opts && opts.onwait) opts.onwait(index)
-      block = await this.replicator.requestBlock(index)
+      if (opts && opts.wait === false) return null
+      if (this.wait === false && (!opts || !opts.wait)) return null
+      if (opts && opts.onwait) opts.onwait(index, this)
+      if (this.onwait) this.onwait(index, this)
+
+      const activeRequests = (opts && opts.activeRequests) || this.activeRequests
+
+      const req = this.replicator.addBlock(activeRequests, index)
+
+      block = this._cacheOnResolve(index, req.promise, this.core.tree.fork)
     }
 
-    if (this.encryption) this.encryption.decrypt(index, block)
-    return this._decode(encoding, block)
+    return block
   }
 
-  download (range) {
-    const linear = !!(range && range.linear)
+  async _cacheOnResolve (index, req, fork) {
+    const block = await req
 
-    let start
-    let end
-    let filter
+    if (this.cache && fork === this.core.tree.fork) {
+      this.cache.set(index, Promise.resolve(block))
+    }
 
-    if (range && range.blocks) {
-      const blocks = range.blocks instanceof Set
-        ? range.blocks
-        : new Set(range.blocks)
+    return block
+  }
 
-      start = range.start || (blocks.size ? min(range.blocks) : 0)
-      end = range.end || (blocks.size ? max(range.blocks) + 1 : 0)
+  createReadStream (opts) {
+    return new ReadStream(this, opts)
+  }
 
-      filter = (i) => blocks.has(i)
-    } else {
-      start = (range && range.start) || 0
-      end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
-    }
+  createWriteStream (opts) {
+    return new WriteStream(this, opts)
+  }
 
-    const r = Replicator.createRange(start, end, filter, linear)
+  download (range) {
+    const req = this._download(range)
 
-    if (this.opened) this.replicator.addRange(r)
-    else this.opening.then(() => this.replicator.addRange(r), noop)
+    // do not crash in the background...
+    req.catch(noop)
 
-    return r
+    return new Download(req)
   }
 
-  // TODO: get rid of this / deprecate it?
-  cancel (request) {
-    // Do nothing for now
+  async _download (range) {
+    if (this.opened === false) await this.opening
+
+    const activeRequests = (range && range.activeRequests) || this.activeRequests
+
+    return this.replicator.addRange(activeRequests, range)
   }
 
   // TODO: get rid of this / deprecate it?
@@ -442,12 +761,17 @@ module.exports = class Hypercore extends EventEmitter {
     range.destroy(null)
   }
 
+  // TODO: get rid of this / deprecate it?
+  cancel (request) {
+    // Do nothing for now
+  }
+
   async truncate (newLength = 0, fork = -1) {
     if (this.opened === false) await this.opening
-    if (this.writable === false) throw new Error('Core is not writable')
+    if (this.writable === false) throw SESSION_NOT_WRITABLE()
 
     if (fork === -1) fork = this.core.tree.fork + 1
-    await this.core.truncate(newLength, fork, this.sign)
+    await this.core.truncate(newLength, fork, this.auth)
 
     // TODO: Should propagate from an event triggered by the oplog
     this.replicator.updateAll()
@@ -455,44 +779,92 @@ module.exports = class Hypercore extends EventEmitter {
 
   async append (blocks) {
     if (this.opened === false) await this.opening
-    if (this.writable === false) throw new Error('Core is not writable')
+    if (this.writable === false) throw SESSION_NOT_WRITABLE()
 
     blocks = Array.isArray(blocks) ? blocks : [blocks]
 
     const preappend = this.encryption && this._preappend
-    const buffers = new Array(blocks.length)
 
-    for (let i = 0; i < blocks.length; i++) {
-      buffers[i] = this._encode(this.valueEncoding, blocks[i])
+    const buffers = this.encodeBatch !== null ? this.encodeBatch(blocks) : new Array(blocks.length)
+
+    if (this.encodeBatch === null) {
+      for (let i = 0; i < blocks.length; i++) {
+        buffers[i] = this._encode(this.valueEncoding, blocks[i])
+      }
     }
 
-    return await this.core.append(buffers, this.sign, { preappend })
+    return await this.core.append(buffers, this.auth, { preappend })
   }
 
-  registerExtension (name, handlers) {
-    return this.extensions.register(name, handlers)
+  async treeHash (length) {
+    if (length === undefined) {
+      await this.ready()
+      length = this.core.length
+    }
+
+    const roots = await this.core.tree.getRoots(length)
+    return this.crypto.tree(roots)
   }
 
-  // called by the extensions
-  onextensionupdate () {
-    if (this.replicator !== null) this.replicator.broadcastOptions()
+  registerExtension (name, handlers = {}) {
+    if (this.extensions.has(name)) {
+      const ext = this.extensions.get(name)
+      ext.handlers = handlers
+      ext.encoding = c.from(handlers.encoding || c.buffer)
+      ext.session = this
+      return ext
+    }
+
+    const ext = {
+      name,
+      handlers,
+      encoding: c.from(handlers.encoding || c.buffer),
+      session: this,
+      send (message, peer) {
+        const buffer = c.encode(this.encoding, message)
+        peer.extension(name, buffer)
+      },
+      broadcast (message) {
+        const buffer = c.encode(this.encoding, message)
+        for (const peer of this.session.peers) {
+          peer.extension(name, buffer)
+        }
+      },
+      destroy () {
+        for (const peer of this.session.peers) {
+          if (peer.extensions.get(name) === ext) peer.extensions.delete(name)
+        }
+        this.session.extensions.delete(name)
+      },
+      _onmessage (state, peer) {
+        const m = this.encoding.decode(state)
+        if (this.handlers.onmessage) this.handlers.onmessage(m, peer)
+      }
+    }
+
+    this.extensions.set(name, ext)
+    for (const peer of this.peers) {
+      peer.extensions.set(name, ext)
+    }
+
+    return ext
   }
 
   _encode (enc, val) {
     const state = { start: this.padding, end: this.padding, buffer: null }
 
-    if (Buffer.isBuffer(val)) {
+    if (b4a.isBuffer(val)) {
       if (state.start === 0) return val
       state.end += val.byteLength
     } else if (enc) {
       enc.preencode(state, val)
     } else {
-      val = Buffer.from(val)
+      val = b4a.from(val)
       if (state.start === 0) return val
       state.end += val.byteLength
     }
 
-    state.buffer = Buffer.allocUnsafe(state.end)
+    state.buffer = b4a.allocUnsafe(state.end)
 
     if (enc) enc.encode(state, val)
     else state.buffer.set(val, state.start)
@@ -501,7 +873,7 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   _decode (enc, block) {
-    block = block.subarray(this.padding)
+    if (this.padding) block = block.subarray(this.padding)
     if (enc) return c.decode(enc, block)
     return block
   }
@@ -513,29 +885,12 @@ function isStream (s) {
   return typeof s === 'object' && s && typeof s.pipe === 'function'
 }
 
-function requireMaybe (name) {
-  try {
-    return require(name)
-  } catch (_) {
-    return null
-  }
+function isRandomAccessClass (fn) {
+  return !!(typeof fn === 'function' && fn.prototype && typeof fn.prototype.open === 'function')
 }
 
 function toHex (buf) {
-  return buf && buf.toString('hex')
-}
-
-function reduce (iter, fn, acc) {
-  for (const item of iter) acc = fn(acc, item)
-  return acc
-}
-
-function min (arr) {
-  return reduce(arr, (a, b) => Math.min(a, b), Infinity)
-}
-
-function max (arr) {
-  return reduce(arr, (a, b) => Math.max(a, b), -Infinity)
+  return buf && b4a.toString(buf, 'hex')
 }
 
 function preappend (blocks) {
@@ -546,3 +901,11 @@ function preappend (blocks) {
     this.encryption.encrypt(offset + i, blocks[i], fork)
   }
 }
+
+function ensureEncryption (core, opts) {
+  if (!opts.encryptionKey) return
+  // Only override the block encryption if its either not already set or if
+  // the caller provided a different key.
+  if (core.encryption && b4a.equals(core.encryption.key, opts.encryptionKey)) return
+  core.encryption = new BlockEncryption(opts.encryptionKey, core.key)
+}