hypercore 10.0.0-alpha.23 → 10.0.0-alpha.26

package/README.md

@@ -101,6 +101,10 @@ Truncate the core to a smaller length.
 Per default this will update the fork id of the core to `+ 1`, but you can set the fork id you prefer with the option.
 Note that the fork id should be monotonely incrementing.
 
+#### `const hash = await core.treeHash([length])`
+
+Get the Merkle Tree hash of the core at a given length, defaulting to the current length of the core.
+
 #### `const stream = core.createReadStream([options])`
 
 Make a read stream. Options include:
@@ -212,6 +216,12 @@ Buffer containing the public key identifying this core.
 
 Populated after `ready` has been emitted. Will be `null` before the event.
 
+#### `core.keyPair`
+
+Object containing buffers of the core's public and secret key
+
+Populated after `ready` has been emitted. Will be `null` before the event.
+
 #### `core.discoveryKey`
 
 Buffer containing a key derived from the core's public key.
@@ -274,6 +284,6 @@ socket.pipe(localCore.replicate(true)).pipe(socket)
 
 Emitted when the core has been appended to (i.e. has a new length / byteLength), either locally or remotely.
 
-#### `core.on('truncate')`
+#### `core.on('truncate', ancestors, forkId)`
 
 Emitted when the core has been truncated, either locally or remotely.

package/index.js

@@ -6,12 +6,12 @@ const c = require('compact-encoding')
 const b4a = require('b4a')
 const Xache = require('xache')
 const NoiseSecretStream = require('@hyperswarm/secret-stream')
+const Protomux = require('protomux')
 const codecs = require('codecs')
 
 const fsctl = requireMaybe('fsctl') || { lock: noop, sparse: noop }
 
 const Replicator = require('./lib/replicator')
-const Extensions = require('./lib/extensions')
 const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
 const { ReadStream, WriteStream } = require('./lib/streams')
@@ -51,14 +51,15 @@ module.exports = class Hypercore extends EventEmitter {
     this.core = null
     this.replicator = null
     this.encryption = null
-    this.extensions = opts.extensions || new Extensions()
+    this.extensions = opts.extensions || new Map()
     this.cache = opts.cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (opts.cache || null)
 
     this.valueEncoding = null
     this.encodeBatch = null
+    this.activeRequests = []
 
     this.key = key || null
-    this.discoveryKey = null
+    this.keyPair = null
     this.readable = true
     this.writable = false
     this.opened = false
@@ -72,6 +73,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.opening.catch(noop)
 
     this._preappend = preappend.bind(this)
+    this._snapshot = opts.snapshot || null
   }
 
   [inspect] (depth, opts) {
@@ -92,10 +94,17 @@ module.exports = class Hypercore extends EventEmitter {
       indent + ')'
   }
 
+  static protomux (stream, opts) {
+    return stream.noiseStream.userData.open(opts)
+  }
+
   static createProtocolStream (isInitiator, opts = {}) {
-    let outerStream = isStream(isInitiator)
-      ? isInitiator
-      : opts.stream
+    let outerStream = Protomux.isProtomux(isInitiator)
+      ? isInitiator.stream
+      : isStream(isInitiator)
+        ? isInitiator
+        : opts.stream
+
     let noiseStream = null
 
     if (outerStream) {
@@ -107,10 +116,16 @@ module.exports = class Hypercore extends EventEmitter {
     if (!noiseStream) throw new Error('Invalid stream')
 
     if (!noiseStream.userData) {
-      const protocol = Replicator.createProtocol(noiseStream, opts)
-      if (opts.keepAlive !== false) protocol.setKeepAlive(true)
+      const protocol = new Protomux(noiseStream)
+
+      if (opts.ondiscoverykey) {
+        protocol.pair({ protocol: 'hypercore/alpha' }, opts.ondiscoverykey)
+      }
+      if (opts.keepAlive !== false) {
+        noiseStream.setKeepAlive(5000)
+        noiseStream.setTimeout(7000)
+      }
       noiseStream.userData = protocol
-      noiseStream.on('error', noop) // All noise errors already propagate through outerStream
     }
 
     return outerStream
@@ -128,6 +143,10 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
+  snapshot () {
+    return this.session({ snapshot: { length: this.length, byteLength: this.byteLength, fork: this.fork } })
+  }
+
   session (opts = {}) {
     if (this.closing) {
       // This makes the closing logic alot easier. If this turns out to be a problem
@@ -153,7 +172,6 @@ module.exports = class Hypercore extends EventEmitter {
     if (!this.sign) this.sign = o.sign
     this.crypto = o.crypto
     this.key = o.key
-    this.discoveryKey = o.discoveryKey
     this.core = o.core
     this.replicator = o.replicator
     this.encryption = o.encryption
@@ -231,10 +249,12 @@ module.exports = class Hypercore extends EventEmitter {
     this.storage = Hypercore.defaultStorage(opts.storage || storage)
 
     this.core = await Core.open(this.storage, {
+      force: opts.force,
       createIfMissing: opts.createIfMissing,
       overwrite: opts.overwrite,
       keyPair,
       crypto: this.crypto,
+      legacy: opts.legacy,
       onupdate: this._oncoreupdate.bind(this)
     })
 
@@ -244,18 +264,19 @@ module.exports = class Hypercore extends EventEmitter {
       }
     }
 
-    this.replicator = new Replicator(this.core, {
-      onupdate: this._onpeerupdate.bind(this)
-    })
-
-    this.discoveryKey = this.crypto.discoveryKey(this.core.header.signer.publicKey)
     this.key = this.core.header.signer.publicKey
+    this.keyPair = this.core.header.signer
+
+    this.replicator = new Replicator(this.core, this.key, {
+      eagerUpdate: true,
+      allowFork: opts.allowFork !== false,
+      onpeerupdate: this._onpeerupdate.bind(this),
+      onupload: this._onupload.bind(this)
+    })
 
     if (!this.encryption && opts.encryptionKey) {
       this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
     }
-
-    this.extensions.attach(this.replicator)
   }
 
   close () {
@@ -274,6 +295,17 @@ module.exports = class Hypercore extends EventEmitter {
     this.readable = false
     this.writable = false
     this.closed = true
+    this.opened = false
+
+    const gc = []
+    for (const ext of this.extensions.values()) {
+      if (ext.session === this) gc.push(ext)
+    }
+    for (const ext of gc) ext.destroy()
+
+    if (this.replicator !== null) {
+      this.replicator.clearRequests(this.activeRequests)
+    }
 
     if (this.sessions.length) {
       // if this is the last session and we are auto closing, trigger that first to enforce error handling
@@ -294,24 +326,34 @@ module.exports = class Hypercore extends EventEmitter {
     const protocol = noiseStream.userData
 
     if (this.opened) {
-      this.replicator.joinProtocol(protocol, this.key, this.discoveryKey)
+      this.replicator.attachTo(protocol)
     } else {
-      this.opening.then(() => this.replicator.joinProtocol(protocol, this.key, this.discoveryKey), protocol.destroy.bind(protocol))
+      this.opening.then(() => this.replicator.attachTo(protocol), protocol.destroy.bind(protocol))
     }
 
     return protocolStream
   }
 
+  get discoveryKey () {
+    return this.replicator === null ? null : this.replicator.discoveryKey
+  }
+
   get length () {
-    return this.core === null ? 0 : this.core.tree.length
+    return this._snapshot
+      ? this._snapshot.length
+      : (this.core === null ? 0 : this.core.tree.length)
   }
 
   get byteLength () {
-    return this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding)
+    return this._snapshot
+      ? this._snapshot.byteLength
+      : (this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding))
   }
 
   get fork () {
-    return this.core === null ? 0 : this.core.tree.fork
+    return this._snapshot
+      ? this._snapshot.fork
+      : (this.core === null ? 0 : this.core.tree.fork)
   }
 
   get peers () {
@@ -330,25 +372,31 @@ module.exports = class Hypercore extends EventEmitter {
     return this.opening
   }
 
+  _onupload (index, value, from) {
+    const byteLength = value.byteLength - this.padding
+
+    for (let i = 0; i < this.sessions.length; i++) {
+      this.sessions[i].emit('upload', index, byteLength, from)
+    }
+  }
+
   _oncoreupdate (status, bitfield, value, from) {
     if (status !== 0) {
       for (let i = 0; i < this.sessions.length; i++) {
         if ((status & 0b10) !== 0) {
           if (this.cache) this.cache.clear()
-          this.sessions[i].emit('truncate', this.core.tree.fork)
+          this.sessions[i].emit('truncate', bitfield.start, this.core.tree.fork)
         }
         if ((status & 0b01) !== 0) {
           this.sessions[i].emit('append')
         }
       }
 
-      this.replicator.broadcastInfo()
+      this.replicator.signalUpgrade()
     }
 
-    if (bitfield && !bitfield.drop) { // TODO: support drop!
-      for (let i = 0; i < bitfield.length; i++) {
-        this.replicator.broadcastBlock(bitfield.start + i)
-      }
+    if (bitfield) {
+      this.replicator.broadcastRange(bitfield.start, bitfield.length, bitfield.drop)
     }
 
     if (value) {
@@ -361,9 +409,14 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   _onpeerupdate (added, peer) {
-    if (added) this.extensions.update(peer)
     const name = added ? 'peer-add' : 'peer-remove'
 
+    if (added) {
+      for (const ext of this.extensions.values()) {
+        peer.extensions.set(ext.name, ext)
+      }
+    }
+
     for (let i = 0; i < this.sessions.length; i++) {
       this.sessions[i].emit(name, peer)
     }
@@ -382,19 +435,32 @@ module.exports = class Hypercore extends EventEmitter {
     return null
   }
 
-  async update () {
+  async update (opts) {
     if (this.opened === false) await this.opening
+
     // TODO: add an option where a writer can bootstrap it's state from the network also
-    if (this.writable) return false
-    return this.replicator.requestUpgrade()
+    if (this.writable || this.closing !== null) return false
+
+    const activeRequests = (opts && opts.activeRequests) || this.activeRequests
+    const req = this.replicator.addUpgrade(activeRequests)
+
+    return req.promise
   }
 
-  async seek (bytes) {
+  async seek (bytes, opts) {
     if (this.opened === false) await this.opening
 
     const s = this.core.tree.seek(bytes, this.padding)
 
-    return (await s.update()) || this.replicator.requestSeek(s)
+    const offset = await s.update()
+    if (offset) return offset
+
+    if (this.closing !== null) throw new Error('Session is closed')
+
+    const activeRequests = (opts && opts.activeRequests) || this.activeRequests
+    const req = this.replicator.addSeek(activeRequests, s)
+
+    return req.promise
   }
 
   async has (index) {
@@ -405,6 +471,8 @@ module.exports = class Hypercore extends EventEmitter {
 
   async get (index, opts) {
     if (this.opened === false) await this.opening
+    if (this.closing !== null) throw new Error('Session is closed')
+
     const c = this.cache && this.cache.get(index)
     if (c) return c
     const fork = this.core.tree.fork
@@ -423,7 +491,11 @@ module.exports = class Hypercore extends EventEmitter {
     } else {
       if (opts && opts.wait === false) return null
       if (opts && opts.onwait) opts.onwait(index)
-      block = await this.replicator.requestBlock(index)
+
+      const activeRequests = (opts && opts.activeRequests) || this.activeRequests
+      const req = this.replicator.addBlock(activeRequests, index)
+
+      block = await req.promise
     }
 
     if (this.encryption) this.encryption.decrypt(index, block)
@@ -439,37 +511,27 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   download (range) {
-    const linear = !!(range && range.linear)
-
-    let start
-    let end
-    let filter
-
-    if (range && range.blocks) {
-      const blocks = range.blocks instanceof Set
-        ? range.blocks
-        : new Set(range.blocks)
-
-      start = range.start || (blocks.size ? min(range.blocks) : 0)
-      end = range.end || (blocks.size ? max(range.blocks) + 1 : 0)
-
-      filter = (i) => blocks.has(i)
-    } else {
-      start = (range && range.start) || 0
-      end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
+    const reqP = this._download(range)
+
+    // do not crash in the background...
+    reqP.catch(noop)
+
+    // TODO: turn this into an actual object...
+    return {
+      async downloaded () {
+        const req = await reqP
+        return req.promise
+      },
+      destroy () {
+        reqP.then(req => req.context && req.context.detach(req), noop)
+      }
     }
-
-    const r = Replicator.createRange(start, end, filter, linear)
-
-    if (this.opened) this.replicator.addRange(r)
-    else this.opening.then(() => this.replicator.addRange(r), noop)
-
-    return r
   }
 
-  // TODO: get rid of this / deprecate it?
-  cancel (request) {
-    // Do nothing for now
+  async _download (range) {
+    if (this.opened === false) await this.opening
+    const activeRequests = (range && range.activeRequests) || this.activeRequests
+    return this.replicator.addRange(activeRequests, range)
   }
 
   // TODO: get rid of this / deprecate it?
@@ -477,6 +539,11 @@ module.exports = class Hypercore extends EventEmitter {
     range.destroy(null)
   }
 
+  // TODO: get rid of this / deprecate it?
+  cancel (request) {
+    // Do nothing for now
+  }
+
   async truncate (newLength = 0, fork = -1) {
     if (this.opened === false) await this.opening
     if (this.writable === false) throw new Error('Core is not writable')
@@ -507,13 +574,58 @@ module.exports = class Hypercore extends EventEmitter {
     return await this.core.append(buffers, this.sign, { preappend })
   }
 
-  registerExtension (name, handlers) {
-    return this.extensions.register(name, handlers)
+  async treeHash (length) {
+    if (length === undefined) {
+      await this.ready()
+      length = this.core.length
+    }
+
+    const roots = await this.core.tree.getRoots(length)
+    return this.crypto.tree(roots)
   }
 
-  // called by the extensions
-  onextensionupdate () {
-    if (this.replicator !== null) this.replicator.broadcastOptions()
+  registerExtension (name, handlers = {}) {
+    if (this.extensions.has(name)) {
+      const ext = this.extensions.get(name)
+      ext.handlers = handlers
+      ext.encoding = c.from(codecs(handlers.encoding) || c.buffer)
+      ext.session = this
+      return ext
+    }
+
+    const ext = {
+      name,
+      handlers,
+      encoding: c.from(codecs(handlers.encoding) || c.buffer),
+      session: this,
+      send (message, peer) {
+        const buffer = c.encode(this.encoding, message)
+        peer.extension(name, buffer)
+      },
+      broadcast (message) {
+        const buffer = c.encode(this.encoding, message)
+        for (const peer of this.session.peers) {
+          peer.extension(name, buffer)
+        }
+      },
+      destroy () {
+        for (const peer of this.session.peers) {
+          peer.extensions.delete(name)
+        }
+        this.session.extensions.delete(name)
+      },
+      _onmessage (state, peer) {
+        const m = this.encoding.decode(state)
+        if (this.handlers.onmessage) this.handlers.onmessage(m, peer)
+      }
+    }
+
+    this.extensions.set(name, ext)
+    for (const peer of this.peers) {
+      peer.extensions.set(name, ext)
+    }
+
+    return ext
   }
 
   _encode (enc, val) {
@@ -563,19 +675,6 @@ function toHex (buf) {
   return buf && b4a.toString(buf, 'hex')
 }
 
-function reduce (iter, fn, acc) {
-  for (const item of iter) acc = fn(acc, item)
-  return acc
-}
-
-function min (arr) {
-  return reduce(arr, (a, b) => Math.min(a, b), Infinity)
-}
-
-function max (arr) {
-  return reduce(arr, (a, b) => Math.max(a, b), -Infinity)
-}
-
 function preappend (blocks) {
   const offset = this.core.tree.length
   const fork = this.core.tree.fork

package/lib/bitfield.js

@@ -41,8 +41,9 @@ module.exports = class Bitfield {
     this.pages = new BigSparseArray()
     this.unflushed = []
     this.storage = storage
+    this.resumed = !!(buf && buf.byteLength >= 4)
 
-    const all = (buf && buf.byteLength >= 4)
+    const all = this.resumed
       ? new Uint32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4))
       : new Uint32Array(1024)
 
@@ -93,8 +94,10 @@ module.exports = class Bitfield {
   clear () {
     return new Promise((resolve, reject) => {
       this.storage.del(0, Infinity, (err) => {
-        if (err) reject(err)
-        else resolve()
+        if (err) return reject(err)
+        this.pages = new BigSparseArray()
+        this.unflushed = []
+        resolve()
       })
     })
   }

package/lib/caps.js

@@ -0,0 +1,34 @@
+const crypto = require('hypercore-crypto')
+const sodium = require('sodium-universal')
+const b4a = require('b4a')
+const c = require('compact-encoding')
+
+// TODO: rename this to "crypto" and move everything hashing related etc in here
+// Also lets move the tree stuff from hypercore-crypto here, and loose the types
+// from the hashes there - they are not needed since we lock the indexes in the tree
+// hash and just makes alignment etc harder in other languages
+
+const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER] = crypto.namespace('hypercore', 3)
+
+exports.replicate = function (isInitiator, key, handshakeHash) {
+  const out = b4a.allocUnsafe(32)
+  sodium.crypto_generichash_batch(out, [isInitiator ? REPLICATE_INITIATOR : REPLICATE_RESPONDER, key], handshakeHash)
+  return out
+}
+
+exports.treeSignable = function (hash, length, fork) {
+  const state = { start: 0, end: 80, buffer: b4a.allocUnsafe(80) }
+  c.raw.encode(state, TREE)
+  c.raw.encode(state, hash)
+  c.uint64.encode(state, length)
+  c.uint64.encode(state, fork)
+  return state.buffer
+}
+
+exports.treeSignableLegacy = function (hash, length, fork) {
+  const state = { start: 0, end: 48, buffer: b4a.allocUnsafe(48) }
+  c.raw.encode(state, hash)
+  c.uint64.encode(state, length)
+  c.uint64.encode(state, fork)
+  return state.buffer
+}

package/lib/core.js

@@ -5,10 +5,10 @@ const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
-const { oplogHeader, oplogEntry } = require('./messages')
+const m = require('./messages')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, sign, onupdate) {
+  constructor (header, crypto, oplog, tree, blocks, bitfield, sign, legacy, onupdate) {
     this.onupdate = onupdate
     this.header = header
     this.crypto = crypto
@@ -24,6 +24,7 @@ module.exports = class Core {
     this._verifies = null
     this._verifiesFlushed = null
     this._mutex = new Mutex()
+    this._legacy = legacy
   }
 
   static async open (storage, opts = {}) {
@@ -57,18 +58,24 @@ module.exports = class Core {
   }
 
   static async resume (oplogFile, treeFile, bitfieldFile, dataFile, opts) {
-    const overwrite = opts.overwrite === true
+    let overwrite = opts.overwrite === true
+
+    const force = opts.force === true
     const createIfMissing = opts.createIfMissing !== false
     const crypto = opts.crypto || hypercoreCrypto
 
     const oplog = new Oplog(oplogFile, {
-      headerEncoding: oplogHeader,
-      entryEncoding: oplogEntry
+      headerEncoding: m.oplog.header,
+      entryEncoding: m.oplog.entry
     })
 
     let { header, entries } = await oplog.open()
 
-    if (!header || overwrite === true) {
+    if (force && opts.keyPair && header && header.signer && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
+      overwrite = true
+    }
+
+    if (!header || overwrite) {
       if (!createIfMissing) {
         throw new Error('No hypercore is stored here')
       }
@@ -103,6 +110,10 @@ module.exports = class Core {
       await tree.clear()
       await blocks.clear()
       await bitfield.clear()
+      entries = []
+    } else if (bitfield.resumed && header.tree.length === 0) {
+      // If this was an old bitfield, reset it since it loads based on disk size atm (TODO: change that)
+      await bitfield.clear()
     }
 
     const sign = opts.sign || (header.signer.secretKey ? this.createSigner(crypto, header.signer) : null)
@@ -119,7 +130,7 @@ module.exports = class Core {
       }
 
       if (e.bitfield) {
-        bitfield.setRange(e.bitfield.start, e.bitfield.length)
+        bitfield.setRange(e.bitfield.start, e.bitfield.length, !e.bitfield.drop)
       }
 
       if (e.treeUpgrade) {
@@ -136,7 +147,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, sign, opts.onupdate || noop)
+    return new this(header, crypto, oplog, tree, blocks, bitfield, sign, !!opts.legacy, opts.onupdate || noop)
   }
 
   _shouldFlush () {
@@ -227,7 +238,7 @@ module.exports = class Core {
       for (const val of values) batch.append(val)
 
       const hash = batch.hash()
-      batch.signature = await sign(batch.signable(hash))
+      batch.signature = await sign(this._legacy ? batch.signableLegacy(hash) : batch.signable(hash))
 
       const entry = {
         userData: null,
@@ -259,10 +270,15 @@ module.exports = class Core {
     }
   }
 
+  _signed (batch, hash) {
+    const signable = this._legacy ? batch.signableLegacy(hash) : batch.signable(hash)
+    return this.crypto.verify(signable, batch.signature, this.header.signer.publicKey)
+  }
+
   async _verifyExclusive ({ batch, bitfield, value, from }) {
     // TODO: move this to tree.js
     const hash = batch.hash()
-    if (!batch.signature || !this.crypto.verify(batch.signable(hash), batch.signature, this.header.signer.publicKey)) {
+    if (!batch.signature || !this._signed(batch, hash)) {
       throw new Error('Remote signature does not match')
     }