hypercore 10.0.0-alpha.22 → 10.0.0-alpha.25

package/README.md

@@ -101,6 +101,10 @@ Truncate the core to a smaller length.
 Per default this will update the fork id of the core to `+ 1`, but you can set the fork id you prefer with the option.
 Note that the fork id should be monotonely incrementing.
 
+#### `const hash = await core.treeHash([length])`
+
+Get the Merkle Tree hash of the core at a given length, defaulting to the current length of the core.
+
 #### `const stream = core.createReadStream([options])`
 
 Make a read stream. Options include:
@@ -212,6 +216,12 @@ Buffer containing the public key identifying this core.
 
 Populated after `ready` has been emitted. Will be `null` before the event.
 
+#### `core.keyPair`
+
+Object containing buffers of the core's public and secret key
+
+Populated after `ready` has been emitted. Will be `null` before the event.
+
 #### `core.discoveryKey`
 
 Buffer containing a key derived from the core's public key.
@@ -274,6 +284,6 @@ socket.pipe(localCore.replicate(true)).pipe(socket)
 
 Emitted when the core has been appended to (i.e. has a new length / byteLength), either locally or remotely.
 
-#### `core.on('truncate')`
+#### `core.on('truncate', ancestors, forkId)`
 
 Emitted when the core has been truncated, either locally or remotely.

package/index.js

@@ -58,6 +58,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.encodeBatch = null
 
     this.key = key || null
+    this.keyPair = null
     this.discoveryKey = null
     this.readable = true
     this.writable = false
@@ -72,6 +73,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.opening.catch(noop)
 
     this._preappend = preappend.bind(this)
+    this._snapshot = opts.snapshot || null
   }
 
   [inspect] (depth, opts) {
@@ -107,7 +109,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (!noiseStream) throw new Error('Invalid stream')
 
     if (!noiseStream.userData) {
-      const protocol = Replicator.createProtocol(noiseStream)
+      const protocol = Replicator.createProtocol(noiseStream, opts)
       if (opts.keepAlive !== false) protocol.setKeepAlive(true)
       noiseStream.userData = protocol
       noiseStream.on('error', noop) // All noise errors already propagate through outerStream
@@ -128,6 +130,10 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
+  snapshot () {
+    return this.session({ snapshot: { length: this.length, byteLength: this.byteLength, fork: this.fork } })
+  }
+
   session (opts = {}) {
     if (this.closing) {
       // This makes the closing logic alot easier. If this turns out to be a problem
@@ -245,11 +251,13 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     this.replicator = new Replicator(this.core, {
-      onupdate: this._onpeerupdate.bind(this)
+      onupdate: this._onpeerupdate.bind(this),
+      onupload: this._onupload.bind(this)
     })
 
     this.discoveryKey = this.crypto.discoveryKey(this.core.header.signer.publicKey)
     this.key = this.core.header.signer.publicKey
+    this.keyPair = this.core.header.signer
 
     if (!this.encryption && opts.encryptionKey) {
       this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
@@ -274,6 +282,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.readable = false
     this.writable = false
     this.closed = true
+    this.opened = false
 
     if (this.sessions.length) {
       // if this is the last session and we are auto closing, trigger that first to enforce error handling
@@ -303,15 +312,21 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   get length () {
-    return this.core === null ? 0 : this.core.tree.length
+    return this._snapshot
+      ? this._snapshot.length
+      : (this.core === null ? 0 : this.core.tree.length)
   }
 
   get byteLength () {
-    return this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding)
+    return this._snapshot
+      ? this._snapshot.byteLength
+      : (this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding))
   }
 
   get fork () {
-    return this.core === null ? 0 : this.core.tree.fork
+    return this._snapshot
+      ? this._snapshot.fork
+      : (this.core === null ? 0 : this.core.tree.fork)
   }
 
   get peers () {
@@ -330,12 +345,20 @@ module.exports = class Hypercore extends EventEmitter {
     return this.opening
   }
 
+  _onupload (index, value, from) {
+    const byteLength = value.byteLength - this.padding
+
+    for (let i = 0; i < this.sessions.length; i++) {
+      this.sessions[i].emit('upload', index, byteLength, from)
+    }
+  }
+
   _oncoreupdate (status, bitfield, value, from) {
     if (status !== 0) {
       for (let i = 0; i < this.sessions.length; i++) {
         if ((status & 0b10) !== 0) {
           if (this.cache) this.cache.clear()
-          this.sessions[i].emit('truncate', this.core.tree.fork)
+          this.sessions[i].emit('truncate', bitfield.start, this.core.tree.fork)
         }
         if ((status & 0b01) !== 0) {
           this.sessions[i].emit('append')
@@ -507,6 +530,16 @@ module.exports = class Hypercore extends EventEmitter {
     return await this.core.append(buffers, this.sign, { preappend })
   }
 
+  async treeHash (length) {
+    if (length === undefined) {
+      await this.ready()
+      length = this.core.length
+    }
+
+    const roots = await this.core.tree.getRoots(length)
+    return this.crypto.tree(roots)
+  }
+
   registerExtension (name, handlers) {
     return this.extensions.register(name, handlers)
   }

package/lib/merkle-tree.js

@@ -392,6 +392,21 @@ module.exports = class MerkleTree {
     return Promise.all(roots)
   }
 
+  async upgradeable (length) {
+    const indexes = flat.fullRoots(2 * length)
+    const roots = new Array(indexes.length)
+
+    for (let i = 0; i < indexes.length; i++) {
+      roots[i] = this.get(indexes[i], false)
+    }
+
+    for (const node of await Promise.all(roots)) {
+      if (node === null) return false
+    }
+
+    return true
+  }
+
   get (index, error = true) {
     let node = this.unflushed.get(index)
 

package/lib/messages.js

@@ -256,16 +256,25 @@ exports.info = {
   preencode (state, i) {
     c.uint.preencode(state, i.length)
     c.uint.preencode(state, i.fork)
+    c.uint.preencode(state, 1) // flags
   },
   encode (state, i) {
     c.uint.encode(state, i.length)
     c.uint.encode(state, i.fork)
+    c.uint.encode(state, (i.uploading ? 1 : 0) | (i.downloading ? 2 : 0))
   },
   decode (state) {
-    return {
+    const i = {
       length: c.uint.decode(state),
-      fork: c.uint.decode(state)
+      fork: c.uint.decode(state),
+      uploading: true,
+      downloading: true
     }
+    if (state.end <= state.start) return i // backwards compat with prev alphas
+    const flags = c.uint.decode(state)
+    i.uploading = (flags & 1) !== 0
+    i.downloading = (flags & 2) !== 0
+    return i
   }
 }
 

package/lib/protocol.js

@@ -136,11 +136,34 @@ class Peer {
     this.resend = false
     this.state = state
     this.extensions = new Map()
+    this.uploading = true
+    this.downloading = true
     this.destroyed = false
 
     this._destroyer = this._safeDestroy.bind(this)
   }
 
+  setUploading (uploading) {
+    if (uploading === this.uploading) return
+    this.uploading = uploading
+    this._sendInfo()
+  }
+
+  setDownloading (downloading) {
+    if (downloading === this.downloading) return
+    this.downloading = downloading
+    this._sendInfo()
+  }
+
+  _sendInfo () {
+    this.info({
+      length: this.handlers.core.tree.length,
+      fork: this.handlers.core.tree.fork,
+      uploading: this.uploading,
+      downloading: this.downloading
+    })
+  }
+
   onmessage (type, state) {
     const handlers = this.handlers
 

package/lib/replicator.js

@@ -14,6 +14,8 @@ class RemoteState {
     this.bitfield = new RemoteBitfield()
     this.length = 0
     this.fork = 0
+    this.uploading = true
+    this.downloading = true
   }
 }
 
@@ -241,6 +243,7 @@ class RequestPool {
 
   update (peer) {
     if (peer.state.inflight >= peer.state.maxInflight) return false
+    if (peer.downloading === false || peer.state.uploading === false) return false
     if (this.paused) return false
 
     if (peer.state.fork > this.core.tree.fork) {
@@ -624,17 +627,19 @@ class RequestPool {
 }
 
 module.exports = class Replicator {
-  constructor (core, { onupdate }) {
+  constructor (core, { onupdate, onupload }) {
     this.core = core
     this.peers = []
     this.requests = new RequestPool(this, core)
     this.updating = null
     this.pendingPeers = new Set()
     this.onupdate = onupdate
+    this.onupload = onupload
   }
 
-  static createProtocol (noiseStream) {
+  static createProtocol (noiseStream, opts) {
     return new Protocol(noiseStream, {
+      ...opts,
       protocolVersion: 0,
       userAgent: USER_AGENT
     })
@@ -658,8 +663,14 @@ module.exports = class Replicator {
   }
 
   broadcastInfo () {
-    const msg = { length: this.core.tree.length, fork: this.core.tree.fork }
-    for (const peer of this.peers) peer.info(msg)
+    for (const peer of this.peers) {
+      peer.info({
+        length: this.core.tree.length,
+        fork: this.core.tree.fork,
+        uploading: peer.uploading,
+        downloading: peer.downloading
+      })
+    }
     this.updateAll()
   }
 
@@ -737,7 +748,12 @@ module.exports = class Replicator {
     this.peers.push(peer)
     this.onupdate(true, peer)
 
-    peer.info({ length: this.core.tree.length, fork: this.core.tree.fork })
+    peer.info({
+      length: this.core.tree.length,
+      fork: this.core.tree.fork,
+      uploading: peer.uploading,
+      downloading: peer.downloading
+    })
 
     // YOLO send over all the pages for now
     const p = pages(this.core)
@@ -752,12 +768,14 @@ module.exports = class Replicator {
     // This is a no-op because there isn't any state to dealloc currently
   }
 
-  oninfo ({ length, fork }, peer) {
+  oninfo ({ length, fork, uploading, downloading }, peer) {
     const len = peer.state.length
     const forked = peer.state.fork !== fork
 
     peer.state.length = length
     peer.state.fork = fork
+    peer.state.downloading = downloading
+    peer.state.uploading = uploading
 
     if (forked) {
       for (let i = peer.state.length; i < len; i++) {
@@ -813,11 +831,13 @@ module.exports = class Replicator {
   async onrequest (req, peer) {
     const fork = req.fork || peer.state.fork
     if (fork !== this.core.tree.fork) return
+    if (!peer.uploading) return
 
     const proof = await this.core.tree.proof(req)
 
     if (req.block && req.block.value) {
       proof.block.value = await this.core.blocks.get(req.block.index)
+      this.onupload(req.block.index, proof.block.value, peer)
     }
 
     peer.data(proof)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.22",
+  "version": "10.0.0-alpha.25",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {