hypercore 10.0.0-alpha.31 → 10.0.0-alpha.34

package/README.md

@@ -280,6 +280,13 @@ const socket = net.connect(...)
 socket.pipe(localCore.replicate(true)).pipe(socket)
 ```
 
+#### `const done = core.findingPeers()`
+
+Create a hook that tells Hypercore you are finding peers for this core in the background. Call `done` when your current discovery iteration is done.
+If you're using Hyperswarm, you'd normally call this after a `swarm.flush()` finishes.
+
+This allows `core.update` to wait for either the `findingPeers` hook to finish or one peer to appear before deciding whether it should wait for a merkle tree update before returning.
+
 #### `core.on('append')`
 
 Emitted when the core has been appended to (i.e. has a new length / byteLength), either locally or remotely.

package/index.js

@@ -65,7 +65,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.opened = false
     this.closed = false
     this.sessions = opts._sessions || [this]
-    this.sign = opts.sign || null
+    this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
 
     this.closing = null
@@ -74,6 +74,7 @@ module.exports = class Hypercore extends EventEmitter {
 
     this._preappend = preappend.bind(this)
     this._snapshot = opts.snapshot || null
+    this._findingPeers = 0
   }
 
   [inspect] (depth, opts) {
@@ -168,13 +169,13 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   _passCapabilities (o) {
-    if (!this.sign) this.sign = o.sign
+    if (!this.auth) this.auth = o.auth
     this.crypto = o.crypto
     this.key = o.key
     this.core = o.core
     this.replicator = o.replicator
     this.encryption = o.encryption
-    this.writable = !!this.sign
+    this.writable = !!(this.auth && this.auth.sign)
     this.autoClose = o.autoClose
   }
 
@@ -184,6 +185,7 @@ module.exports = class Hypercore extends EventEmitter {
     this._passCapabilities(from)
     this.sessions = from.sessions
     this.storage = from.storage
+    this.replicator.findingPeers += this._findingPeers
 
     this.sessions.push(this)
   }
@@ -204,10 +206,12 @@ module.exports = class Hypercore extends EventEmitter {
     // but we only do this to validate the keypair to help catch bugs so yolo
     if (this.key && keyPair) keyPair.publicKey = this.key
 
-    if (opts.sign) {
-      this.sign = opts.sign
+    if (opts.auth) {
+      this.auth = opts.auth
+    } else if (opts.sign) {
+      this.auth = Core.createAuth(this.crypto, keyPair, opts)
     } else if (keyPair && keyPair.secretKey) {
-      this.sign = Core.createSigner(this.crypto, keyPair)
+      this.auth = Core.createAuth(this.crypto, keyPair)
     }
 
     if (isFirst) {
@@ -219,8 +223,8 @@ module.exports = class Hypercore extends EventEmitter {
       }
     }
 
-    if (!this.sign) this.sign = this.core.defaultSign
-    this.writable = !!this.sign
+    if (!this.auth) this.auth = this.core.defaultAuth
+    this.writable = !!this.auth.sign
 
     if (opts.valueEncoding) {
       this.valueEncoding = c.from(codecs(opts.valueEncoding))
@@ -249,6 +253,7 @@ module.exports = class Hypercore extends EventEmitter {
       keyPair,
       crypto: this.crypto,
       legacy: opts.legacy,
+      auth: opts.auth,
       onupdate: this._oncoreupdate.bind(this)
     })
 
@@ -268,6 +273,8 @@ module.exports = class Hypercore extends EventEmitter {
       onupload: this._onupload.bind(this)
     })
 
+    this.replicator.findingPeers += this._findingPeers
+
     if (!this.encryption && opts.encryptionKey) {
       this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
     }
@@ -298,9 +305,12 @@ module.exports = class Hypercore extends EventEmitter {
     for (const ext of gc) ext.destroy()
 
     if (this.replicator !== null) {
+      this.replicator.findingPeers -= this._findingPeers
       this.replicator.clearRequests(this.activeRequests)
     }
 
+    this._findingPeers = 0
+
     if (this.sessions.length) {
       // if this is the last session and we are auto closing, trigger that first to enforce error handling
       if (this.sessions.length === 1 && this.autoClose) await this.sessions[0].close()
@@ -429,6 +439,22 @@ module.exports = class Hypercore extends EventEmitter {
     return null
   }
 
+  findingPeers () {
+    this._findingPeers++
+    if (this.replicator !== null && !this.closing) this.replicator.findingPeers++
+
+    let once = true
+
+    return () => {
+      if (this.closing || !once) return
+      once = false
+      this._findingPeers--
+      if (this.replicator !== null && --this.replicator.findingPeers === 0) {
+        this.replicator.updateAll()
+      }
+    }
+  }
+
   async update (opts) {
     if (this.opened === false) await this.opening
 
@@ -438,6 +464,7 @@ module.exports = class Hypercore extends EventEmitter {
     const activeRequests = (opts && opts.activeRequests) || this.activeRequests
     const req = this.replicator.addUpgrade(activeRequests)
 
+    // TODO: if snapshot, also update the length/byteLength to latest
     return req.promise
   }
 
@@ -543,7 +570,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (this.writable === false) throw new Error('Core is not writable')
 
     if (fork === -1) fork = this.core.tree.fork + 1
-    await this.core.truncate(newLength, fork, this.sign)
+    await this.core.truncate(newLength, fork, this.auth)
 
     // TODO: Should propagate from an event triggered by the oplog
     this.replicator.updateAll()
@@ -565,7 +592,7 @@ module.exports = class Hypercore extends EventEmitter {
       }
     }
 
-    return await this.core.append(buffers, this.sign, { preappend })
+    return await this.core.append(buffers, this.auth, { preappend })
   }
 
   async treeHash (length) {

package/lib/core.js

@@ -8,7 +8,7 @@ const Bitfield = require('./bitfield')
 const m = require('./messages')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, sign, legacy, onupdate) {
+  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate) {
     this.onupdate = onupdate
     this.header = header
     this.crypto = crypto
@@ -16,7 +16,7 @@ module.exports = class Core {
     this.tree = tree
     this.blocks = blocks
     this.bitfield = bitfield
-    this.defaultSign = sign
+    this.defaultAuth = auth
     this.truncating = 0
 
     this._maxOplogSize = 65536
@@ -51,10 +51,21 @@ module.exports = class Core {
     }
   }
 
-  // TODO: we should prob have a general "auth" abstraction instead somewhere?
-  static createSigner (crypto, { publicKey, secretKey }) {
-    if (!crypto.validateKeyPair({ publicKey, secretKey })) throw new Error('Invalid key pair')
-    return signable => crypto.sign(signable, secretKey)
+  static createAuth (crypto, { publicKey, secretKey }, opts = {}) {
+    if (secretKey && !crypto.validateKeyPair({ publicKey, secretKey })) throw new Error('Invalid key pair')
+
+    const sign = opts.sign
+      ? opts.sign
+      : secretKey
+        ? (signable) => crypto.sign(signable, secretKey)
+        : undefined
+
+    return {
+      sign,
+      verify (signable, signature) {
+        return crypto.verify(signable, signature, publicKey)
+      }
+    }
   }
 
   static async resume (oplogFile, treeFile, bitfieldFile, dataFile, opts) {
@@ -116,7 +127,7 @@ module.exports = class Core {
       await bitfield.clear()
     }
 
-    const sign = opts.sign || (header.signer.secretKey ? this.createSigner(crypto, header.signer) : null)
+    const auth = opts.auth || this.createAuth(crypto, header.signer)
 
     for (const e of entries) {
       if (e.userData) {
@@ -147,7 +158,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, sign, !!opts.legacy, opts.onupdate || noop)
+    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop)
   }
 
   _shouldFlush () {
@@ -212,13 +223,13 @@ module.exports = class Core {
     }
   }
 
-  async truncate (length, fork, sign = this.defaultSign) {
+  async truncate (length, fork, auth = this.defaultAuth) {
     this.truncating++
     await this._mutex.lock()
 
     try {
       const batch = await this.tree.truncate(length, fork)
-      batch.signature = await sign(batch.signable())
+      batch.signature = await auth.sign(batch.signable())
       await this._truncate(batch, null)
     } finally {
       this.truncating--
@@ -226,7 +237,7 @@ module.exports = class Core {
     }
   }
 
-  async append (values, sign = this.defaultSign, hooks = {}) {
+  async append (values, auth = this.defaultAuth, hooks = {}) {
     await this._mutex.lock()
 
     try {
@@ -238,7 +249,7 @@ module.exports = class Core {
       for (const val of values) batch.append(val)
 
       const hash = batch.hash()
-      batch.signature = await sign(this._legacy ? batch.signableLegacy(hash) : batch.signable(hash))
+      batch.signature = await auth.sign(this._legacy ? batch.signableLegacy(hash) : batch.signable(hash))
 
       const entry = {
         userData: null,
@@ -270,9 +281,9 @@ module.exports = class Core {
     }
   }
 
-  _signed (batch, hash) {
+  _signed (batch, hash, auth = this.defaultAuth) {
     const signable = this._legacy ? batch.signableLegacy(hash) : batch.signable(hash)
-    return this.crypto.verify(signable, batch.signature, this.header.signer.publicKey)
+    return auth.verify(signable, batch.signature)
   }
 
   async _verifyExclusive ({ batch, bitfield, value, from }) {

package/lib/replicator.js

@@ -842,6 +842,7 @@ module.exports = class Replicator {
     this.onpeerupdate = onpeerupdate
     this.onupload = onupload
     this.peers = []
+    this.findingPeers = 0 // updateable from the outside
 
     this._inflight = new InflightTracker()
     this._blocks = new BlockTracker(core)
@@ -854,6 +855,7 @@ module.exports = class Replicator {
     this._reorgs = []
     this._ranges = []
 
+    this._hadPeers = false
     this._ifAvailable = 0
     this._updatesPending = 0
     this._applyingReorg = false
@@ -959,6 +961,7 @@ module.exports = class Replicator {
   // Do this when we have more tests.
   _checkUpgradeIfAvailable () {
     if (this._ifAvailable > 0 || this._upgrade === null || this._upgrade.refs.length === 0) return
+    if (this._hadPeers === false && this.findingPeers > 0) return
 
     // check if a peer can upgrade us
 
@@ -1047,6 +1050,7 @@ module.exports = class Replicator {
   }
 
   _addPeer (peer) {
+    this._hadPeers = true
     this.peers.push(peer)
     this.updatePeer(peer)
     this.onpeerupdate(true, peer)
@@ -1382,6 +1386,10 @@ module.exports = class Replicator {
   }
 
   _updatePeerNonPrimary (peer) {
+    if (peer.inflight >= peer.maxInflight) {
+      return false
+    }
+
     const ranges = new RandomIterator(this._ranges)
 
     for (const r of ranges) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.31",
+  "version": "10.0.0-alpha.34",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {