hypercore 10.22.3 → 10.24.0

package/lib/messages.js

@@ -169,13 +169,12 @@ wire.handshake = {
     c.fixed32.preencode(state, m.capability)
   },
   encode (state, m) {
-    c.uint.encode(state, m.manifest ? 1 : 0) // flags for the future
+    c.uint.encode(state, 0) // flags
     c.fixed32.encode(state, m.capability)
   },
   decode (state) {
-    const flags = c.uint.decode(state)
+    c.uint.decode(state) // flags
     return {
-      manifest: (flags & 1) !== 0,
       capability: c.fixed32.decode(state)
     }
   }
@@ -499,7 +498,7 @@ wire.sync = {
     c.uint.preencode(state, m.remoteLength)
   },
   encode (state, m) {
-    c.uint.encode(state, (m.canUpgrade ? 1 : 0) | (m.uploading ? 2 : 0) | (m.downloading ? 4 : 0))
+    c.uint.encode(state, (m.canUpgrade ? 1 : 0) | (m.uploading ? 2 : 0) | (m.downloading ? 4 : 0) | (m.hasManifest ? 8 : 0))
     c.uint.encode(state, m.fork)
     c.uint.encode(state, m.length)
     c.uint.encode(state, m.remoteLength)
@@ -513,7 +512,8 @@ wire.sync = {
       remoteLength: c.uint.decode(state),
       canUpgrade: (flags & 1) !== 0,
       uploading: (flags & 2) !== 0,
-      downloading: (flags & 4) !== 0
+      downloading: (flags & 4) !== 0,
+      hasManifest: (flags & 8) !== 0
     }
   }
 }
@@ -868,3 +868,89 @@ oplog.header = {
     }
   }
 }
+
+const uintArray = c.array(c.uint)
+
+const patchEncoding = {
+  preencode (state, n) {
+    c.uint.preencode(state, n.start)
+    c.uint.preencode(state, n.length)
+    uintArray.preencode(state, n.nodes)
+  },
+  encode (state, n) {
+    c.uint.encode(state, n.start)
+    c.uint.encode(state, n.length)
+    uintArray.encode(state, n.nodes)
+  },
+  decode (state) {
+    return {
+      start: c.uint.decode(state),
+      length: c.uint.decode(state),
+      nodes: uintArray.decode(state)
+    }
+  }
+}
+
+const multisigInput = {
+  preencode (state, n) {
+    state.end++
+    c.uint.preencode(state, n.signer)
+    c.fixed64.preencode(state, n.signature)
+    if (n.patch) patchEncoding.preencode(state, n.patch)
+  },
+  encode (state, n) {
+    c.uint.encode(state, n.patch ? 1 : 0)
+    c.uint.encode(state, n.signer)
+    c.fixed64.encode(state, n.signature)
+    if (n.patch) patchEncoding.encode(state, n.patch)
+  },
+  decode (state) {
+    const flags = c.uint.decode(state)
+    return {
+      signer: c.uint.decode(state),
+      signature: c.fixed64.decode(state),
+      patch: (flags & 1) ? patchEncoding.decode(state) : null
+    }
+  }
+}
+
+const multisigInputArray = c.array(multisigInput)
+
+const compactNode = {
+  preencode (state, n) {
+    c.uint.preencode(state, n.index)
+    c.uint.preencode(state, n.size)
+    c.fixed32.preencode(state, n.hash)
+  },
+  encode (state, n) {
+    c.uint.encode(state, n.index)
+    c.uint.encode(state, n.size)
+    c.fixed32.encode(state, n.hash)
+  },
+  decode (state) {
+    return {
+      index: c.uint.decode(state),
+      size: c.uint.decode(state),
+      hash: c.fixed32.decode(state)
+    }
+  }
+}
+
+const compactNodeArray = c.array(compactNode)
+
+exports.multiSignature = {
+  preencode (state, s) {
+    multisigInputArray.preencode(state, s.proofs)
+    compactNodeArray.preencode(state, s.nodes)
+  },
+  encode (state, s) {
+    multisigInputArray.encode(state, s.proofs)
+    compactNodeArray.encode(state, s.nodes)
+  },
+  decode (state) {
+    return {
+      proofs: multisigInputArray.decode(state),
+      nodes: compactNodeArray.decode(state)
+    }
+  }
+}

package/lib/multisig.js

@@ -0,0 +1,123 @@
+const c = require('compact-encoding')
+const b4a = require('b4a')
+const encoding = require('./messages').multiSignature
+
+module.exports = {
+  assemble,
+  inflate,
+  partialSignature,
+  signableLength
+}
+
+function inflate (data) {
+  const compressedInputs = c.decode(encoding, data)
+  const inputs = []
+
+  for (const proof of compressedInputs.proofs) {
+    inputs.push({
+      signer: proof.signer,
+      signature: proof.signature,
+      patch: inflateUpgrade(proof.patch, compressedInputs.nodes)
+    })
+  }
+
+  return inputs
+}
+
+async function partialSignature (tree, signer, from, to = tree.length, signature = tree.signature) {
+  if (from > tree.length) return null
+  const patch = to <= from ? null : await upgrade(tree, from, to)
+
+  return {
+    signer,
+    signature,
+    patch
+  }
+}
+
+async function upgrade (tree, from, to) {
+  const p = await tree.proof({ upgrade: { start: from, length: to - from } })
+  p.upgrade.additionalNodes = []
+  p.upgrade.signature = null
+  return p.upgrade
+}
+
+function signableLength (lengths, quorum) {
+  if (quorum <= 0) quorum = 1
+  if (quorum > lengths.length) return 0
+
+  return lengths.sort(cmp)[quorum - 1]
+}
+
+function cmp (a, b) {
+  return b - a
+}
+
+function assemble (inputs) {
+  const proofs = []
+  const nodes = []
+
+  for (const u of inputs) {
+    proofs.push(compressProof(u, nodes))
+  }
+
+  return c.encode(encoding, { proofs, nodes })
+}
+
+function compareNode (a, b) {
+  if (a.index !== b.index) return false
+  if (a.size !== b.size) return false
+  return b4a.equals(a.hash, b.hash)
+}
+
+function compressProof (proof, nodes) {
+  return {
+    signer: proof.signer,
+    signature: proof.signature,
+    patch: compressUpgrade(proof.patch, nodes)
+  }
+}
+
+function compressUpgrade (p, nodes) {
+  if (!p) return null
+
+  const u = {
+    start: p.start,
+    length: p.length,
+    nodes: []
+  }
+
+  for (const node of p.nodes) {
+    let present = false
+    for (let i = 0; i < nodes.length; i++) {
+      if (!compareNode(nodes[i], node)) continue
+
+      u.nodes.push(i)
+      present = true
+      break
+    }
+
+    if (present) continue
+    u.nodes.push(nodes.push(node) - 1)
+  }
+
+  return u
+}
+
+function inflateUpgrade (s, nodes) {
+  if (!s) return null
+
+  const upgrade = {
+    start: s.start,
+    length: s.length,
+    nodes: [],
+    additionalNodes: [],
+    signature: null
+  }
+
+  for (const i of s.nodes) {
+    upgrade.nodes.push(nodes[i])
+  }
+
+  return upgrade
+}

package/lib/replicator.js

@@ -314,7 +314,7 @@ class Peer {
     this.remoteUploading = true
     this.remoteDownloading = true
     this.remoteSynced = false
-    this.remoteManifest = false
+    this.remoteHasManifest = false
 
     this.segmentsWanted = new Set()
     this.broadcastedNonSparse = false
@@ -384,11 +384,12 @@ class Peer {
       remoteLength: this.core.tree.fork === this.remoteFork ? this.remoteLength : 0,
       canUpgrade: this.canUpgrade,
       uploading: true,
-      downloading: this.replicator.downloading
+      downloading: this.replicator.downloading,
+      hasManifest: !!this.core.header.manifest && this.core.compat === false
     })
   }
 
-  onopen ({ manifest, capability }) {
+  onopen ({ capability }) {
     const expected = caps.replicate(this.stream.isInitiator === false, this.replicator.key, this.stream.handshakeHash)
 
     if (b4a.equals(capability, expected) !== true) { // TODO: change this to a rejection instead, less leakage
@@ -397,7 +398,6 @@ class Peer {
 
     if (this.remoteOpened === true) return
     this.remoteOpened = true
-    this.remoteManifest = manifest
 
     this.protomux.cork()
 
@@ -440,7 +440,7 @@ class Peer {
     }
   }
 
-  async onsync ({ fork, length, remoteLength, canUpgrade, uploading, downloading }) {
+  async onsync ({ fork, length, remoteLength, canUpgrade, uploading, downloading, hasManifest }) {
     const lengthChanged = length !== this.remoteLength
     const sameFork = fork === this.core.tree.fork
 
@@ -450,6 +450,7 @@ class Peer {
     this.remoteCanUpgrade = canUpgrade
     this.remoteUploading = uploading
     this.remoteDownloading = downloading
+    this.remoteHasManifest = hasManifest
 
     if (this.remoteDownloading === false && this.replicator.downloading === false) {
       // idling, shut it down...
@@ -587,25 +588,29 @@ class Peer {
       }
     }
 
-    if (proof !== null) {
-      if (proof.block !== null) {
-        this.replicator.onupload(proof.block.index, proof.block.value, this)
+    if (proof === null) {
+      if (msg.manifest && this.core.header.manifest) {
+        const manifest = this.core.header.manifest
+        this.wireData.send({ request: msg.id, fork: this.core.tree.fork, block: null, hash: null, seek: null, upgrade: null, manifest })
+        return
       }
 
-      this.wireData.send({
-        request: msg.id,
-        fork: msg.fork,
-        block: proof.block,
-        hash: proof.hash,
-        seek: proof.seek,
-        upgrade: proof.upgrade,
-        manifest: proof.manifest
-      })
+      this.wireNoData.send({ request: msg.id })
       return
     }
 
-    this.wireNoData.send({
-      request: msg.id
+    if (proof.block !== null) {
+      this.replicator.onupload(proof.block.index, proof.block.value, this)
+    }
+
+    this.wireData.send({
+      request: msg.id,
+      fork: msg.fork,
+      block: proof.block,
+      hash: proof.hash,
+      seek: proof.seek,
+      upgrade: proof.upgrade,
+      manifest: proof.manifest
     })
   }
 
@@ -777,11 +782,16 @@ class Peer {
         ? null
         : { start: this.core.tree.length, length: this.remoteLength - this.core.tree.length },
       // remote manifest check can be removed eventually...
-      manifest: needsUpgrade && !this.core.header.manifest && this.remoteManifest && !this.core.compat,
+      manifest: this.core.header.manifest === null && this.remoteHasManifest === true,
       priority
     }
   }
 
+  _requestManifest () {
+    const req = this._makeRequest(false, 0)
+    this._send(req)
+  }
+
   _requestUpgrade (u) {
     const req = this._makeRequest(true, 0)
     if (req === null) return false
@@ -1052,6 +1062,7 @@ module.exports = class Replicator {
     this._ifAvailable = 0
     this._updatesPending = 0
     this._applyingReorg = null
+    this._manifestPeer = null
 
     const self = this
     this._onstreamclose = onstreamclose
@@ -1311,6 +1322,8 @@ module.exports = class Replicator {
   _removePeer (peer) {
     this.peers.splice(this.peers.indexOf(peer), 1)
 
+    if (this._manifestPeer === peer) this._manifestPeer = null
+
     for (const req of this._inflight) {
       if (req.peer !== peer) continue
       this._inflight.remove(req.id)
@@ -1544,6 +1557,10 @@ module.exports = class Replicator {
       this._clearInflightReorgs(req)
     }
 
+    if (this._manifestPeer === peer && this.core.header.manifest !== null) {
+      this._manifestPeer = null
+    }
+
     if (this._seeks.length > 0 || this._ranges.length > 0) this._updateNonPrimary()
     else this.updatePeer(peer)
   }
@@ -1629,6 +1646,10 @@ module.exports = class Replicator {
     return this._upgrade !== null && this._upgrade.inflight.length === 0
   }
 
+  _maybeRequestManifest () {
+    return this.core.header.manifest === null && this._manifestPeer === null
+  }
+
   _updateFork (peer) {
     if (this._applyingReorg !== null || this.allowFork === false || peer.remoteFork <= this.core.tree.fork) {
       return false
@@ -1649,6 +1670,12 @@ module.exports = class Replicator {
       return false
     }
 
+    // Eagerly request the manifest even if the remote length is 0. If not 0 we'll get as part of the upgrade request...
+    if (this._maybeRequestManifest() === true && peer.remoteLength === 0 && peer.remoteHasManifest === true) {
+      this._manifestPeer = peer
+      peer._requestManifest()
+    }
+
     for (const s of this._seeks) {
       if (s.inflight.length > 0) continue // TODO: one per peer is better
       if (peer._requestSeek(s) === true) {
@@ -1812,7 +1839,6 @@ module.exports = class Replicator {
     const stream = protomux.stream
 
     peer.channel.open({
-      manifest: true,
       capability: caps.replicate(stream.isInitiator, this.key, stream.handshakeHash)
     })
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.22.3",
+  "version": "10.24.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -45,6 +45,7 @@
     "flat-tree": "^1.9.0",
     "hypercore-crypto": "^3.2.1",
     "hypercore-errors": "^1.1.0",
+    "hypercore-id-encoding": "^1.2.0",
     "is-options": "^1.0.1",
     "protomux": "^3.5.0",
     "quickbit-universal": "^2.1.1",