hypercore 10.31.12 → 10.32.1

package/lib/messages.js

@@ -65,76 +65,120 @@ const signer = {
 
 const signerArray = c.array(signer)
 
-const multipleSigners = {
-  preencode (state, m) {
-    state.end++ // flags
-    c.uint.preencode(state, m.quorum)
-    signerArray.preencode(state, m.signers)
+const prologue = {
+  preencode (state, p) {
+    c.fixed32.preencode(state, p.hash)
+    c.uint.preencode(state, p.length)
   },
-  encode (state, m) {
-    c.uint.encode(state, m.allowPatched ? 1 : 0)
-    c.uint.encode(state, m.quorum)
-    signerArray.encode(state, m.signers)
+  encode (state, p) {
+    c.fixed32.encode(state, p.hash)
+    c.uint.encode(state, p.length)
   },
   decode (state) {
-    const flags = c.uint.decode(state)
     return {
-      allowPatched: (flags & 1) !== 0,
-      quorum: c.uint.decode(state),
-      signers: signerArray.decode(state)
+      hash: c.fixed32.decode(state),
+      length: c.uint.decode(state)
     }
   }
 }
 
-const manifest = exports.manifest = {
+const manifestv0 = {
   preencode (state, m) {
-    c.uint.preencode(state, 0) // version
     hashes.preencode(state, m.hash)
-    c.uint.preencode(state, 2) // type
-
-    if (m.static) {
-      c.fixed32.preencode(state, m.static)
-    }
-
-    if (m.signer) {
-      signer.preencode(state, m.signer)
-    }
-
-    if (m.multipleSigners) {
-      multipleSigners.preencode(state, m.multipleSigners)
+    state.end++ // type
+
+    if (m.quorum === 1 && m.signers.length === 1 && !m.allowPatch) {
+      signer.preencode(state, m.signers[0])
+    } else {
+      state.end++ // flags
+      c.uint.preencode(state, m.quorum)
+      signerArray.preencode(state, m.signers)
     }
   },
   encode (state, m) {
-    c.uint.encode(state, 0) // version
     hashes.encode(state, m.hash)
-    c.uint.encode(state, m.signer ? 1 : m.multipleSigners ? 2 : 0)
 
-    if (m.static) {
-      c.fixed32.encode(state, m.static)
+    if (m.quorum === 1 && m.signers.length === 1 && !m.allowPatch) {
+      c.uint.encode(state, 1)
+      signer.encode(state, m.signers[0])
+    } else {
+      c.uint.encode(state, 2)
+      c.uint.encode(state, m.allowPatched ? 1 : 0)
+      c.uint.encode(state, m.quorum)
+      signerArray.encode(state, m.signers)
     }
+  },
+  decode (state) {
+    const hash = hashes.decode(state)
+    const type = c.uint.decode(state)
+
+    if (type === 0) throw new Error('Type 0 is deprecated')
+    if (type > 2) throw new Error('Unknown type: ' + type)
 
-    if (m.signer) {
-      signer.encode(state, m.signer)
+    if (type === 1) {
+      return {
+        version: 0,
+        hash,
+        allowPatch: false,
+        quorum: 1,
+        signers: [signer.decode(state)],
+        prologue: null
+      }
     }
 
-    if (m.multipleSigners) {
-      multipleSigners.encode(state, m.multipleSigners)
+    const flags = c.uint.decode(state)
+
+    return {
+      version: 0,
+      hash,
+      allowPatch: (flags & 1) !== 0,
+      quorum: c.uint.decode(state),
+      signers: signerArray.decode(state),
+      prologue: null
     }
+  }
+}
+
+const manifest = exports.manifest = {
+  preencode (state, m) {
+    state.end++ // version
+    if (m.version === 0) return manifestv0.preencode(state, m)
+
+    state.end++ // flags
+    hashes.preencode(state, m.hash)
+
+    c.uint.preencode(state, m.quorum)
+    signerArray.preencode(state, m.signers)
+    if (m.prologue) prologue.preencode(state, m.prologue)
+  },
+  encode (state, m) {
+    c.uint.encode(state, m.version)
+    if (m.version === 0) return manifestv0.encode(state, m)
+
+    c.uint.encode(state, (m.allowPatch ? 1 : 0) | (m.prologue ? 2 : 0))
+    hashes.encode(state, m.hash)
+
+    c.uint.encode(state, m.quorum)
+    signerArray.encode(state, m.signers)
+    if (m.prologue) prologue.encode(state, m.prologue)
   },
   decode (state) {
-    const version = c.uint.decode(state)
-    if (version !== 0) throw new Error('Invalid version: ' + version)
+    const v = c.uint.decode(state)
+    if (v === 0) return manifestv0.decode(state)
+    if (v !== 1) throw new Error('Unknown version: ' + v)
 
+    const flags = c.uint.decode(state)
     const hash = hashes.decode(state)
-    const type = c.uint.decode(state)
-
-    if (type > 2) throw new Error('Unknown type: ' + type)
+    const quorum = c.uint.decode(state)
+    const signers = signerArray.decode(state)
 
     return {
+      version: 1,
       hash,
-      static: type === 0 ? c.fixed32.decode(state) : null,
-      signer: type === 1 ? signer.decode(state) : null,
-      multipleSigners: type === 2 ? multipleSigners.decode(state) : null
+      allowPatch: (flags & 1) !== 0,
+      quorum,
+      signers,
+      prologue: (flags & 2) === 0 ? null : prologue.decode(state)
     }
   }
 }
@@ -831,14 +875,16 @@ oplog.header = {
         external: null,
         key: old.signer.publicKey,
         manifest: {
+          version: 0,
           hash: old.types.tree,
-          static: null,
-          signer: {
+          allowPatch: false,
+          quorum: 1,
+          signers: [{
             signature: old.types.signer,
             namespace: DEFAULT_NAMESPACE,
             publicKey: old.signer.publicKey
-          },
-          multipleSigners: null
+          }],
+          prologue: null
         },
         keyPair: old.signer.secretKey ? old.signer : null,
         userData: old.userData,
@@ -875,7 +921,27 @@ oplog.header = {
 
 const uintArray = c.array(c.uint)
 
-const patchEncoding = {
+const multisigInput = {
+  preencode (state, inp) {
+    c.uint.preencode(state, inp.signer)
+    c.fixed64.preencode(state, inp.signature)
+    c.uint.preencode(state, inp.patch)
+  },
+  encode (state, inp) {
+    c.uint.encode(state, inp.signer)
+    c.fixed64.encode(state, inp.signature)
+    c.uint.encode(state, inp.patch)
+  },
+  decode (state) {
+    return {
+      signer: c.uint.decode(state),
+      signature: c.fixed64.decode(state),
+      patch: c.uint.decode(state)
+    }
+  }
+}
+
+const patchEncodingv0 = {
   preencode (state, n) {
     c.uint.preencode(state, n.start)
     c.uint.preencode(state, n.length)
@@ -895,29 +961,30 @@ const patchEncoding = {
   }
 }
 
-const multisigInput = {
+const multisigInputv0 = {
   preencode (state, n) {
     state.end++
     c.uint.preencode(state, n.signer)
     c.fixed64.preencode(state, n.signature)
-    if (n.patch) patchEncoding.preencode(state, n.patch)
+    if (n.patch) patchEncodingv0.preencode(state, n.patch)
   },
   encode (state, n) {
     c.uint.encode(state, n.patch ? 1 : 0)
     c.uint.encode(state, n.signer)
     c.fixed64.encode(state, n.signature)
-    if (n.patch) patchEncoding.encode(state, n.patch)
+    if (n.patch) patchEncodingv0.encode(state, n.patch)
   },
   decode (state) {
     const flags = c.uint.decode(state)
     return {
       signer: c.uint.decode(state),
       signature: c.fixed64.decode(state),
-      patch: (flags & 1) ? patchEncoding.decode(state) : null
+      patch: (flags & 1) ? patchEncodingv0.decode(state) : null
     }
   }
 }
 
+const multisigInputArrayv0 = c.array(multisigInputv0)
 const multisigInputArray = c.array(multisigInput)
 
 const compactNode = {
@@ -942,19 +1009,36 @@ const compactNode = {
 
 const compactNodeArray = c.array(compactNode)
 
+exports.multiSignaturev0 = {
+  preencode (state, s) {
+    multisigInputArrayv0.preencode(state, s.proofs)
+    compactNodeArray.preencode(state, s.patch)
+  },
+  encode (state, s) {
+    multisigInputArrayv0.encode(state, s.proofs)
+    compactNodeArray.encode(state, s.patch)
+  },
+  decode (state) {
+    return {
+      proofs: multisigInputArrayv0.decode(state),
+      patch: compactNodeArray.decode(state)
+    }
+  }
+}
+
 exports.multiSignature = {
   preencode (state, s) {
     multisigInputArray.preencode(state, s.proofs)
-    compactNodeArray.preencode(state, s.nodes)
+    compactNodeArray.preencode(state, s.patch)
   },
   encode (state, s) {
     multisigInputArray.encode(state, s.proofs)
-    compactNodeArray.encode(state, s.nodes)
+    compactNodeArray.encode(state, s.patch)
   },
   decode (state) {
     return {
       proofs: multisigInputArray.decode(state),
-      nodes: compactNodeArray.decode(state)
+      patch: compactNodeArray.decode(state)
     }
   }
 }

package/lib/multisig.js

@@ -1,45 +1,42 @@
 const c = require('compact-encoding')
 const b4a = require('b4a')
-const encoding = require('./messages').multiSignature
+const flat = require('flat-tree')
+const { multiSignature, multiSignaturev0 } = require('./messages')
 
 module.exports = {
+  assemblev0,
   assemble,
+  inflatev0,
   inflate,
   partialSignature,
   signableLength
 }
 
-function inflate (data) {
-  const compressedInputs = c.decode(encoding, data)
-  const inputs = []
-
-  for (const proof of compressedInputs.proofs) {
-    inputs.push({
-      signer: proof.signer,
-      signature: proof.signature,
-      patch: inflateUpgrade(proof.patch, compressedInputs.nodes)
-    })
-  }
+function inflatev0 (data) {
+  return c.decode(multiSignaturev0, data)
+}
 
-  return inputs
+function inflate (data) {
+  return c.decode(multiSignature, data)
 }
 
 async function partialSignature (tree, signer, from, to = tree.length, signature = tree.signature) {
   if (from > tree.length) return null
-  const patch = to <= from ? null : await upgrade(tree, from, to)
+  const nodes = to <= from ? null : await upgradeNodes(tree, from, to)
+
+  if (signature.byteLength !== 64) signature = c.decode(multiSignature, signature).proofs[0].signature
 
   return {
     signer,
     signature,
-    patch
+    patch: nodes ? to - from : 0,
+    nodes
   }
 }
 
-async function upgrade (tree, from, to) {
+async function upgradeNodes (tree, from, to) {
   const p = await tree.proof({ upgrade: { start: from, length: to - from } })
-  p.upgrade.additionalNodes = []
-  p.upgrade.signature = null
-  return p.upgrade
+  return p.upgrade.nodes
 }
 
 function signableLength (lengths, quorum) {
@@ -53,15 +50,39 @@ function cmp (a, b) {
   return b - a
 }
 
+function assemblev0 (inputs) {
+  const proofs = []
+  const patch = []
+
+  for (const u of inputs) {
+    proofs.push(compressProof(u, patch))
+  }
+
+  return c.encode(multiSignaturev0, { proofs, patch })
+}
+
 function assemble (inputs) {
   const proofs = []
-  const nodes = []
+  const patch = []
+  const seen = new Set()
 
   for (const u of inputs) {
-    proofs.push(compressProof(u, nodes))
+    if (u.nodes) {
+      for (const node of u.nodes) {
+        if (seen.has(node.index)) continue
+        seen.add(node.index)
+        patch.push(node)
+      }
+    }
+
+    proofs.push({
+      signer: u.signer,
+      signature: u.signature,
+      patch: u.patch
+    })
   }
 
-  return c.encode(encoding, { proofs, nodes })
+  return c.encode(multiSignature, { proofs, patch })
 }
 
 function compareNode (a, b) {
@@ -74,16 +95,14 @@ function compressProof (proof, nodes) {
   return {
     signer: proof.signer,
     signature: proof.signature,
-    patch: compressUpgrade(proof.patch, nodes)
+    patch: proof.patch ? compressUpgrade(proof, nodes) : null
   }
 }
 
 function compressUpgrade (p, nodes) {
-  if (!p) return null
-
   const u = {
-    start: p.start,
-    length: p.length,
+    start: flat.rightSpan(p.nodes[p.nodes.length - 1].index) / 2 + 1,
+    length: p.patch,
     nodes: []
   }
 
@@ -103,21 +122,3 @@ function compressUpgrade (p, nodes) {
 
   return u
 }
-
-function inflateUpgrade (s, nodes) {
-  if (!s) return null
-
-  const upgrade = {
-    start: s.start,
-    length: s.length,
-    nodes: [],
-    additionalNodes: [],
-    signature: null
-  }
-
-  for (const i of s.nodes) {
-    upgrade.nodes.push(nodes[i])
-  }
-
-  return upgrade
-}

package/lib/replicator.js

@@ -7,6 +7,7 @@ const RemoteBitfield = require('./remote-bitfield')
 const { REQUEST_CANCELLED, REQUEST_TIMEOUT, INVALID_CAPABILITY, SNAPSHOT_NOT_AVAILABLE } = require('hypercore-errors')
 const m = require('./messages')
 const caps = require('./caps')
+const { createTracer } = require('hypertrace')
 
 const DEFAULT_MAX_INFLIGHT = [32, 512]
 const SCALE_LATENCY = 50
@@ -268,6 +269,7 @@ class BlockTracker {
 
 class Peer {
   constructor (replicator, protomux, channel, session) {
+    this.tracer = createTracer(this, { parent: replicator.core.tracer })
     this.core = replicator.core
     this.replicator = replicator
     this.stream = protomux.stream
@@ -429,6 +431,8 @@ class Peer {
   }
 
   onclose (isRemote) {
+    this.tracer.trace('onclose')
+
     // we might have signalled to the remote that we are done (ie not downloading) and the remote might agree on that
     // if that happens, the channel might be closed by the remote. if so just renegotiate it.
     // TODO: add a CLOSE_REASON to mux to we can make this cleaner...
@@ -564,6 +568,8 @@ class Peer {
   }
 
   async onrequest (msg) {
+    this.tracer.trace('onrequest', msg)
+
     if (!this.protomux.drained || this.receiverQueue.length) {
       this.receiverQueue.push(msg)
       return
@@ -661,6 +667,8 @@ class Peer {
   }
 
   async ondata (data) {
+    this.tracer.trace('ondata', data)
+
     // always allow a fork conflict proof to be sent
     if (data.request === 0 && data.upgrade && data.upgrade.start === 0) {
       if (await this.core.checkConflict(data, this)) return
@@ -720,6 +728,8 @@ class Peer {
   }
 
   onnodata ({ request }) {
+    this.tracer.trace('onnodata', { request })
+
     const req = request > 0 ? this.replicator._inflight.get(request) : null
 
     if (req === null || req.peer !== this) return
@@ -1190,6 +1200,8 @@ class Peer {
       return
     }
 
+    this.tracer.trace('send', req)
+
     this.wireRequest.send(req)
   }
 }
@@ -1203,6 +1215,7 @@ module.exports = class Replicator {
     onupload = noop,
     oninvalid = noop
   } = {}) {
+    this.tracer = createTracer(this)
     this.key = key
     this.discoveryKey = core.crypto.discoveryKey(key)
     this.core = core