hypercore 10.22.2 → 10.23.0

package/lib/core.js

@@ -7,25 +7,24 @@ const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
 const Info = require('./info')
-const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = require('hypercore-errors')
-const c = require('compact-encoding')
+const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE, INVALID_CHECKSUM } = require('hypercore-errors')
 const m = require('./messages')
-const caps = require('./caps')
+const { manifestHash, createVerifier, createManifest, defaultSignerManifest, isCompat } = require('./manifest')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, bigHeader, tree, blocks, bitfield, auth, legacy, onupdate, onconflict) {
+  constructor (header, compat, crypto, oplog, bigHeader, tree, blocks, bitfield, verifier, legacy, onupdate, onconflict) {
     this.onupdate = onupdate
     this.onconflict = onconflict
     this.preupdate = null
     this.header = header
-    this.compat = !!(header.manifest && header.manifest.signer && b4a.equals(header.key, header.manifest.signer.publicKey))
+    this.compat = compat
     this.crypto = crypto
     this.oplog = oplog
     this.bigHeader = bigHeader
     this.tree = tree
     this.blocks = blocks
     this.bitfield = bitfield
-    this.defaultAuth = auth
+    this.verifier = verifier
     this.truncating = 0
     this.updating = false
     this.closed = false
@@ -54,26 +53,14 @@ module.exports = class Core {
     }
   }
 
-  static createAuth (crypto, header) {
-    const manifest = header.manifest || defaultSignerManifest(header.keyPair.publicKey)
-    const secretKey = header.keyPair && header.keyPair.secretKey
-    const publicKey = manifest.signer.publicKey
-    const sign = signable => crypto.sign(signable, secretKey)
-
-    return {
-      sign: secretKey ? sign : null,
-      verify (signable, signature) {
-        return crypto.verify(signable, signature, publicKey)
-      }
-    }
-  }
-
   static async resume (oplogFile, treeFile, bitfieldFile, dataFile, headerFile, opts) {
     let overwrite = opts.overwrite === true
 
     const force = opts.force === true
     const createIfMissing = opts.createIfMissing !== false
     const crypto = opts.crypto || hypercoreCrypto
+    // kill this flag soon
+    const legacy = !!opts.legacy
 
     const oplog = new Oplog(oplogFile, {
       headerEncoding: m.oplog.header,
@@ -81,6 +68,9 @@ module.exports = class Core {
       readonly: opts.readonly
     })
 
+    // default to true for now if no manifest is provided
+    let compat = opts.compat === true || (opts.compat !== false && !opts.manifest)
+
     let { header, entries } = await oplog.open()
 
     if (force && opts.key && header && !b4a.equals(header.key, opts.key)) {
@@ -94,18 +84,19 @@ module.exports = class Core {
         throw STORAGE_EMPTY('No Hypercore is stored here')
       }
 
-      if (opts.compat) {
+      if (compat) {
         if (opts.key && opts.keyPair && !b4a.equals(opts.key, opts.keyPair.publicKey)) {
-          throw BAD_ARGUMENT('Key must match publicKey when in compat mode.')
+          throw BAD_ARGUMENT('Key must match publicKey when in compat mode')
         }
       }
 
       const keyPair = opts.keyPair || (opts.key ? null : crypto.keyPair())
-      const manifest = opts.manifest || (opts.key && !opts.compat) ? null : defaultSignerManifest(opts.key || keyPair.publicKey)
+      const defaultManifest = !opts.manifest && (!!opts.compat || !opts.key || !!(keyPair && b4a.equals(opts.key, keyPair.publicKey)))
+      const manifest = defaultManifest ? defaultSignerManifest(opts.key || keyPair.publicKey) : createManifest(opts.manifest)
 
       header = {
         external: null,
-        key: opts.key || (opts.compat ? manifest.signer.publicKey : hashManifest(manifest)),
+        key: opts.key || (compat ? manifest.signer.publicKey : manifestHash(manifest)),
         manifest,
         keyPair,
         userData: [],
@@ -126,10 +117,22 @@ module.exports = class Core {
       header = await bigHeader.load(header.external)
     }
 
+    if (opts.manifest) {
+      // if we provide a manifest and no key, verify that the stored key is the same
+      if (!opts.key && !isValidManifest(header.key, createManifest(opts.manifest))) {
+        throw STORAGE_CONFLICT('Manifest does not hash to provided key')
+      }
+    }
+
     if (opts.key && !b4a.equals(header.key, opts.key)) {
       throw STORAGE_CONFLICT('Another Hypercore is stored here')
     }
 
+    // if we signalled compat, but already now this core isn't disable it
+    if (compat && header.manifest && !isCompat(header.key, header.manifest)) {
+      compat = false
+    }
+
     const tree = await MerkleTree.open(treeFile, { crypto, ...header.tree })
     const bitfield = await Bitfield.open(bitfieldFile, tree)
     const blocks = new BlockStore(dataFile, tree)
@@ -149,7 +152,7 @@ module.exports = class Core {
       while (bitfield.get(header.hints.contiguousLength)) header.hints.contiguousLength++
     }
 
-    const auth = opts.auth || (header.manifest ? this.createAuth(crypto, header) : null)
+    const verifier = header.manifest ? createVerifier(header.manifest, { compat: isCompat(header.key, header.manifest), crypto, legacy }) : null
 
     for (const e of entries) {
       if (e.userData) {
@@ -181,7 +184,22 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, bigHeader, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop, opts.onconflict || noop)
+    return new this(header, compat, crypto, oplog, bigHeader, tree, blocks, bitfield, verifier, legacy, opts.onupdate || noop, opts.onconflict || noop)
+  }
+
+  setManifest (manifest, keyPair) {
+    if (!manifest && b4a.equals(keyPair.publicKey, this.header.key)) manifest = defaultSignerManifest(this.header.key)
+    if (!manifest) return
+
+    const compat = isCompat(this.header.key, manifest)
+    const verifier = createVerifier(manifest, { compat, crypto: this.crypto, legacy: this._legacy })
+
+    this.compat = compat
+    this.header.manifest = manifest
+    this.verifier = verifier
+    this._manifestFlushed = false
+
+    this.onupdate(0b10000, null, null, null)
   }
 
   _shouldFlush () {
@@ -199,7 +217,7 @@ module.exports = class Core {
     return false
   }
 
-  async copyFrom (src, signature, auth = this.defaultAuth) {
+  async copyFrom (src, signature) {
     this._mutex.lock()
 
     try {
@@ -242,14 +260,16 @@ module.exports = class Core {
       this.tree.roots = [...src.tree.roots]
       this.tree.length = src.tree.length
       this.tree.byteLength = src.tree.byteLength
-      this.tree.signature = null // must provide signature
-
-      this.tree.signature = signature || auth.sign(this.tree.signable())
 
-      if (signature && !this._verifyBatchUpgrade(this.tree, null)) {
-        // TODO: how to handle signature failure?
+      try {
+        const batch = this.tree.batch()
+        batch.signature = signature
+        this._verifyBatchUpgrade(batch, this.header.manifest)
+        this.tree.signature = signature
+      } catch (err) {
         this.tree.signature = null
-        throw INVALID_SIGNATURE('Clone was provided with an invalid signature')
+        // TODO: how to handle signature failure?
+        throw err
       }
 
       this.header.tree.length = this.tree.length
@@ -263,6 +283,17 @@ module.exports = class Core {
     }
   }
 
+  async flush () {
+    await this._mutex.lock()
+    try {
+      this._manifestFlushed = true
+      this._autoFlush = 4
+      await this._flushOplog()
+    } finally {
+      this._mutex.unlock()
+    }
+  }
+
   async _flushOplog () {
     // TODO: the apis using this, actually do not need to wait for the bitfields, tree etc to flush
     // as their mutations are already stored in the oplog. We could potentially just run this in the
@@ -316,13 +347,16 @@ module.exports = class Core {
     }
   }
 
-  async truncate (length, fork, auth = this.defaultAuth) {
+  async truncate (length, fork, { signature, keyPair = this.header.keyPair } = {}) {
     this.truncating++
     await this._mutex.lock()
 
+    // upsert compat manifest
+    if (this.verifier === null && keyPair) this.setManifest(null, keyPair)
+
     try {
       const batch = await this.tree.truncate(length, fork)
-      batch.signature = await auth.sign(batch.signable(), batch)
+      batch.signature = signature || this.verifier.sign(batch, keyPair)
       await this._truncate(batch, null)
     } finally {
       this.truncating--
@@ -398,11 +432,14 @@ module.exports = class Core {
     })
   }
 
-  async append (values, auth = this.defaultAuth, hooks = {}) {
+  async append (values, { signature, keyPair = this.header.keyPair, preappend } = {}) {
     await this._mutex.lock()
 
+    // upsert compat manifest
+    if (this.verifier === null && keyPair) this.setManifest(null, keyPair)
+
     try {
-      if (hooks.preappend) await hooks.preappend(values)
+      if (preappend) await preappend(values)
 
       if (!values.length) {
         return { length: this.tree.length, byteLength: this.tree.byteLength }
@@ -411,8 +448,7 @@ module.exports = class Core {
       const batch = this.tree.batch()
       for (const val of values) batch.append(val)
 
-      const hash = batch.hash()
-      batch.signature = await auth.sign(this._legacy ? batch.signableLegacy(hash) : batch.signable(hash), batch)
+      batch.signature = signature || this.verifier.sign(batch, keyPair)
 
       const entry = {
         userData: null,
@@ -433,7 +469,7 @@ module.exports = class Core {
       batch.commit()
 
       this.header.tree.length = batch.length
-      this.header.tree.rootHash = hash
+      this.header.tree.rootHash = batch.hash()
       this.header.tree.signature = batch.signature
 
       const status = 0b0001 | updateContig(this.header, entry.bitfield, this.bitfield)
@@ -448,27 +484,25 @@ module.exports = class Core {
   }
 
   _verifyBatchUpgrade (batch, manifest) {
-    const hash = batch.hash()
-    const signable = this._legacy ? batch.signableLegacy(hash) : batch.signable(hash)
-
     if (!this.header.manifest) {
-      if (!manifest) { // compat mode, remove in future version
-        manifest = defaultSignerManifest(this.header.key)
-      } else if (!manifest || !b4a.equals(this.header.key, hashManifest(manifest))) {
+      if (!manifest && this.compat) manifest = defaultSignerManifest(this.header.key)
+
+      if (!manifest || !(isValidManifest(this.header.key, manifest) || (this.compat && isCompat(this.header.key, manifest)))) {
         throw INVALID_SIGNATURE('Proof contains an invalid manifest') // TODO: proper error type
       }
     }
 
-    const auth = this.defaultAuth || Core.createAuth(this.crypto, { ...this.header, manifest })
+    const verifier = this.verifier || createVerifier(manifest, { compat: isCompat(this.header.key, manifest), crypto: this.crypto, legacy: this._legacy })
 
-    if (!batch.signature || !auth.verify(signable, batch.signature, batch)) {
+    if (!batch.signature || !verifier.verify(batch, batch.signature)) {
       throw INVALID_SIGNATURE('Proof contains an invalid signature')
     }
 
-    this.defaultAuth = auth
     if (!this.header.manifest) {
-      this.compat = !!manifest.signer && b4a.equals(this.header.key, manifest.signer.publicKey)
+      this.compat = isCompat(this.header.key, manifest)
       this.header.manifest = manifest
+      this.verifier = verifier
+      this.onupdate(0b10000, null, null, null)
     }
   }
 
@@ -548,7 +582,7 @@ module.exports = class Core {
       await this.oplog.append(entries, false)
 
       for (let i = 0; i < verifies.length; i++) {
-        const { batch, bitfield, value, from } = verifies[i]
+        const { batch, bitfield, value, manifest, from } = verifies[i]
 
         if (!batch.commitable()) {
           verifies[i] = null // signal that we cannot commit this one
@@ -562,6 +596,12 @@ module.exports = class Core {
           status = updateContig(this.header, bitfield, this.bitfield)
         }
 
+        // if we got a manifest AND its strictly a non compat one, lets store it
+        if (manifest && this.header.manifest === null) {
+          if (!isValidManifest(this.header.key, manifest)) throw INVALID_CHECKSUM('Manifest hash does not match')
+          this.setManifest(manifest, null)
+        }
+
         batch.commit()
 
         this.onupdate(status, bitfield, value, from)
@@ -730,6 +770,10 @@ function updateContig (header, upd, bitfield) {
   return 0b1000
 }
 
+function isValidManifest (key, manifest) {
+  return b4a.equals(key, manifestHash(manifest))
+}
+
 function addReorgHint (list, tree, batch) {
   if (tree.length === 0 || tree.fork === batch.fork) return
 
@@ -788,21 +832,4 @@ async function flushHeader (oplog, bigHeader, header) {
   }
 }
 
-function defaultSignerManifest (publicKey) {
-  return {
-    hash: 'blake2b',
-    static: null,
-    signer: {
-      signature: 'ed25519',
-      namespace: caps.DEFAULT_NAMESPACE,
-      publicKey
-    },
-    multipleSigners: null
-  }
-}
-
-function hashManifest (manifest) {
-  return caps.manifestHash(c.encode(m.manifest, manifest))
-}
-
 function noop () {}

package/lib/manifest.js

@@ -0,0 +1,218 @@
+const defaultCrypto = require('hypercore-crypto')
+const b4a = require('b4a')
+const c = require('compact-encoding')
+const { BAD_ARGUMENT } = require('hypercore-errors')
+
+const m = require('./messages')
+const multisig = require('./multisig')
+const caps = require('./caps')
+
+module.exports = {
+  manifestHash,
+  isCompat,
+  defaultSignerManifest,
+  createManifest,
+  createVerifier
+}
+
+class StaticVerifier {
+  constructor (treeHash) {
+    this.treeHash = treeHash
+  }
+
+  sign () {
+    return null
+  }
+
+  verify (batch, signature) {
+    return b4a.equals(batch.hash(), this.treeHash)
+  }
+}
+
+class CompatVerifier {
+  constructor (crypto, signer, legacy) {
+    validateSigner(signer)
+
+    this.legacy = legacy
+    this.crypto = crypto
+    this.publicKey = signer.publicKey
+  }
+
+  sign (batch, keyPair) {
+    if (!keyPair || !keyPair.secretKey) throw BAD_ARGUMENT('No signer was passed')
+    return this.crypto.sign(batch.signableCompat(this.legacy), keyPair.secretKey)
+  }
+
+  verify (batch, signature) {
+    return this.crypto.verify(batch.signableCompat(this.legacy), signature, this.publicKey)
+  }
+}
+
+class SingleVerifier {
+  constructor (crypto, signer) {
+    validateSigner(signer)
+
+    this.crypto = crypto
+    this.publicKey = signer.publicKey
+    this.namespace = signer.namespace
+  }
+
+  sign (batch, keyPair) {
+    if (!keyPair || !keyPair.secretKey) throw BAD_ARGUMENT('No signer was passed')
+    return this.crypto.sign(batch.signable(this.namespace), keyPair.secretKey)
+  }
+
+  verify (batch, signature) {
+    return this.crypto.verify(batch.signable(this.namespace), signature, this.publicKey)
+  }
+}
+
+class MultiVerifier {
+  constructor (crypto, multipleSigners) {
+    this.signers = multipleSigners.signers
+    this.quorum = multipleSigners.quorum
+    this.allowPatched = multipleSigners.allowPatched
+    this.verifiers = this.signers.map(s => new SingleVerifier(crypto, s))
+
+    if (this.verifiers.length < this.quorum || (this.quorum === 0)) throw BAD_ARGUMENT('Invalid quorum')
+  }
+
+  sign () {
+    throw BAD_ARGUMENT('Multi signature must be provided')
+  }
+
+  verify (batch, signature) {
+    const inputs = multisig.inflate(signature)
+
+    if (inputs.length < this.quorum) return false
+
+    const tried = new Uint8Array(this.verifiers.length)
+
+    for (let i = 0; i < this.quorum; i++) {
+      const inp = inputs[i]
+
+      let tree = batch
+
+      if (inp.patch) {
+        if (!this.allowPatched) return false
+
+        tree = batch.clone()
+        const proof = { fork: tree.fork, block: null, hash: null, seek: null, upgrade: inp.patch, manifest: null }
+
+        try {
+          if (!tree.verifyUpgrade(proof)) return false
+        } catch {
+          return false
+        }
+      }
+
+      if (inp.signer >= this.verifiers.length || tried[inp.signer]) return false
+      tried[inp.signer] = 1
+
+      if (!this.verifiers[inp.signer].verify(tree, inp.signature)) return false
+    }
+
+    return true
+  }
+}
+
+function createVerifier (manifest, { compat = false, crypto = defaultCrypto, legacy = false } = {}) {
+  if (compat && manifest.signer) {
+    return new CompatVerifier(crypto, manifest.signer, legacy)
+  }
+
+  if (manifest.static) {
+    return new StaticVerifier(manifest.static)
+  }
+
+  if (manifest.signer) {
+    return new SingleVerifier(crypto, manifest.signer)
+  }
+
+  if (manifest.multipleSigners) {
+    return new MultiVerifier(crypto, manifest.multipleSigners)
+  }
+
+  throw BAD_ARGUMENT('No signer was provided')
+}
+
+function createManifest (inp) {
+  if (!inp) return null
+
+  const manifest = {
+    hash: 'blake2b',
+    static: null,
+    signer: null,
+    multipleSigners: null
+  }
+
+  if (inp.hash && inp.hash !== 'blake2b') throw BAD_ARGUMENT('Only Blake2b hashes are supported')
+
+  if (inp.static) {
+    if (!(b4a.isBuffer(inp.static) && inp.static.byteLength === 32)) throw BAD_ARGUMENT('Invalid static manifest')
+    manifest.static = inp.static
+    return manifest
+  }
+
+  if (inp.signer) {
+    manifest.signer = parseSigner(inp.signer)
+    return manifest
+  }
+
+  if (inp.multipleSigners) {
+    manifest.multipleSigners = parseMultipleSigners(inp.multipleSigners)
+    return manifest
+  }
+
+  throw BAD_ARGUMENT('No signer was provided')
+}
+
+function parseMultipleSigners (m) {
+  if (m.signers.length < m.quorum || !(m.quorum > 0)) throw BAD_ARGUMENT('Invalid quorum')
+
+  return {
+    allowPatched: !!m.allowPatched,
+    quorum: m.quorum,
+    signers: m.signers.map(parseSigner)
+  }
+}
+
+function parseSigner (signer) {
+  validateSigner(signer)
+  return {
+    signature: 'ed25519',
+    namespace: signer.namespace || caps.DEFAULT_NAMESPACE,
+    publicKey: signer.publicKey
+  }
+}
+
+function validateSigner (signer) {
+  if (!signer || !signer.publicKey) throw BAD_ARGUMENT('Signer missing public key')
+  if (signer.signature !== 'ed25519') throw BAD_ARGUMENT('Only Ed25519 signatures are supported')
+}
+
+function defaultSignerManifest (publicKey) {
+  return {
+    hash: 'blake2b',
+    static: null,
+    signer: {
+      signature: 'ed25519',
+      namespace: caps.DEFAULT_NAMESPACE,
+      publicKey
+    },
+    multipleSigners: null
+  }
+}
+
+function manifestHash (manifest) {
+  const state = { start: 0, end: 32, buffer: null }
+  m.manifest.preencode(state, manifest)
+  state.buffer = b4a.allocUnsafe(state.end)
+  c.raw.encode(state, caps.MANIFEST)
+  m.manifest.encode(state, manifest)
+  return defaultCrypto.hash(state.buffer)
+}
+
+function isCompat (key, manifest) {
+  return !!(manifest && manifest.signer && b4a.equals(key, manifest.signer.publicKey))
+}

package/lib/merkle-tree.js

@@ -79,12 +79,12 @@ class MerkleTreeBatch {
     return this.hashCached
   }
 
-  signable (hash = this.hash()) {
-    return caps.treeSignable(hash, this.length, this.fork)
+  signable (namespace) {
+    return caps.treeSignable(namespace, this.hash(), this.length, this.fork)
   }
 
-  signableLegacy (hash = this.hash()) {
-    return caps.treeSignableLegacy(hash, this.length, this.fork)
+  signableCompat (noHeader) {
+    return caps.treeSignableCompat(this.hash(), this.length, this.fork, noHeader)
   }
 
   get (index) {
@@ -425,8 +425,8 @@ module.exports = class MerkleTree {
     return this.crypto.tree(this.roots)
   }
 
-  signable (hash = this.hash()) {
-    return caps.treeSignable(hash, this.length, this.fork)
+  signable (namespace) {
+    return caps.treeSignable(namespace, this.hash(), this.length, this.fork)
   }
 
   getRoots (length) {