hypercore 10.20.1 → 10.21.0

package/index.js

@@ -243,6 +243,11 @@ module.exports = class Hypercore extends EventEmitter {
     return s
   }
 
+  setKeyPair (keyPair) {
+    this.auth = Core.createAuth(this.crypto, { keyPair })
+    this.writable = !this._readonly && !!this.auth && !!this.auth.sign
+  }
+
   _passCapabilities (o) {
     if (!this.auth) this.auth = o.auth
 
@@ -252,7 +257,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.core = o.core
     this.replicator = o.replicator
     this.encryption = o.encryption
-    this.writable = !this._readonly && !!(this.auth && this.auth.sign)
+    this.writable = !this._readonly && !!this.auth && !!this.auth.sign
     this.autoClose = o.autoClose
 
     if (this.snapshotted && this.core && !this._snapshot) this._updateSnapshot()
@@ -282,26 +287,16 @@ module.exports = class Hypercore extends EventEmitter {
     if (!isFirst) await opts._opening
     if (opts.preload) opts = { ...opts, ...(await this._retryPreload(opts.preload)) }
 
-    const keyPair = (key && opts.keyPair)
-      ? { ...opts.keyPair, publicKey: key }
-      : key
-        ? { publicKey: key, secretKey: null }
-        : opts.keyPair
-
-    // This only works if the hypercore was fully loaded,
-    // but we only do this to validate the keypair to help catch bugs so yolo
-    if (this.key && keyPair) keyPair.publicKey = this.key
+    const keyPair = opts.keyPair
 
     if (opts.auth) {
       this.auth = opts.auth
-    } else if (opts.sign) {
-      this.auth = Core.createAuth(this.crypto, keyPair, opts)
     } else if (keyPair && keyPair.secretKey) {
-      this.auth = Core.createAuth(this.crypto, keyPair)
+      this.setKeyPair(keyPair)
     }
 
     if (isFirst) {
-      await this._openCapabilities(keyPair, storage, opts)
+      await this._openCapabilities(key, storage, opts)
       // Only the root session should pass capabilities to other sessions.
       for (let i = 0; i < this.sessions.length; i++) {
         const s = this.sessions[i]
@@ -318,7 +313,7 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     if (!this.auth) this.auth = this.core.defaultAuth
-    this.writable = !this._readonly && !!this.auth.sign
+    this.writable = !this._readonly && !!this.auth && !!this.auth.sign
 
     if (opts.valueEncoding) {
       this.valueEncoding = c.from(opts.valueEncoding)
@@ -350,18 +345,20 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
-  async _openCapabilities (keyPair, storage, opts) {
+  async _openCapabilities (key, storage, opts) {
     if (opts.from) return this._openFromExisting(opts.from, opts)
 
     const unlocked = !!opts.unlocked
     this.storage = Hypercore.defaultStorage(opts.storage || storage, { unlocked, writable: !unlocked })
 
     this.core = await Core.open(this.storage, {
+      compat: opts.compat !== false, // default to true for now
       force: opts.force,
       createIfMissing: opts.createIfMissing,
       readonly: unlocked,
       overwrite: opts.overwrite,
-      keyPair,
+      key,
+      keyPair: opts.keyPair,
       crypto: this.crypto,
       legacy: opts.legacy,
       auth: opts.auth,
@@ -375,8 +372,8 @@ module.exports = class Hypercore extends EventEmitter {
       }
     }
 
-    this.key = this.core.header.signer.publicKey
-    this.keyPair = this.core.header.signer
+    this.key = this.core.header.key
+    this.keyPair = this.core.header.keyPair
     this.id = z32.encode(this.key)
 
     this.replicator = new Replicator(this.core, this.key, {
@@ -404,10 +401,10 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     return {
-      length: this.core.header.contiguousLength,
+      length: this.core.header.hints.contiguousLength,
       byteLength: 0,
       fork: this.core.tree.fork,
-      compatLength: this.core.header.contiguousLength
+      compatLength: this.core.header.hints.contiguousLength
     }
   }
 
@@ -479,13 +476,8 @@ module.exports = class Hypercore extends EventEmitter {
     let auth = this.core.defaultAuth
     if (opts.auth) {
       auth = opts.auth
-    } else if (opts.sign && keyPair) {
-      auth = Core.createAuth(this.crypto, keyPair, opts)
-    } else if (opts.sign) {
-      // TODO: dangerous to just update sign?
-      auth.sign = opts.sign
     } else if (keyPair && keyPair.secretKey) {
-      auth = Core.createAuth(this.crypto, keyPair)
+      auth = Core.createAuth(this.crypto, { keyPair, manifest: { signer: { publicKey: keyPair.publicKey } } })
     }
 
     const upgrade = opts.upgrade === undefined ? null : opts.upgrade
@@ -496,8 +488,10 @@ module.exports = class Hypercore extends EventEmitter {
     const timeout = opts.timeout === undefined ? this.timeout : opts.timeout
 
     const Clz = this.constructor
+
     return new Clz(storage, key, {
       ...opts,
+      keyPair,
       sparse,
       wait,
       onwait,
@@ -569,7 +563,7 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   get contiguousLength () {
-    return this.core === null ? 0 : this.core.header.contiguousLength
+    return this.core === null ? 0 : this.core.header.hints.contiguousLength
   }
 
   get contiguousByteLength () {
@@ -658,7 +652,7 @@ module.exports = class Hypercore extends EventEmitter {
         }
       }
 
-      const contig = this.core.header.contiguousLength
+      const contig = this.core.header.hints.contiguousLength
 
       // When the contig length catches up, broadcast the non-sparse length to peers
       if (appendedNonSparse && contig === this.core.tree.length) {

package/lib/big-header.js

@@ -0,0 +1,55 @@
+const c = require('compact-encoding')
+const { oplog } = require('./messages')
+
+module.exports = class BigHeader {
+  constructor (storage) {
+    this.storage = storage
+  }
+
+  async load (external) {
+    const buf = await new Promise((resolve, reject) => {
+      this.storage.read(external.start, external.length, (err, buf) => {
+        if (err) return reject(err)
+        resolve(buf)
+      })
+    })
+
+    const header = c.decode(oplog.header, buf)
+    header.external = external
+    return header
+  }
+
+  async flush (header) {
+    const external = header.external || { start: 0, length: 0 }
+    header.external = null
+
+    const buf = c.encode(oplog.header, header)
+
+    let start = 0
+    if (buf.byteLength > external.start) {
+      start = external.start + external.length
+      const rem = start & 4095
+      if (rem > 0) start += (4096 - rem)
+    }
+
+    header.external = { start, length: buf.byteLength }
+
+    await new Promise((resolve, reject) => {
+      this.storage.write(start, buf, (err) => {
+        if (err) return reject(err)
+        resolve()
+      })
+    })
+
+    return header
+  }
+
+  close () {
+    return new Promise((resolve, reject) => {
+      this.storage.close((err) => {
+        if (err) return reject(err)
+        resolve()
+      })
+    })
+  }
+}

package/lib/caps.js

@@ -2,11 +2,14 @@ const crypto = require('hypercore-crypto')
 const sodium = require('sodium-universal')
 const b4a = require('b4a')
 const c = require('compact-encoding')
+const m = require('./messages')
 
 // TODO: rename this to "crypto" and move everything hashing related etc in here
 // Also lets move the tree stuff from hypercore-crypto here
 
-const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER] = crypto.namespace('hypercore', 3)
+const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER, MANIFEST, DEFAULT_NAMESPACE] = crypto.namespace('hypercore', 5)
+
+exports.DEFAULT_NAMESPACE = DEFAULT_NAMESPACE
 
 exports.replicate = function (isInitiator, key, handshakeHash) {
   const out = b4a.allocUnsafe(32)
@@ -14,6 +17,17 @@ exports.replicate = function (isInitiator, key, handshakeHash) {
   return out
 }
 
+exports.manifestHash = function (manifest) {
+  const state = { start: 0, end: 32, buffer: null }
+  m.manifest.preencode(state, manifest)
+  state.buffer = b4a.allocUnsafe(state.end)
+  c.raw.encode(state, MANIFEST)
+  m.manifest.encode(state, manifest)
+  const out = b4a.allocUnsafe(32)
+  sodium.crypto_generichash(out, state.buffer)
+  return out
+}
+
 exports.treeSignable = function (hash, length, fork) {
   const state = { start: 0, end: 80, buffer: b4a.allocUnsafe(80) }
   c.raw.encode(state, TREE)

package/lib/core.js

@@ -1,6 +1,7 @@
 const hypercoreCrypto = require('hypercore-crypto')
 const b4a = require('b4a')
 const Oplog = require('./oplog')
+const BigHeader = require('./big-header')
 const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
@@ -8,15 +9,18 @@ const Bitfield = require('./bitfield')
 const Info = require('./info')
 const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = require('hypercore-errors')
 const m = require('./messages')
+const caps = require('./caps')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate, onconflict) {
+  constructor (header, crypto, oplog, bigHeader, tree, blocks, bitfield, auth, legacy, onupdate, onconflict) {
     this.onupdate = onupdate
     this.onconflict = onconflict
     this.preupdate = null
     this.header = header
+    this.compat = !!(header.manifest && header.manifest.signer && b4a.equals(header.key, header.manifest.signer.publicKey))
     this.crypto = crypto
     this.oplog = oplog
+    this.bigHeader = bigHeader
     this.tree = tree
     this.blocks = blocks
     this.bitfield = bitfield
@@ -24,6 +28,7 @@ module.exports = class Core {
     this.truncating = 0
     this.closed = false
 
+    this._manifestFlushed = !!header.manifest
     this._maxOplogSize = 65536
     this._autoFlush = 1
     this._verifies = null
@@ -37,35 +42,31 @@ module.exports = class Core {
     const treeFile = storage('tree')
     const bitfieldFile = storage('bitfield')
     const dataFile = storage('data')
+    const headerFile = storage('header')
 
     try {
-      return await this.resume(oplogFile, treeFile, bitfieldFile, dataFile, opts)
+      return await this.resume(oplogFile, treeFile, bitfieldFile, dataFile, headerFile, opts)
     } catch (err) {
-      await closeAll(oplogFile, treeFile, bitfieldFile, dataFile)
+      await closeAll(oplogFile, treeFile, bitfieldFile, dataFile, headerFile)
       throw err
     }
   }
 
-  static createAuth (crypto, { publicKey, secretKey }, opts = {}) {
-    if (secretKey && !crypto.validateKeyPair({ publicKey, secretKey })) {
-      throw BAD_ARGUMENT('Invalid key pair')
-    }
-
-    const sign = opts.sign
-      ? opts.sign
-      : secretKey
-        ? (signable) => crypto.sign(signable, secretKey)
-        : undefined
+  static createAuth (crypto, header) {
+    const manifest = header.manifest || defaultSignerManifest(header.keyPair.publicKey)
+    const secretKey = header.keyPair && header.keyPair.secretKey
+    const publicKey = manifest.signer.publicKey
+    const sign = signable => crypto.sign(signable, secretKey)
 
     return {
-      sign,
+      sign: secretKey ? sign : null,
       verify (signable, signature) {
         return crypto.verify(signable, signature, publicKey)
       }
     }
   }
 
-  static async resume (oplogFile, treeFile, bitfieldFile, dataFile, opts) {
+  static async resume (oplogFile, treeFile, bitfieldFile, dataFile, headerFile, opts) {
     let overwrite = opts.overwrite === true
 
     const force = opts.force === true
@@ -80,17 +81,31 @@ module.exports = class Core {
 
     let { header, entries } = await oplog.open()
 
-    if (force && opts.keyPair && header && header.signer && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
+    if (force && opts.key && header && !b4a.equals(header.key, opts.key)) {
       overwrite = true
     }
 
+    const bigHeader = new BigHeader(headerFile)
+
     if (!header || overwrite) {
       if (!createIfMissing) {
         throw STORAGE_EMPTY('No Hypercore is stored here')
       }
 
+      if (opts.compat) {
+        if (opts.key && opts.keyPair && !b4a.equals(opts.key, opts.keyPair.publicKey)) {
+          throw BAD_ARGUMENT('Key must match publicKey when in compat mode.')
+        }
+      }
+
+      const keyPair = opts.keyPair || (opts.key ? null : crypto.keyPair())
+      const manifest = opts.manifest || (opts.key && !opts.compat) ? null : defaultSignerManifest(opts.key || keyPair.publicKey)
+
       header = {
-        types: { tree: 'blake2b', bitfield: 'raw', signer: 'ed25519' },
+        external: null,
+        key: opts.key || (opts.compat ? manifest.signer.publicKey : caps.manifestHash(manifest)),
+        manifest,
+        keyPair,
         userData: [],
         tree: {
           fork: 0,
@@ -98,17 +113,18 @@ module.exports = class Core {
           rootHash: null,
           signature: null
         },
-        signer: opts.keyPair || crypto.keyPair(),
         hints: {
-          reorgs: []
-        },
-        contiguousLength: 0
+          reorgs: [],
+          contiguousLength: 0
+        }
       }
 
-      await oplog.flush(header)
+      await flushHeader(oplog, bigHeader, header)
+    } else if (header.external) {
+      header = await bigHeader.load(header.external)
     }
 
-    if (opts.keyPair && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
+    if (opts.key && !b4a.equals(header.key, opts.key)) {
       throw STORAGE_CONFLICT('Another Hypercore is stored here')
     }
 
@@ -127,11 +143,11 @@ module.exports = class Core {
     }
 
     // compat from earlier version that do not store contig length
-    if (header.contiguousLength === 0) {
-      while (bitfield.get(header.contiguousLength)) header.contiguousLength++
+    if (header.hints.contiguousLength === 0) {
+      while (bitfield.get(header.hints.contiguousLength)) header.hints.contiguousLength++
     }
 
-    const auth = opts.auth || this.createAuth(crypto, header.signer)
+    const auth = opts.auth || (header.manifest ? this.createAuth(crypto, header) : null)
 
     for (const e of entries) {
       if (e.userData) {
@@ -163,7 +179,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop, opts.onconflict || noop)
+    return new this(header, crypto, oplog, bigHeader, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop, opts.onconflict || noop)
   }
 
   _shouldFlush () {
@@ -173,6 +189,11 @@ module.exports = class Core {
       return true
     }
 
+    if (!this._manifestFlushed && this.header.manifest) {
+      this._manifestFlushed = true
+      return true
+    }
+
     return false
   }
 
@@ -223,7 +244,7 @@ module.exports = class Core {
 
       this.tree.signature = signature || auth.sign(this.tree.signable())
 
-      if (signature && !this._signed(this.tree)) {
+      if (signature && !this._verifyBatchUpgrade(this.tree, null)) {
         // TODO: how to handle signature failure?
         this.tree.signature = null
         throw INVALID_SIGNATURE('Clone was provided with an invalid signature')
@@ -246,7 +267,8 @@ module.exports = class Core {
     // background. Might be easier to impl that where it is called instead and keep this one simple.
     await this.bitfield.flush()
     await this.tree.flush()
-    await this.oplog.flush(this.header)
+
+    return flushHeader(this.oplog, this.bigHeader, this.header)
   }
 
   _appendBlocks (values) {
@@ -325,8 +347,8 @@ module.exports = class Core {
 
       this.bitfield.setRange(start, end - start, false)
 
-      if (start < this.header.contiguousLength) {
-        this.header.contiguousLength = start
+      if (start < this.header.hints.contiguousLength) {
+        this.header.hints.contiguousLength = start
       }
 
       start = this.bitfield.lastSet(start) + 1
@@ -423,18 +445,34 @@ module.exports = class Core {
     }
   }
 
-  _signed (batch, hash, auth = this.defaultAuth) {
+  _verifyBatchUpgrade (batch, manifest) {
+    const hash = batch.hash()
     const signable = this._legacy ? batch.signableLegacy(hash) : batch.signable(hash)
-    return auth.verify(signable, batch.signature, batch)
-  }
 
-  async _verifyExclusive ({ batch, bitfield, value, from }) {
-    // TODO: move this to tree.js
-    const hash = batch.hash()
-    if (!batch.signature || !this._signed(batch, hash)) {
+    if (!this.header.manifest) {
+      if (!manifest) { // compat mode, remove in future version
+        manifest = defaultSignerManifest(this.header.key)
+      } else if (!manifest || !b4a.equals(this.header.key, caps.manifestHash(manifest))) {
+        throw INVALID_SIGNATURE('Proof contains an invalid manifest') // TODO: proper error type
+      }
+    }
+
+    const auth = this.defaultAuth || Core.createAuth(this.crypto, { ...this.header, manifest })
+
+    if (!batch.signature || !auth.verify(signable, batch.signature, batch)) {
       throw INVALID_SIGNATURE('Proof contains an invalid signature')
     }
 
+    this.defaultAuth = auth
+    if (!this.header.manifest) {
+      this.compat = !!manifest.signer && b4a.equals(this.header.key, manifest.signer.publicKey)
+      this.header.manifest = manifest
+    }
+  }
+
+  async _verifyExclusive ({ batch, bitfield, value, manifest, from }) {
+    this._verifyBatchUpgrade(batch, manifest)
+
     await this._mutex.lock()
 
     try {
@@ -447,7 +485,7 @@ module.exports = class Core {
         bitfield
       }
 
-      if (this.preupdate !== null) await this.preupdate(batch, this.header.signer.publicKey)
+      if (this.preupdate !== null) await this.preupdate(batch, this.header.key)
       if (bitfield) await this._writeBlock(batch, bitfield.start, value)
 
       await this.oplog.append([entry], false)
@@ -463,7 +501,7 @@ module.exports = class Core {
 
       this.header.tree.fork = batch.fork
       this.header.tree.length = batch.length
-      this.header.tree.rootHash = batch.rootHash
+      this.header.tree.rootHash = batch.hash()
       this.header.tree.signature = batch.signature
 
       this.onupdate(status, bitfield, value, from)
@@ -541,9 +579,7 @@ module.exports = class Core {
 
     const batch = this.tree.verifyFullyRemote(proof)
 
-    if (!batch.signature || !this._signed(batch, batch.hash())) {
-      throw INVALID_SIGNATURE('Proof contains an invalid signature with no input from us')
-    }
+    this._verifyBatchUpgrade(batch, proof.manifest)
 
     const remoteTreeHash = this.crypto.tree(proof.upgrade.nodes)
     const localTreeHash = this.crypto.tree(await this.tree.getRoots(proof.upgrade.length))
@@ -554,11 +590,18 @@ module.exports = class Core {
     return true
   }
 
+  async verifyReorg (proof) {
+    const batch = await this.tree.reorg(proof)
+
+    this._verifyBatchUpgrade(batch, proof.manifest)
+
+    return batch
+  }
+
   async verify (proof, from) {
     // We cannot apply "other forks" atm.
     // We should probably still try and they are likely super similar for non upgrades
     // but this is easy atm (and the above layer will just retry)
-
     if (proof.fork !== this.tree.fork) return false
 
     const batch = await this.tree.verify(proof)
@@ -569,6 +612,7 @@ module.exports = class Core {
       batch,
       bitfield: value && { drop: false, start: proof.block.index, length: 1 },
       value,
+      manifest: proof.manifest,
       from
     }
 
@@ -646,7 +690,8 @@ module.exports = class Core {
       this.oplog.close(),
       this.bitfield.close(),
       this.tree.close(),
-      this.blocks.close()
+      this.blocks.close(),
+      this.bigHeader.close()
     ])
   }
 }
@@ -654,7 +699,7 @@ module.exports = class Core {
 function updateContig (header, upd, bitfield) {
   const end = upd.start + upd.length
 
-  let c = header.contiguousLength
+  let c = header.hints.contiguousLength
 
   if (upd.drop) {
     // If we dropped a block in the current contig range, "downgrade" it
@@ -668,16 +713,16 @@ function updateContig (header, upd, bitfield) {
     }
   }
 
-  if (c === header.contiguousLength) {
+  if (c === header.hints.contiguousLength) {
     return 0b0000
   }
 
-  if (c > header.contiguousLength) {
-    header.contiguousLength = c
+  if (c > header.hints.contiguousLength) {
+    header.hints.contiguousLength = c
     return 0b0100
   }
 
-  header.contiguousLength = c
+  header.hints.contiguousLength = c
   return 0b1000
 }
 
@@ -725,4 +770,31 @@ function closeAll (...storages) {
   })
 }
 
+async function flushHeader (oplog, bigHeader, header) {
+  if (header.external) {
+    await bigHeader.flush(header)
+  }
+
+  try {
+    await oplog.flush(header)
+  } catch (err) {
+    if (err.code !== 'OPLOG_HEADER_OVERFLOW') throw err
+    await bigHeader.flush(header)
+    await oplog.flush(header)
+  }
+}
+
+function defaultSignerManifest (publicKey) {
+  return {
+    hash: 'blake2b',
+    static: null,
+    signer: {
+      signature: 'ed25519',
+      namespace: caps.DEFAULT_NAMESPACE,
+      publicKey
+    },
+    multipleSigners: null
+  }
+}
+
 function noop () {}

package/lib/merkle-tree.js

@@ -48,6 +48,7 @@ class MerkleTreeBatch {
     this.ancestors = tree.length
     this.byteLength = tree.byteLength
     this.signature = null
+    this.hashCached = null
 
     this.treeLength = tree.length
     this.treeFork = tree.fork
@@ -74,7 +75,8 @@ class MerkleTreeBatch {
   }
 
   hash () {
-    return this.tree.crypto.tree(this.roots)
+    if (this.hashCached === null) this.hashCached = this.tree.crypto.tree(this.roots)
+    return this.hashCached
   }
 
   signable (hash = this.hash()) {
@@ -122,6 +124,7 @@ class MerkleTreeBatch {
   }
 
   appendRoot (node, ite) {
+    this.hashCached = null
     this.upgraded = true
     this.length += ite.factor / 2
     this.byteLength += node.size
@@ -1215,7 +1218,7 @@ async function generateProof (tree, block, hash, seek, upgrade) {
   }
 
   const [pNode, pSeek, pUpgrade, pAdditional] = await settleProof(p)
-  const result = { fork, block: null, hash: null, seek: null, upgrade: null }
+  const result = { fork, block: null, hash: null, seek: null, upgrade: null, manifest: null }
 
   if (block) {
     result.block = {

package/lib/messages.js

@@ -4,6 +4,140 @@ const { INVALID_OPLOG_VERSION } = require('hypercore-errors')
 
 const EMPTY = b4a.alloc(0)
 
+const hashes = {
+  preencode (state, m) {
+    state.end++ // small uint
+  },
+  encode (state, m) {
+    if (m === 'blake2b') {
+      c.uint.encode(state, 0)
+      return
+    }
+
+    throw new Error('Unknown hash: ' + m)
+  },
+  decode (state) {
+    const n = c.uint.decode(state)
+    if (n === 0) return 'blake2b'
+    throw new Error('Unknown hash id: ' + n)
+  }
+}
+
+const signatures = {
+  preencode (state, m) {
+    state.end++ // small uint
+  },
+  encode (state, m) {
+    if (m === 'ed25519') {
+      c.uint.encode(state, 0)
+      return
+    }
+
+    throw new Error('Unknown signature: ' + m)
+  },
+  decode (state) {
+    const n = c.uint.decode(state)
+    if (n === 0) return 'ed25519'
+    throw new Error('Unknown signature id: ' + n)
+  }
+}
+
+const signer = {
+  preencode (state, m) {
+    signatures.preencode(state, m.signature)
+    c.fixed32.preencode(state, m.namespace)
+    c.fixed32.preencode(state, m.publicKey)
+  },
+  encode (state, m) {
+    signatures.encode(state, m.signature)
+    c.fixed32.encode(state, m.namespace)
+    c.fixed32.encode(state, m.publicKey)
+  },
+  decode (state) {
+    return {
+      signature: signatures.decode(state),
+      namespace: c.fixed32.decode(state),
+      publicKey: c.fixed32.decode(state)
+    }
+  }
+}
+
+const signerArray = c.array(signer)
+
+const multipleSigners = {
+  preencode (state, m) {
+    state.end++ // flags
+    c.uint.preencode(state, m.quorum)
+    signerArray.preencode(state, m.signers)
+  },
+  encode (state, m) {
+    c.uint.encode(state, m.allowPatched ? 1 : 0)
+    c.uint.encode(state, m.quorum)
+    signerArray.encode(state, m.signers)
+  },
+  decode (state) {
+    const flags = c.uint.decode(state)
+    return {
+      allowPatched: (flags & 1) !== 0,
+      quorum: c.uint.decode(state),
+      signers: signerArray.decode(state)
+    }
+  }
+}
+
+const manifest = exports.manifest = {
+  preencode (state, m) {
+    c.uint.preencode(state, 0) // version
+    hashes.preencode(state, m.hash)
+    c.uint.preencode(state, 2) // type
+
+    if (m.static) {
+      c.fixed32.preencode(state, m.static)
+    }
+
+    if (m.signer) {
+      signer.preencode(state, m.signer)
+    }
+
+    if (m.multipleSigners) {
+      multipleSigners.preencode(state, m.multipleSigners)
+    }
+  },
+  encode (state, m) {
+    c.uint.encode(state, 0) // version
+    hashes.encode(state, m.hash)
+    c.uint.encode(state, m.signer ? 1 : m.multipleSigners ? 2 : 0)
+
+    if (m.static) {
+      c.fixed32.encode(state, m.static)
+    }
+
+    if (m.signer) {
+      signer.encode(state, m.signer)
+    }
+
+    if (m.multipleSigners) {
+      multipleSigners.encode(state, m.multipleSigners)
+    }
+  },
+  decode (state) {
+    const version = c.uint.decode(state)
+    if (version !== 0) throw new Error('Invalid version: ' + version)
+
+    const hash = hashes.decode(state)
+    const type = c.uint.decode(state)
+
+    if (type > 2) throw new Error('Unknown type: ' + type)
+
+    return {
+      hash,
+      static: type === 0 ? c.fixed32.decode(state) : null,
+      signer: type === 1 ? signer.decode(state) : null,
+      multipleSigners: type === 2 ? multipleSigners.decode(state) : null
+    }
+  }
+}
+
 const node = {
   preencode (state, n) {
     c.uint.preencode(state, n.index)
@@ -30,16 +164,17 @@ const wire = exports.wire = {}
 
 wire.handshake = {
   preencode (state, m) {
-    c.uint.preencode(state, 0) // flags for the future
+    c.uint.preencode(state, 0)
     c.fixed32.preencode(state, m.capability)
   },
   encode (state, m) {
-    c.uint.encode(state, 0) // flags for the future
+    c.uint.encode(state, m.manifest ? 1 : 0) // flags for the future
     c.fixed32.encode(state, m.capability)
   },
   decode (state) {
-    c.uint.decode(state) // flags for the future
+    const flags = c.uint.decode(state)
     return {
+      manifest: (flags & 1) !== 0,
       capability: c.fixed32.decode(state)
     }
   }
@@ -106,7 +241,7 @@ wire.request = {
     if (m.priority) c.uint.preencode(state, m.priority)
   },
   encode (state, m) {
-    const flags = (m.block ? 1 : 0) | (m.hash ? 2 : 0) | (m.seek ? 4 : 0) | (m.upgrade ? 8 : 0) | (m.priority ? 16 : 0)
+    const flags = (m.block ? 1 : 0) | (m.hash ? 2 : 0) | (m.seek ? 4 : 0) | (m.upgrade ? 8 : 0) | (m.manifest ? 16 : 0) | (m.priority ? 32 : 0)
 
     c.uint.encode(state, flags)
     c.uint.encode(state, m.id)
@@ -128,7 +263,8 @@ wire.request = {
       hash: flags & 2 ? requestBlock.decode(state) : null,
       seek: flags & 4 ? requestSeek.decode(state) : null,
       upgrade: flags & 8 ? requestUpgrade.decode(state) : null,
-      priority: flags & 16 ? c.uint.decode(state) : 0
+      manifest: (flags & 16) !== 0,
+      priority: flags & 32 ? c.uint.decode(state) : 0
     }
   }
 }
@@ -237,9 +373,10 @@ wire.data = {
     if (m.hash) dataHash.preencode(state, m.hash)
     if (m.seek) dataSeek.preencode(state, m.seek)
     if (m.upgrade) dataUpgrade.preencode(state, m.upgrade)
+    if (m.manifest) manifest.preencode(state, m.manifest)
   },
   encode (state, m) {
-    const flags = (m.block ? 1 : 0) | (m.hash ? 2 : 0) | (m.seek ? 4 : 0) | (m.upgrade ? 8 : 0)
+    const flags = (m.block ? 1 : 0) | (m.hash ? 2 : 0) | (m.seek ? 4 : 0) | (m.upgrade ? 8 : 0) | (m.manifest ? 16 : 0)
 
     c.uint.encode(state, flags)
     c.uint.encode(state, m.request)
@@ -249,6 +386,7 @@ wire.data = {
     if (m.hash) dataHash.encode(state, m.hash)
     if (m.seek) dataSeek.encode(state, m.seek)
     if (m.upgrade) dataUpgrade.encode(state, m.upgrade)
+    if (m.manifest) manifest.encode(state, m.manifest)
   },
   decode (state) {
     const flags = c.uint.decode(state)
@@ -259,7 +397,8 @@ wire.data = {
       block: flags & 1 ? dataBlock.decode(state) : null,
       hash: flags & 2 ? dataHash.decode(state) : null,
       seek: flags & 4 ? dataSeek.decode(state) : null,
-      upgrade: flags & 8 ? dataUpgrade.decode(state) : null
+      upgrade: flags & 8 ? dataUpgrade.decode(state) : null,
+      manifest: flags & 16 ? manifest.decode(state) : null
     }
   }
 }
@@ -562,13 +701,16 @@ const reorgHintArray = c.array(reorgHint)
 const hints = {
   preencode (state, h) {
     reorgHintArray.preencode(state, h.reorgs)
+    c.uint.preencode(state, h.contiguousLength)
   },
   encode (state, h) {
     reorgHintArray.encode(state, h.reorgs)
+    c.uint.encode(state, h.contiguousLength)
   },
   decode (state) {
     return {
-      reorgs: reorgHintArray.decode(state)
+      reorgs: reorgHintArray.decode(state),
+      contiguousLength: state.start < state.end ? c.uint.decode(state) : 0
     }
   }
 }
@@ -616,41 +758,112 @@ const types = {
   }
 }
 
+const externalHeader = {
+  preencode (state, m) {
+    c.uint.preencode(state, m.start)
+    c.uint.preencode(state, m.length)
+  },
+  encode (state, m) {
+    c.uint.encode(state, m.start)
+    c.uint.encode(state, m.length)
+  },
+  decode (state) {
+    return {
+      start: c.uint.decode(state),
+      length: c.uint.decode(state)
+    }
+  }
+}
+
 const keyValueArray = c.array(keyValue)
 
 oplog.header = {
   preencode (state, h) {
-    state.end += 1 // version
-    types.preencode(state, h.types)
+    state.end += 2 // version + flags
+    if (h.external) {
+      externalHeader.preencode(state, h.external)
+      return
+    }
+    c.fixed32.preencode(state, h.key)
+    if (h.manifest) manifest.preencode(state, h.manifest)
+    if (h.keyPair) keyPair.preencode(state, h.keyPair)
     keyValueArray.preencode(state, h.userData)
     treeHeader.preencode(state, h.tree)
-    keyPair.preencode(state, h.signer)
     hints.preencode(state, h.hints)
-    c.uint.preencode(state, h.contiguousLength)
   },
   encode (state, h) {
-    state.buffer[state.start++] = 0 // version
-    types.encode(state, h.types)
+    c.uint.encode(state, 1)
+    if (h.external) {
+      c.uint.encode(state, 1) // ONLY set the first big for clarity
+      externalHeader.encode(state, h.external)
+      return
+    }
+    c.uint.encode(state, (h.manifest ? 2 : 0) | (h.keyPair ? 4 : 0))
+    c.fixed32.encode(state, h.key)
+    if (h.manifest) manifest.encode(state, h.manifest)
+    if (h.keyPair) keyPair.encode(state, h.keyPair)
     keyValueArray.encode(state, h.userData)
     treeHeader.encode(state, h.tree)
-    keyPair.encode(state, h.signer)
     hints.encode(state, h.hints)
-    c.uint.encode(state, h.contiguousLength)
   },
   decode (state) {
     const version = c.uint.decode(state)
 
-    if (version !== 0) {
-      throw INVALID_OPLOG_VERSION('Invalid header version. Expected 0, got ' + version)
+    if (version > 1) {
+      throw INVALID_OPLOG_VERSION('Invalid header version. Expected <= 1, got ' + version)
+    }
+
+    if (version === 0) {
+      const old = {
+        types: types.decode(state),
+        userData: keyValueArray.decode(state),
+        tree: treeHeader.decode(state),
+        signer: keyPair.decode(state),
+        hints: hints.decode(state)
+      }
+
+      return {
+        key: old.signer.publicKey,
+        manifest: {
+          namespace: b4a.alloc(32),
+          hash: old.types.tree,
+          static: null,
+          signer: {
+            signature: old.types.signer,
+            entropy: b4a.alloc(32),
+            publicKey: old.signer.publicKey
+          },
+          multipleSigners: null
+        },
+        keyPair: old.signer.secretKey ? old.signer : null,
+        userData: old.userData,
+        tree: old.tree,
+        hints: old.hints
+      }
+    }
+
+    const flags = c.uint.decode(state)
+
+    if (flags & 1) {
+      return {
+        external: externalHeader.decode(state),
+        key: null,
+        manifest: null,
+        keyPair: null,
+        userData: null,
+        tree: null,
+        hints: null
+      }
     }
 
     return {
-      types: types.decode(state),
+      external: null,
+      key: c.fixed32.decode(state),
+      manifest: (flags & 2) !== 0 ? manifest.decode(state) : null,
+      keyPair: (flags & 4) !== 0 ? keyPair.decode(state) : null,
       userData: keyValueArray.decode(state),
       tree: treeHeader.decode(state),
-      signer: keyPair.decode(state),
-      hints: hints.decode(state),
-      contiguousLength: state.end > state.start ? c.uint.decode(state) : 0
+      hints: hints.decode(state)
     }
   }
 }

package/lib/oplog.js

@@ -1,7 +1,7 @@
 const cenc = require('compact-encoding')
 const b4a = require('b4a')
 const { crc32 } = require('crc-universal')
-const { OPLOG_CORRUPT } = require('hypercore-errors')
+const { OPLOG_CORRUPT, OPLOG_HEADER_OVERFLOW } = require('hypercore-errors')
 
 module.exports = class Oplog {
   constructor (storage, { pageSize = 4096, headerEncoding = cenc.raw, entryEncoding = cenc.raw, readonly = false } = {}) {
@@ -155,6 +155,7 @@ module.exports = class Oplog {
     const bit = (this._headers[i] + 1) & 1
 
     this.headerEncoding.preencode(state, header)
+    if (state.end > this._pageSize) throw OPLOG_HEADER_OVERFLOW()
     state.buffer = b4a.allocUnsafe(state.end)
     this.headerEncoding.encode(state, header)
     this._addHeader(state, state.end - 8, bit, 0)

package/lib/replicator.js

@@ -313,6 +313,7 @@ class Peer {
     this.remoteUploading = true
     this.remoteDownloading = true
     this.remoteSynced = false
+    this.remoteManifest = false
 
     this.segmentsWanted = new Set()
     this.broadcastedNonSparse = false
@@ -385,7 +386,7 @@ class Peer {
     })
   }
 
-  onopen ({ capability }) {
+  onopen ({ manifest, capability }) {
     const expected = caps.replicate(this.stream.isInitiator === false, this.replicator.key, this.stream.handshakeHash)
 
     if (b4a.equals(capability, expected) !== true) { // TODO: change this to a rejection instead, less leakage
@@ -394,6 +395,7 @@ class Peer {
 
     if (this.remoteOpened === true) return
     this.remoteOpened = true
+    this.remoteManifest = manifest
 
     this.protomux.cork()
 
@@ -519,6 +521,10 @@ class Peer {
       proof.block.value = await this.core.blocks.get(index)
     }
 
+    if (msg.manifest && !this.core.compat) {
+      proof.manifest = this.core.header.manifest
+    }
+
     return proof
   }
 
@@ -575,7 +581,8 @@ class Peer {
         block: proof.block,
         hash: proof.hash,
         seek: proof.seek,
-        upgrade: proof.upgrade
+        upgrade: proof.upgrade,
+        manifest: proof.manifest
       })
       return
     }
@@ -752,12 +759,14 @@ class Peer {
       upgrade: needsUpgrade === false
         ? null
         : { start: this.core.tree.length, length: this.remoteLength - this.core.tree.length },
+      // remote manifest check can be removed eventually...
+      manifest: needsUpgrade && !this.core.header.manifest && this.remoteManifest && !this.core.compat,
       priority
     }
   }
 
   _requestUpgrade (u) {
-    const req = this._makeRequest(true)
+    const req = this._makeRequest(true, 0)
     if (req === null) return false
 
     this._send(req)
@@ -771,7 +780,7 @@ class Peer {
     if (fork !== this.remoteFork) return false
 
     if (s.seeker.start >= length) {
-      const req = this._makeRequest(true)
+      const req = this._makeRequest(true, 0)
 
       // We need an upgrade for the seek, if non can be provided, skip
       if (req === null) return false
@@ -913,9 +922,10 @@ class Peer {
   }
 
   _requestForkProof (f) {
-    const req = this._makeRequest(false)
+    const req = this._makeRequest(false, 0)
 
     req.upgrade = { start: 0, length: this.remoteLength }
+    req.manifest = !this.core.header.manifest
 
     f.inflight.push(req)
     this._send(req)
@@ -936,7 +946,7 @@ class Peer {
 
       if (this.remoteBitfield.get(index) === false) continue
 
-      const req = this._makeRequest(false)
+      const req = this._makeRequest(false, 0)
 
       req.hash = { index: 2 * index, nodes: f.batch.want.nodes }
 
@@ -1526,6 +1536,7 @@ module.exports = class Replicator {
   }
 
   async _onreorgdata (peer, req, data) {
+    const newBatch = data.upgrade && await this.core.verifyReorg(data)
     const f = this._addReorg(data.fork, peer)
 
     if (f === null) {
@@ -1538,7 +1549,7 @@ module.exports = class Replicator {
     if (f.batch) {
       await f.batch.update(data)
     } else if (data.upgrade) {
-      f.batch = await this.core.tree.reorg(data)
+      f.batch = newBatch
 
       // Remove "older" reorgs in progress as we just verified this one.
       this._clearOldReorgs(f.fork)
@@ -1769,6 +1780,7 @@ module.exports = class Replicator {
     const stream = protomux.stream
 
     peer.channel.open({
+      manifest: true,
       capability: caps.replicate(stream.isInitiator, this.key, stream.handshakeHash)
     })
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.20.1",
+  "version": "10.21.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -44,7 +44,7 @@
     "fast-fifo": "^1.3.0",
     "flat-tree": "^1.9.0",
     "hypercore-crypto": "^3.2.1",
-    "hypercore-errors": "^1.0.0",
+    "hypercore-errors": "^1.1.0",
     "is-options": "^1.0.1",
     "protomux": "^3.5.0",
     "quickbit-universal": "^2.1.1",