hypercore 10.31.12 → 10.32.0

package/index.js

@@ -9,6 +9,7 @@ const NoiseSecretStream = require('@hyperswarm/secret-stream')
 const Protomux = require('protomux')
 const z32 = require('z32')
 const id = require('hypercore-id-encoding')
+const { createTracer } = require('hypertrace')
 
 const Replicator = require('./lib/replicator')
 const Core = require('./lib/core')
@@ -16,7 +17,7 @@ const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
 const Batch = require('./lib/batch')
-const { manifestHash, defaultSignerManifest, createVerifier, createManifest, isCompat } = require('./lib/manifest')
+const { manifestHash, defaultSignerManifest, createManifest, isCompat, sign } = require('./lib/verifier')
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const {
   BAD_ARGUMENT,
@@ -48,6 +49,7 @@ module.exports = class Hypercore extends EventEmitter {
 
     this[promises] = true
 
+    this.tracer = createTracer(this)
     this.storage = null
     this.crypto = opts.crypto || hypercoreCrypto
     this.core = null
@@ -133,7 +135,7 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   static key (manifest, { compat } = {}) {
-    return compat ? manifest.signer.publicKey : manifestHash(createManifest(manifest))
+    return compat ? manifest.signers[0].publicKey : manifestHash(createManifest(manifest))
   }
 
   static discoveryKey (key) {
@@ -266,6 +268,8 @@ module.exports = class Hypercore extends EventEmitter {
     this.writable = this._isWritable()
     this.autoClose = o.autoClose
 
+    if (o.core) this.tracer.setParent(o.core.tracer)
+
     if (this.snapshotted && this.core && !this._snapshot) this._updateSnapshot()
   }
 
@@ -376,6 +380,7 @@ module.exports = class Hypercore extends EventEmitter {
       onupdate: this._oncoreupdate.bind(this),
       onconflict: this._oncoreconflict.bind(this)
     })
+    this.tracer.setParent(this.core.tracer)
 
     if (opts.userData) {
       for (const [key, value] of Object.entries(opts.userData)) {
@@ -489,14 +494,14 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     const manifest = opts.manifest || defaultSignerManifest(keyPair.publicKey)
-    const key = opts.key || (opts.compat !== false ? manifest.signer.publicKey : manifestHash(manifest))
+    const key = opts.key || (opts.compat !== false ? manifest.signers[0].publicKey : manifestHash(manifest))
 
     if (b4a.equals(key, this.key)) {
       throw BAD_ARGUMENT('Clone cannot share verification information')
     }
 
     const signature = opts.signature === undefined
-      ? createVerifier(createManifest(manifest), { compat: isCompat(key, manifest) }).sign(this.core.tree.batch(), keyPair)
+      ? sign(createManifest(manifest), this.core.tree.batch(), keyPair, { compat: isCompat(key, manifest) })
       : opts.signature
 
     const sparse = opts.sparse === false ? false : this.sparse
@@ -847,6 +852,9 @@ module.exports = class Hypercore extends EventEmitter {
 
   async get (index, opts) {
     if (this.opened === false) await this.opening
+
+    this.tracer.trace('get', { index })
+
     if (this.closing !== null) throw SESSION_CLOSED()
     if (this._snapshot !== null && index >= this._snapshot.compatLength) throw SNAPSHOT_NOT_AVAILABLE()
 
@@ -961,6 +969,8 @@ module.exports = class Hypercore extends EventEmitter {
   async _download (range) {
     if (this.opened === false) await this.opening
 
+    this.tracer.trace('download', { range })
+
     const activeRequests = (range && range.activeRequests) || this.activeRequests
 
     return this.replicator.addRange(activeRequests, range)
@@ -1003,6 +1013,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (writable === false) throw SESSION_NOT_WRITABLE()
 
     blocks = Array.isArray(blocks) ? blocks : [blocks]
+    this.tracer.trace('append', { blocks })
 
     const preappend = this.encryption && this._preappend
 

package/lib/batch.js

@@ -187,9 +187,9 @@ module.exports = class HypercoreBatch extends EventEmitter {
     }
   }
 
-  async restoreBatch (length) {
+  async restoreBatch (length, blocks) {
     if (this.opened === false) await this.opening
-    if (length >= this._sessionLength) return this.createTreeBatch(length)
+    if (length >= this._sessionLength) return this.createTreeBatch(length, blocks)
     return this.session.core.tree.restoreBatch(length)
   }
 

package/lib/core.js

@@ -7,12 +7,14 @@ const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
 const Info = require('./info')
-const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE, INVALID_CHECKSUM } = require('hypercore-errors')
+const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_OPERATION, INVALID_SIGNATURE, INVALID_CHECKSUM } = require('hypercore-errors')
 const m = require('./messages')
-const { manifestHash, createVerifier, createManifest, defaultSignerManifest, isCompat } = require('./manifest')
+const Verifier = require('./verifier')
+const { createTracer } = require('hypertrace')
 
 module.exports = class Core {
   constructor (header, compat, crypto, oplog, bigHeader, tree, blocks, bitfield, verifier, legacy, onupdate, onconflict) {
+    this.tracer = createTracer(this)
     this.onupdate = onupdate
     this.onconflict = onconflict
     this.preupdate = null
@@ -92,11 +94,11 @@ module.exports = class Core {
 
       const keyPair = opts.keyPair || (opts.key ? null : crypto.keyPair())
       const defaultManifest = !opts.manifest && (!!opts.compat || !opts.key || !!(keyPair && b4a.equals(opts.key, keyPair.publicKey)))
-      const manifest = defaultManifest ? defaultSignerManifest(opts.key || keyPair.publicKey) : createManifest(opts.manifest)
+      const manifest = defaultManifest ? Verifier.defaultSignerManifest(opts.key || keyPair.publicKey) : Verifier.createManifest(opts.manifest)
 
       header = {
         external: null,
-        key: opts.key || (compat ? manifest.signer.publicKey : manifestHash(manifest)),
+        key: opts.key || (compat ? manifest.signers[0].publicKey : Verifier.manifestHash(manifest)),
         manifest,
         keyPair,
         userData: [],
@@ -119,7 +121,7 @@ module.exports = class Core {
 
     if (opts.manifest) {
       // if we provide a manifest and no key, verify that the stored key is the same
-      if (!opts.key && !isValidManifest(header.key, createManifest(opts.manifest))) {
+      if (!opts.key && !Verifier.isValidManifest(header.key, Verifier.createManifest(opts.manifest))) {
         throw STORAGE_CONFLICT('Manifest does not hash to provided key')
       }
     }
@@ -129,13 +131,15 @@ module.exports = class Core {
     }
 
     // if we signalled compat, but already now this core isn't disable it
-    if (compat && header.manifest && !isCompat(header.key, header.manifest)) {
+    if (compat && header.manifest && !Verifier.isCompat(header.key, header.manifest)) {
       compat = false
-    } else if (!compat && header.manifest && isCompat(header.key, header.manifest)) {
+    } else if (!compat && header.manifest && Verifier.isCompat(header.key, header.manifest)) {
       compat = true
     }
 
-    const tree = await MerkleTree.open(treeFile, { crypto, ...header.tree })
+    const prologue = header.manifest ? header.manifest.prologue : null
+
+    const tree = await MerkleTree.open(treeFile, { crypto, prologue, ...header.tree })
     const bitfield = await Bitfield.open(bitfieldFile, tree)
     const blocks = new BlockStore(dataFile, tree)
 
@@ -154,7 +158,7 @@ module.exports = class Core {
       while (bitfield.get(header.hints.contiguousLength)) header.hints.contiguousLength++
     }
 
-    const verifier = header.manifest ? createVerifier(header.manifest, { compat: isCompat(header.key, header.manifest), crypto, legacy }) : null
+    const verifier = header.manifest ? new Verifier(header.manifest, { compat: Verifier.isCompat(header.key, header.manifest), crypto, legacy }) : null
 
     for (const e of entries) {
       if (e.userData) {
@@ -190,11 +194,13 @@ module.exports = class Core {
   }
 
   setManifest (manifest, keyPair) {
-    if (!manifest && b4a.equals(keyPair.publicKey, this.header.key)) manifest = defaultSignerManifest(this.header.key)
+    if (!manifest && b4a.equals(keyPair.publicKey, this.header.key)) manifest = Verifier.defaultSignerManifest(this.header.key)
     if (!manifest) return
 
-    const compat = isCompat(this.header.key, manifest)
-    const verifier = createVerifier(manifest, { compat, crypto: this.crypto, legacy: this._legacy })
+    const compat = Verifier.isCompat(this.header.key, manifest)
+    const verifier = new Verifier(manifest, { compat, crypto: this.crypto, legacy: this._legacy })
+
+    if (verifier.prologue) this.tree.setPrologue(verifier.prologue)
 
     this.compat = compat
     this.header.manifest = manifest
@@ -219,7 +225,7 @@ module.exports = class Core {
     return false
   }
 
-  async copyFrom (src, signature, { length = src.tree.length } = {}) {
+  async copyFrom (src, signature, { length = src.tree.length, additional = [] } = {}) {
     await this._mutex.lock()
 
     try {
@@ -229,16 +235,19 @@ module.exports = class Core {
       throw err
     }
 
+    const initialLength = this.tree.length
+
     try {
       const updates = []
 
       let pos = 0
+      const copyLength = Math.min(src.tree.length, length)
 
-      while (pos < length) {
+      while (pos < copyLength) {
         const segmentStart = maximumSegmentStart(pos, src.bitfield, this.bitfield)
         if (segmentStart >= length || segmentStart < 0) break
 
-        const segmentEnd = Math.min(length, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
+        const segmentEnd = Math.min(src.tree.length, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
 
         const segment = []
 
@@ -262,7 +271,7 @@ module.exports = class Core {
         })
       }
 
-      for (let i = 0; i < length * 2; i++) {
+      for (let i = 0; i < copyLength * 2; i++) {
         const node = await src.tree.get(i, false)
         if (node === null) continue
 
@@ -271,22 +280,50 @@ module.exports = class Core {
 
       await this.tree.flush()
 
-      if (length > this.tree.length) {
-        this.tree.fork = src.tree.fork
-        this.tree.roots = [...src.tree.roots]
-        this.tree.length = src.tree.length
-        this.tree.byteLength = src.tree.byteLength
-
-        if (length < this.tree.length) {
-          const batch = await src.tree.truncate(length)
-          this.tree.roots = [...batch.roots]
-          this.tree.length = batch.length
-          this.tree.byteLength = batch.byteLength
+      let batch = this.tree.batch()
+
+      // add additional blocks
+      if (length > src.tree.length) {
+        const missing = length - src.tree.length
+
+        if (additional.length < missing) {
+          throw INVALID_OPERATION('Insufficient additional nodes were passed')
         }
 
+        const source = src.tree.batch()
+
+        batch.roots = [...source.roots]
+        batch.length = source.length
+        batch.byteLength = source.byteLength
+
+        const blocks = additional.length === missing ? additional : additional.slice(0, missing)
+
+        await this.blocks.putBatch(source.length, blocks, source.byteLength)
+        this.bitfield.setRange(source.length, missing, true)
+
+        updates.push({
+          drop: false,
+          start: source.length,
+          length: missing
+        })
+
+        for (const block of blocks) await batch.append(block)
+      } else {
+        const source = length < src.tree.length ? await src.tree.truncate(length) : src.tree.batch()
+
+        this.tree.roots = [...source.roots]
+        this.tree.length = source.length
+        this.tree.byteLength = source.byteLength
+
+        // update batch
+        batch = this.tree.batch()
+      }
+
+      // verify if upgraded
+      if (batch.length > initialLength) {
         try {
-          const batch = this.tree.batch()
           batch.signature = signature
+
           this._verifyBatchUpgrade(batch, this.header.manifest)
           this.tree.signature = signature
         } catch (err) {
@@ -294,12 +331,16 @@ module.exports = class Core {
           // TODO: how to handle signature failure?
           throw err
         }
-
-        this.header.tree.length = this.tree.length
-        this.header.tree.rootHash = this.tree.hash()
-        this.header.tree.signature = this.tree.signature
       }
 
+      await batch.commit()
+
+      this.tree.fork = src.tree.fork
+
+      this.header.tree.length = this.tree.length
+      this.header.tree.rootHash = this.tree.hash()
+      this.header.tree.signature = this.tree.signature
+
       this.header.userData = src.header.userData.slice(0)
       this.header.hints.contiguousLength = Math.min(src.header.hints.contiguousLength, this.header.tree.length)
 
@@ -379,6 +420,10 @@ module.exports = class Core {
   }
 
   async truncate (length, fork, { signature, keyPair = this.header.keyPair } = {}) {
+    if (this.tree.prologue && length < this.tree.prologue.length) {
+      throw INVALID_OPERATION('Truncation breaks prologue')
+    }
+
     this.truncating++
     await this._mutex.lock()
 
@@ -553,6 +598,11 @@ module.exports = class Core {
       const batch = this.tree.batch()
       for (const val of values) batch.append(val)
 
+      // only multisig can have prologue so signature is always present
+      if (this.tree.prologue && batch.length < this.tree.prologue.length) {
+        throw INVALID_OPERATION('Append is not consistent with prologue')
+      }
+
       batch.signature = signature || this.verifier.sign(batch, keyPair)
 
       const entry = {
@@ -590,21 +640,21 @@ module.exports = class Core {
 
   _verifyBatchUpgrade (batch, manifest) {
     if (!this.header.manifest) {
-      if (!manifest && this.compat) manifest = defaultSignerManifest(this.header.key)
+      if (!manifest && this.compat) manifest = Verifier.defaultSignerManifest(this.header.key)
 
-      if (!manifest || !(isValidManifest(this.header.key, manifest) || (this.compat && isCompat(this.header.key, manifest)))) {
+      if (!manifest || !(Verifier.isValidManifest(this.header.key, manifest) || (this.compat && Verifier.isCompat(this.header.key, manifest)))) {
         throw INVALID_SIGNATURE('Proof contains an invalid manifest') // TODO: proper error type
       }
     }
 
-    const verifier = this.verifier || createVerifier(manifest, { compat: isCompat(this.header.key, manifest), crypto: this.crypto, legacy: this._legacy })
+    const verifier = this.verifier || new Verifier(manifest, { compat: Verifier.isCompat(this.header.key, manifest), crypto: this.crypto, legacy: this._legacy })
 
     if (!verifier.verify(batch, batch.signature)) {
       throw INVALID_SIGNATURE('Proof contains an invalid signature')
     }
 
     if (!this.header.manifest) {
-      this.compat = isCompat(this.header.key, manifest)
+      this.compat = Verifier.isCompat(this.header.key, manifest)
       this.header.manifest = manifest
       this.verifier = verifier
       this.onupdate(0b10000, null, null, null)
@@ -703,7 +753,7 @@ module.exports = class Core {
 
         // if we got a manifest AND its strictly a non compat one, lets store it
         if (manifest && this.header.manifest === null) {
-          if (!isValidManifest(this.header.key, manifest)) throw INVALID_CHECKSUM('Manifest hash does not match')
+          if (!Verifier.isValidManifest(this.header.key, manifest)) throw INVALID_CHECKSUM('Manifest hash does not match')
           this.setManifest(manifest, null)
         }
 
@@ -879,10 +929,6 @@ function updateContig (header, upd, bitfield) {
   return 0b1000
 }
 
-function isValidManifest (key, manifest) {
-  return b4a.equals(key, manifestHash(manifest))
-}
-
 function addReorgHint (list, tree, batch) {
   if (tree.length === 0 || tree.fork === batch.fork) return
 

package/lib/merkle-tree.js

@@ -360,13 +360,14 @@ class ByteSeeker {
 }
 
 module.exports = class MerkleTree {
-  constructor (storage, roots, fork, signature) {
+  constructor (storage, roots, fork, signature, prologue) {
     this.crypto = crypto
     this.fork = fork
     this.roots = roots
     this.length = roots.length ? totalSpan(roots) / 2 : 0
     this.byteLength = totalSize(roots)
     this.signature = signature
+    this.prologue = prologue
 
     this.storage = storage
     this.unflushed = new Map()
@@ -424,6 +425,10 @@ module.exports = class MerkleTree {
     return Promise.all(roots)
   }
 
+  setPrologue ({ hash, length }) {
+    this.prologue = { hash, length }
+  }
+
   async upgradeable (length) {
     const indexes = flat.fullRoots(2 * length)
     const roots = new Array(indexes.length)
@@ -718,7 +723,7 @@ module.exports = class MerkleTree {
       roots.push(await getStoredNode(storage, index, null, true))
     }
 
-    return new MerkleTree(storage, roots, opts.fork || 0, opts.signature || null)
+    return new MerkleTree(storage, roots, opts.fork || 0, opts.signature || null, opts.prologue || null)
   }
 }
 
@@ -811,6 +816,15 @@ function verifyTree ({ block, hash, seek }, crypto, nodes) {
 }
 
 function verifyUpgrade ({ fork, upgrade }, blockRoot, batch) {
+  const prologue = batch.tree.prologue
+
+  if (prologue) {
+    const { start, length } = upgrade
+    if (start < prologue.length && (start !== 0 || length !== prologue.length)) {
+      throw INVALID_PROOF('Upgrade does not satisfy prologue')
+    }
+  }
+
   const q = new NodeQueue(upgrade.nodes, blockRoot)
 
   let grow = batch.roots.length > 0
@@ -840,6 +854,12 @@ function verifyUpgrade ({ fork, upgrade }, blockRoot, batch) {
     batch.appendRoot(q.shift(ite.index), ite)
   }
 
+  if (prologue && batch.length === prologue.length) {
+    if (!b4a.equals(prologue.hash, batch.hash())) {
+      throw INVALID_PROOF('Invalid hash')
+    }
+  }
+
   const extra = upgrade.additionalNodes
 
   ite.seek(batch.roots[batch.roots.length - 1].index)
@@ -1176,6 +1196,11 @@ async function generateProof (tree, block, hash, seek, upgrade) {
   // Important that this does not throw inbetween making the promise arrays
   // and finalise being called, otherwise there will be lingering promises in the background
 
+  if (tree.prologue && upgrade) {
+    upgrade.start = upgrade.start < tree.prologue.length ? 0 : upgrade.start
+    upgrade.length = upgrade.start < tree.prologue.length ? tree.prologue.length : upgrade.length
+  }
+
   const fork = tree.fork
   const signature = tree.signature
   const head = 2 * tree.length