hypercore 11.24.0 → 11.26.0

package/index.js

@@ -9,6 +9,10 @@ const Protomux = require('protomux')
 const id = require('hypercore-id-encoding')
 const safetyCatch = require('safety-catch')
 const unslab = require('unslab')
+const flat = require('flat-tree')
+
+const { SMALL_WANTS } = require('./lib/feature-flags')
+const { UPDATE_COMPAT } = require('./lib/wants')
 
 const inspect = require('./lib/inspect')
 const Core = require('./lib/core')
@@ -20,6 +24,7 @@ const Replicator = require('./lib/replicator')
 const { manifestHash, createManifest } = require('./lib/verifier')
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const { MerkleTree } = require('./lib/merkle-tree')
+const { proof, verify } = require('./lib/fully-remote-proof')
 const {
   ASSERTION,
   BAD_ARGUMENT,
@@ -85,6 +90,8 @@ class Hypercore extends EventEmitter {
     this._findingPeers = 0
     this._active = opts.weak ? !!opts.active : opts.active !== false
 
+    this.waits = 0
+
     this._sessionIndex = -1
     this._stateIndex = -1 // maintained by session state
     this._monitorIndex = -1 // maintained by replication state
@@ -103,6 +110,13 @@ class Hypercore extends EventEmitter {
 
   static DefaultEncryption = DefaultEncryption
 
+  static SMALL_WANTS = SMALL_WANTS
+
+  static enable(flag) {
+    const enableCompat = (flag & SMALL_WANTS) === 0
+    UPDATE_COMPAT(enableCompat)
+  }
+
   static key(manifest, { compat, version, namespace } = {}) {
     if (b4a.isBuffer(manifest)) {
       manifest = { version, signers: [{ publicKey: manifest, namespace }] }
@@ -284,6 +298,15 @@ class Hypercore extends EventEmitter {
       throw err
     }
 
+    // Setup automatic recovery if in repair mode
+    if (this.core._repairMode) {
+      const recoverTreeNodeFromPeersBound = this.recoverTreeNodeFromPeers.bind(this)
+      this.once('repaired', () => {
+        this.off('peer-add', recoverTreeNodeFromPeersBound)
+      })
+      this.on('peer-add', recoverTreeNodeFromPeersBound)
+    }
+
     this.emit('ready')
 
     // if we are a weak session the core might have closed...
@@ -858,6 +881,7 @@ class Hypercore extends EventEmitter {
 
     if (!this._shouldWait(opts, this.wait)) return null
 
+    this.waits++
     if (opts && opts.onwait) opts.onwait(index, this)
     if (this.onwait) this.onwait(index, this)
 
@@ -1021,6 +1045,52 @@ class Hypercore extends EventEmitter {
     return batch
   }
 
+  generateRemoteProofForTreeNode(treeNodeIndex) {
+    const blockProofIndex = flat.rightSpan(treeNodeIndex) / 2
+    return proof(this, {
+      index: blockProofIndex,
+      // + 1 to length so the block is included
+      upgrade: { start: 0, length: blockProofIndex + 1 }
+    })
+  }
+
+  async recoverFromRemoteProof(remoteProof) {
+    this.core.replicator.setPushOnly(true)
+    this.core._repairMode = true
+    await this.core.state.mutex.lock()
+    const p = await verify(this.core.db, remoteProof)
+    if (!p) return false
+
+    const tx = this.core.storage.write()
+    for (const node of p.proof.upgrade.nodes) {
+      tx.putTreeNode(node)
+    }
+    await tx.flush()
+
+    this.core.state.mutex.unlock()
+    const succeed = p.proof.upgrade.nodes.length !== 0
+    if (succeed) {
+      this.core.replicator.setPushOnly(false)
+    }
+    return succeed
+  }
+
+  recoverTreeNodeFromPeers() {
+    this.core.replicator.setPushOnly(true)
+
+    for (const peer of this.core.replicator.peers) {
+      const req = {
+        id: 0,
+        fork: this.fork,
+        upgrade: {
+          start: 0,
+          length: this.length
+        }
+      }
+      peer.wireRequest.send(req)
+    }
+  }
+
   registerExtension(name, handlers = {}) {
     if (this.extensions.has(name)) {
       const ext = this.extensions.get(name)

package/lib/core.js

@@ -61,6 +61,7 @@ module.exports = class Core {
     this._verifies = null
     this._verifiesFlushed = null
     this._legacy = !!opts.legacy
+    this._repairMode = false
 
     this.opening = this._open(opts)
     this.opening.catch(noop)
@@ -269,12 +270,16 @@ module.exports = class Core {
       fork: header.tree.fork,
       length: header.tree.length,
       signature: header.tree.signature,
-      roots: header.tree.length
-        ? await MerkleTree.getRootsFromStorage(storage, header.tree.length)
-        : [],
+      roots: await maybeGetRootsFromStorage(storage, header.tree.length),
       prologue
     }
 
+    this._repairMode = !treeInfo.roots.length && treeInfo.length && !overwrite
+    // Set push only mode if in repair mode
+    if (this._repairMode) {
+      this.replicator.setPushOnly(true)
+    }
+
     if (overwrite) {
       const tx = storage.write()
       tx.deleteTreeNodeRange(0, -1)
@@ -573,7 +578,7 @@ module.exports = class Core {
       return true
     }
 
-    const roots = await MerkleTree.getRootsFromStorage(this.storage, proof.upgrade.length)
+    const roots = await maybeGetRootsFromStorage(this.storage, proof.upgrade.length)
     const remoteTreeHash = crypto.tree(proof.upgrade.nodes)
     const localTreeHash = crypto.tree(roots)
 
@@ -595,8 +600,10 @@ module.exports = class Core {
 
       const verifyBatch = MerkleTree.verifyFullyRemote(this.state, await treeProof.settle())
       this._verifyBatchUpgrade(verifyBatch, this.header.manifest)
-    } catch {
-      return true
+    } catch (err) {
+      if (err.code !== 'INVALID_OPERATION') return true
+
+      return !(await this._repairTreeNodes(proof))
     }
 
     // both proofs are valid, now check if they forked
@@ -637,6 +644,68 @@ module.exports = class Core {
     return true
   }
 
+  async _repairTreeNodes(proof) {
+    if (!this._repairMode) return false
+
+    await this.state.mutex.lock()
+
+    for (let i = this.monitors.length - 1; i >= 0; i--) {
+      this.monitors[i].emit('repairing', proof)
+    }
+
+    try {
+      const batch = MerkleTree.verifyFullyRemote(this.state, proof)
+      this._verifyBatchUpgrade(batch, proof.manifest)
+    } catch {
+      this.state.mutex.unlock()
+      for (let i = this.monitors.length - 1; i >= 0; i--) {
+        this.monitors[i].emit('repair-failed')
+      }
+      return false
+    }
+
+    // Attempt to repair local merkle tree with remote roots
+    const tx = this.storage.write()
+    for (const node of proof.upgrade.nodes) {
+      tx.putTreeNode(node)
+    }
+    await tx.flush()
+
+    // Try local proof again
+    try {
+      const rx = this.state.storage.read()
+      const treeProofPromise = MerkleTree.proof(this.state, rx, {
+        block: null,
+        hash: null,
+        seek: null,
+        upgrade: {
+          start: 0,
+          length: proof.upgrade.length
+        }
+      })
+
+      rx.tryFlush()
+
+      const treeProof = await treeProofPromise
+
+      const verifyBatch = MerkleTree.verifyFullyRemote(this.state, await treeProof.settle())
+      this._verifyBatchUpgrade(verifyBatch, this.header.manifest)
+    } catch (err) {
+      this.state.mutex.unlock()
+      for (let i = this.monitors.length - 1; i >= 0; i--) {
+        this.monitors[i].emit('repair-failed')
+      }
+      return false
+    }
+
+    for (let i = this.monitors.length - 1; i >= 0; i--) {
+      this.monitors[i].emit('repaired', proof)
+    }
+    this.replicator.setPushOnly(false)
+    this.state.mutex.unlock()
+    return true
+  }
+
   async verifyReorg(proof) {
     const batch = new ReorgBatch(this.state)
     await MerkleTree.reorg(this.state, proof, batch)
@@ -899,6 +968,16 @@ function parseHeader(info) {
   }
 }
 
+async function maybeGetRootsFromStorage(storage, length) {
+  try {
+    return length ? await MerkleTree.getRootsFromStorage(storage, length) : []
+  } catch (err) {
+    if (err.code !== 'INVALID_OPERATION') throw err
+    // If INVALID_OPERATION, a root is likely missing so load empty roots to try to recover
+    return []
+  }
+}
+
 function noop() {}
 
 async function getCoreInfo(storage) {

package/lib/feature-flags.js

@@ -0,0 +1,3 @@
+module.exports = {
+  SMALL_WANTS: 1
+}

package/lib/fully-remote-proof.js

@@ -65,7 +65,9 @@ async function verify(storage, buffer, { referrer = null } = {}) {
 
   rx.tryFlush()
 
-  const roots = await Promise.all(rootPromises)
+  let roots = await Promise.all(rootPromises)
+  if (roots.some(isNull)) roots = []
+
   const length = head ? head.length : 0
 
   if (!auth.manifest || !auth.manifest.signers.length) return null
@@ -100,10 +102,10 @@ async function verify(storage, buffer, { referrer = null } = {}) {
   return result
 }
 
-async function proof(sender, { index, block = null } = {}) {
+async function proof(sender, { index, block = null, upgrade = null } = {}) {
   const proof = await sender.proof({
     block: block ? { index, nodes: 0 } : null,
-    upgrade: { start: 0, length: sender.length }
+    upgrade: upgrade ? upgrade : { start: 0, length: sender.length }
   })
 
   if (block) proof.block.value = block
@@ -122,3 +124,7 @@ async function proof(sender, { index, block = null } = {}) {
 
   return state.buffer
 }
+
+function isNull(x) {
+  return x === null
+}

package/lib/merkle-tree.js

@@ -231,6 +231,11 @@ class MerkleTreeBatch {
 
   commit(tx) {
     if (tx === undefined) throw INVALID_OPERATION('No database batch was passed')
+
+    if (this.session.core._repairMode) {
+      throw ASSERTION('Cannot commit while repair mode is on')
+    }
+
     if (!this.commitable()) {
       throw INVALID_OPERATION('Tree was modified during batch, refusing to commit')
     }
@@ -768,6 +773,18 @@ class MerkleTree {
     rx.tryFlush()
     return Promise.all([offset, size])
   }
+
+  static upgradeLength(proof) {
+    const extra = proof.additionalNodes
+
+    if (extra.length) {
+      const last = extra[extra.length - 1]
+      const index = flat.rightSpan(last.index)
+      return index / 2 + 1
+    }
+
+    return proof.start + proof.length
+  }
 }
 
 module.exports = {

package/lib/replicator.js

@@ -31,7 +31,8 @@ const ReceiverQueue = require('./receiver-queue')
 const HotswapQueue = require('./hotswap-queue')
 const RemoteBitfield = require('./remote-bitfield')
 const { MerkleTree } = require('./merkle-tree')
-const { LocalWants, RemoteWants, WANT_BATCH } = require('./wants')
+const w = require('./wants')
+const { LocalWants, RemoteWants } = w
 const {
   REQUEST_CANCELLED,
   REQUEST_TIMEOUT,
@@ -675,6 +676,9 @@ class Peer {
   }
 
   sendSync() {
+    // Skip syncs if in repair mode
+    if (this.core._repairMode) return
+
     if (this.syncsProcessing !== 0) {
       this.needsSync = true
       return
@@ -997,55 +1001,92 @@ class Peer {
       priority: 0
     }
 
-    const remoteLength = Math.max(this.remoteLength, this.pushedLength)
+    await this.replicator._txLock.lock()
+    try {
+      const remoteLength = Math.max(this.remoteLength, this.pushedLength)
 
-    msg.block = {
-      index,
-      nodes: MerkleTree.maxMissingNodes(2 * index, remoteLength)
-    }
+      msg.block = {
+        index,
+        nodes: MerkleTree.maxMissingNodes(2 * index, remoteLength)
+      }
 
-    if (index >= remoteLength) {
-      msg.upgrade = {
-        start: remoteLength,
-        length: this.core.state.length - remoteLength
+      if (index >= remoteLength) {
+        msg.upgrade = {
+          start: remoteLength,
+          length: this.core.state.length - remoteLength
+        }
       }
-    }
 
-    let req = null
-    const batch = this.core.storage.read()
-    try {
-      req = await this._getProof(batch, msg, true)
-    } catch (err) {
-      this.replicator._oninvalidrequest(err, msg, this)
-      return
-    }
+      let req = null
+      const batch = this.core.storage.read()
+      try {
+        req = await this._getProof(batch, msg, true)
+      } catch (err) {
+        this.replicator._oninvalidrequest(err, msg, this)
+        return
+      }
 
-    if (req === null) return
-    batch.tryFlush()
+      if (req === null) return
+      batch.tryFlush()
 
-    await this._fulfillRequest(req, true)
+      await this._fulfillRequest(req, true)
+    } finally {
+      this.replicator._txLock.unlock()
+    }
   }
 
   async _handleRequest(msg) {
-    const batch = this.core.storage.read()
+    const locked = !!msg.upgrade && this.remoteAllowPush
 
-    // TODO: could still be answerable if (index, fork) is an ancestor of the current fork
-    const req =
-      msg.fork === this.core.state.fork
-        ? await this._getProof(batch, msg, false)
-        : new ProofRequest(msg, null, null, null)
+    if (locked) {
+      // ensure contig push proofs
+      await this.replicator._txLock.lock()
+      const end = msg.upgrade.start + msg.upgrade.length
 
-    if (req === null) {
-      this.wireNoData.send({ request: msg.id, reason: INVALID_REQUEST })
-      return
+      if (this.pushedLength < end) {
+        msg.upgrade.start = this.pushedLength
+        msg.upgrade.length = end - this.pushedLength
+      }
     }
 
-    batch.tryFlush()
+    try {
+      const batch = this.core.storage.read()
 
-    await this._fulfillRequest(req, false)
+      // TODO: could still be answerable if (index, fork) is an ancestor of the current fork
+      const req =
+        msg.fork === this.core.state.fork
+          ? await this._getProof(batch, msg, false)
+          : new ProofRequest(msg, null, null, null)
+
+      if (req === null) {
+        this.wireNoData.send({ request: msg.id, reason: INVALID_REQUEST })
+        return
+      }
+
+      batch.tryFlush()
+
+      await this._fulfillRequest(req, false)
+    } finally {
+      if (locked) this.replicator._txLock.unlock()
+    }
   }
 
   async _fulfillRequest(req, pushing) {
+    if (this.core._repairMode) {
+      // if cancelled do not reply
+      if (this.remoteRequests.get(req.msg.id) !== req.msg) {
+        return
+      }
+
+      // sync from now on, so safe to delete from the map
+      this.remoteRequests.delete(req.msg.id)
+
+      // If cutting off the fulfillment, ensure promise is caught.
+      // While repairing merkle tree nodes can be `nil` so can throw.
+      req.fulfill().catch(safetyCatch)
+      return
+    }
+
     const proof = await req.fulfill()
 
     if (!pushing) {
@@ -1090,7 +1131,7 @@ class Peer {
     }
 
     if (proof.upgrade) {
-      const remoteLength = proof.upgrade.start + proof.upgrade.length
+      const remoteLength = MerkleTree.upgradeLength(proof.upgrade)
       if (remoteLength > this.pushedLength) this.pushedLength = remoteLength
     }
 
@@ -1151,13 +1192,13 @@ class Peer {
   }
 
   async ondata(data) {
-    if (data.request !== 0) return this._handleData(data)
+    if (data.request !== 0 && !data.upgrade) return this._handleData(data)
 
-    await this.replicator._pushLock.lock()
+    await this.replicator._rxLock.lock()
     try {
       await this._handleData(data)
     } finally {
-      this.replicator._pushLock.unlock()
+      this.replicator._rxLock.unlock()
     }
   }
 
@@ -1177,6 +1218,12 @@ class Peer {
       return
     }
 
+    // Skip accepting data if in repair mode to avoid corruption
+    if (this.core._repairMode) {
+      this.replicator._onnodata(this, req, NOT_AVAILABLE)
+      return
+    }
+
     if (req === null && reorg === false && data.block) {
       // mark this as inflight to avoid parallel requests
       this.replicator._markInflight(data.block.index)
@@ -1582,6 +1629,8 @@ class Peer {
   }
 
   _sendBlockRequest(req, b) {
+    if (this.core._repairMode) return
+
     req.block = { index: b.index, nodes: 0 }
     this.replicator._markInflight(b.index)
 
@@ -1682,6 +1731,7 @@ class Peer {
   }
 
   _addRangeBatch(r, offset, end) {
+    const { WANT_BATCH } = w
     while (offset < end) {
       const next = this.core.bitfield.findFirst(false, offset)
       if (next === -1 || next >= end) return false
@@ -1697,6 +1747,7 @@ class Peer {
   }
 
   _populateRangeBatches(r) {
+    const { WANT_BATCH } = w
     for (let i = r.batches.length - 1; i >= 0; i--) {
       const b = r.batches[i]
       const minStart = b.batch * WANT_BATCH
@@ -1752,6 +1803,7 @@ class Peer {
   }
 
   _requestRange(r) {
+    const { WANT_BATCH } = w
     if (this.syncsProcessing > 0) return false
 
     const { length, fork } = this.core.state
@@ -1952,7 +2004,9 @@ module.exports = class Replicator {
     this._reorgs = []
     this._ranges = []
 
-    this._pushLock = new Mutex()
+    this._rxLock = new Mutex()
+    this._txLock = new Mutex()
+
     this._hadPeers = false
     this._active = 0
     this._ifAvailable = 0

package/lib/session-state.js

@@ -4,7 +4,7 @@ const assert = require('nanoassert')
 const flat = require('flat-tree')
 const quickbit = require('quickbit-universal')
 
-const { INVALID_OPERATION, INVALID_SIGNATURE } = require('hypercore-errors')
+const { INVALID_OPERATION, INVALID_SIGNATURE, ASSERTION } = require('hypercore-errors')
 
 const Mutex = require('./mutex')
 const Bitfield = require('./bitfield')
@@ -30,7 +30,7 @@ module.exports = class SessionState {
     // merkle state
     this.roots = treeInfo.roots.length ? treeInfo.roots : []
     this.fork = treeInfo.fork || 0
-    this.length = MerkleTree.span(this.roots) / 2
+    this.length = this.roots.length ? MerkleTree.span(this.roots) / 2 : treeInfo.length
     this.byteLength = MerkleTree.size(this.roots)
     this.prologue = treeInfo.prologue || null
     this.signature = treeInfo.signature || null
@@ -917,6 +917,10 @@ module.exports = class SessionState {
       'Can only commit into default state'
     )
 
+    if (this.core._repairMode) {
+      throw ASSERTION('Cannot commit while repair mode is on')
+    }
+
     let srcLocked = false
     await this.mutex.lock()
 

package/lib/wants.js

@@ -1,7 +1,7 @@
-const COMPAT = true // should be flipped once this is widely deployed
-const BATCH_UINTS = COMPAT ? 65536 : 128
-const BATCH_BYTES = BATCH_UINTS * 4
-const BATCH = BATCH_BYTES * 8 // in bits
+let COMPAT = true
+let BATCH_UINTS = COMPAT ? 65536 : 128
+let BATCH_BYTES = BATCH_UINTS * 4
+let BATCH = BATCH_BYTES * 8 // in bits
 const MAX_REMOTE_BATCHES = 4
 const MAX_RANGE = 8 * 256 * 1024
 const MIN_RANGE = 32 // 4bit ints
@@ -295,9 +295,18 @@ class RemoteWants {
   }
 }
 
+function UPDATE_COMPAT(isCompat) {
+  COMPAT = isCompat
+  BATCH_UINTS = COMPAT ? 65536 : 128
+  BATCH_BYTES = BATCH_UINTS * 4
+  BATCH = BATCH_BYTES * 8 // in bits
+  exports.WANT_BATCH = BATCH
+}
+
 exports.LocalWants = LocalWants
 exports.RemoteWants = RemoteWants
 exports.WANT_BATCH = BATCH
+exports.UPDATE_COMPAT = UPDATE_COMPAT
 
 function validateBatchRange(range) {
   if (range.length > MAX_RANGE || range.length < MIN_RANGE) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.24.0",
+  "version": "11.26.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {