hypercore 11.23.1 → 11.25.0

package/index.js

@@ -9,6 +9,7 @@ const Protomux = require('protomux')
 const id = require('hypercore-id-encoding')
 const safetyCatch = require('safety-catch')
 const unslab = require('unslab')
+const flat = require('flat-tree')
 
 const inspect = require('./lib/inspect')
 const Core = require('./lib/core')
@@ -20,6 +21,7 @@ const Replicator = require('./lib/replicator')
 const { manifestHash, createManifest } = require('./lib/verifier')
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const { MerkleTree } = require('./lib/merkle-tree')
+const { proof, verify } = require('./lib/fully-remote-proof')
 const {
   ASSERTION,
   BAD_ARGUMENT,
@@ -85,6 +87,8 @@ class Hypercore extends EventEmitter {
     this._findingPeers = 0
     this._active = opts.weak ? !!opts.active : opts.active !== false
 
+    this.waits = 0
+
     this._sessionIndex = -1
     this._stateIndex = -1 // maintained by session state
     this._monitorIndex = -1 // maintained by replication state
@@ -284,6 +288,15 @@ class Hypercore extends EventEmitter {
       throw err
     }
 
+    // Setup automatic recovery if in repair mode
+    if (this.core._repairMode) {
+      const recoverTreeNodeFromPeersBound = this.recoverTreeNodeFromPeers.bind(this)
+      this.once('repaired', () => {
+        this.off('peer-add', recoverTreeNodeFromPeersBound)
+      })
+      this.on('peer-add', recoverTreeNodeFromPeersBound)
+    }
+
     this.emit('ready')
 
     // if we are a weak session the core might have closed...
@@ -318,6 +331,11 @@ class Hypercore extends EventEmitter {
       this.encodeBatch = opts.encodeBatch
     }
 
+    // one session sets for pushOnly for all
+    if (opts.pushOnly === true) {
+      this.core.replicator.setPushOnly(true)
+    }
+
     if (parent) {
       if (parent._stateIndex === -1) await parent.ready()
       if (!this.keyPair) this.keyPair = parent.keyPair
@@ -853,12 +871,14 @@ class Hypercore extends EventEmitter {
 
     if (!this._shouldWait(opts, this.wait)) return null
 
+    this.waits++
     if (opts && opts.onwait) opts.onwait(index, this)
     if (this.onwait) this.onwait(index, this)
 
     const activeRequests = (opts && opts.activeRequests) || this.activeRequests
 
-    const req = this.core.replicator.addBlock(activeRequests, index)
+    const force = opts ? opts.force === true : false
+    const req = this.core.replicator.addBlock(activeRequests, index, force)
     req.snapshot = index < this.length
 
     const timeout = opts && opts.timeout !== undefined ? opts.timeout : this.timeout
@@ -980,6 +1000,7 @@ class Hypercore extends EventEmitter {
   async treeHash(length = -1) {
     if (this.opened === false) await this.opening
     if (length === -1) length = this.length
+    if (length > 0 && !(await this.has(length - 1))) await this.get(length - 1)
 
     const roots = await MerkleTree.getRoots(this.state, length)
     return crypto.tree(roots)
@@ -1014,6 +1035,52 @@ class Hypercore extends EventEmitter {
     return batch
   }
 
+  generateRemoteProofForTreeNode(treeNodeIndex) {
+    const blockProofIndex = flat.rightSpan(treeNodeIndex) / 2
+    return proof(this, {
+      index: blockProofIndex,
+      // + 1 to length so the block is included
+      upgrade: { start: 0, length: blockProofIndex + 1 }
+    })
+  }
+
+  async recoverFromRemoteProof(remoteProof) {
+    this.core.replicator.setPushOnly(true)
+    this.core._repairMode = true
+    await this.core.state.mutex.lock()
+    const p = await verify(this.core.db, remoteProof)
+    if (!p) return false
+
+    const tx = this.core.storage.write()
+    for (const node of p.proof.upgrade.nodes) {
+      tx.putTreeNode(node)
+    }
+    await tx.flush()
+
+    this.core.state.mutex.unlock()
+    const succeed = p.proof.upgrade.nodes.length !== 0
+    if (succeed) {
+      this.core.replicator.setPushOnly(false)
+    }
+    return succeed
+  }
+
+  recoverTreeNodeFromPeers() {
+    this.core.replicator.setPushOnly(true)
+
+    for (const peer of this.core.replicator.peers) {
+      const req = {
+        id: 0,
+        fork: this.fork,
+        upgrade: {
+          start: 0,
+          length: this.length
+        }
+      }
+      peer.wireRequest.send(req)
+    }
+  }
+
   registerExtension(name, handlers = {}) {
     if (this.extensions.has(name)) {
       const ext = this.extensions.get(name)
@@ -1150,6 +1217,7 @@ function initOnce(session, storage, key, opts) {
     notDownloadingLinger: opts.notDownloadingLinger,
     allowFork: opts.allowFork !== false,
     allowPush: !!opts.allowPush,
+    pushOnly: !!opts.pushOnly,
     alwaysLatestBlock: !!opts.allowLatestBlock,
     inflightRange: opts.inflightRange,
     compat: opts.compat === true,

package/lib/core.js

@@ -61,6 +61,7 @@ module.exports = class Core {
     this._verifies = null
     this._verifiesFlushed = null
     this._legacy = !!opts.legacy
+    this._repairMode = false
 
     this.opening = this._open(opts)
     this.opening.catch(noop)
@@ -269,12 +270,16 @@ module.exports = class Core {
       fork: header.tree.fork,
       length: header.tree.length,
       signature: header.tree.signature,
-      roots: header.tree.length
-        ? await MerkleTree.getRootsFromStorage(storage, header.tree.length)
-        : [],
+      roots: await maybeGetRootsFromStorage(storage, header.tree.length),
       prologue
     }
 
+    this._repairMode = !treeInfo.roots.length && treeInfo.length && !overwrite
+    // Set push only mode if in repair mode
+    if (this._repairMode) {
+      this.replicator.setPushOnly(true)
+    }
+
     if (overwrite) {
       const tx = storage.write()
       tx.deleteTreeNodeRange(0, -1)
@@ -573,7 +578,7 @@ module.exports = class Core {
       return true
     }
 
-    const roots = await MerkleTree.getRootsFromStorage(this.storage, proof.upgrade.length)
+    const roots = await maybeGetRootsFromStorage(this.storage, proof.upgrade.length)
     const remoteTreeHash = crypto.tree(proof.upgrade.nodes)
     const localTreeHash = crypto.tree(roots)
 
@@ -595,8 +600,10 @@ module.exports = class Core {
 
       const verifyBatch = MerkleTree.verifyFullyRemote(this.state, await treeProof.settle())
       this._verifyBatchUpgrade(verifyBatch, this.header.manifest)
-    } catch {
-      return true
+    } catch (err) {
+      if (err.code !== 'INVALID_OPERATION') return true
+
+      return !(await this._repairTreeNodes(proof))
     }
 
     // both proofs are valid, now check if they forked
@@ -637,6 +644,68 @@ module.exports = class Core {
     return true
   }
 
+  async _repairTreeNodes(proof) {
+    if (!this._repairMode) return false
+
+    await this.state.mutex.lock()
+
+    for (let i = this.monitors.length - 1; i >= 0; i--) {
+      this.monitors[i].emit('repairing', proof)
+    }
+
+    try {
+      const batch = MerkleTree.verifyFullyRemote(this.state, proof)
+      this._verifyBatchUpgrade(batch, proof.manifest)
+    } catch {
+      this.state.mutex.unlock()
+      for (let i = this.monitors.length - 1; i >= 0; i--) {
+        this.monitors[i].emit('repair-failed')
+      }
+      return false
+    }
+
+    // Attempt to repair local merkle tree with remote roots
+    const tx = this.storage.write()
+    for (const node of proof.upgrade.nodes) {
+      tx.putTreeNode(node)
+    }
+    await tx.flush()
+
+    // Try local proof again
+    try {
+      const rx = this.state.storage.read()
+      const treeProofPromise = MerkleTree.proof(this.state, rx, {
+        block: null,
+        hash: null,
+        seek: null,
+        upgrade: {
+          start: 0,
+          length: proof.upgrade.length
+        }
+      })
+
+      rx.tryFlush()
+
+      const treeProof = await treeProofPromise
+
+      const verifyBatch = MerkleTree.verifyFullyRemote(this.state, await treeProof.settle())
+      this._verifyBatchUpgrade(verifyBatch, this.header.manifest)
+    } catch (err) {
+      this.state.mutex.unlock()
+      for (let i = this.monitors.length - 1; i >= 0; i--) {
+        this.monitors[i].emit('repair-failed')
+      }
+      return false
+    }
+
+    for (let i = this.monitors.length - 1; i >= 0; i--) {
+      this.monitors[i].emit('repaired', proof)
+    }
+    this.replicator.setPushOnly(false)
+    this.state.mutex.unlock()
+    return true
+  }
+
   async verifyReorg(proof) {
     const batch = new ReorgBatch(this.state)
     await MerkleTree.reorg(this.state, proof, batch)
@@ -899,6 +968,16 @@ function parseHeader(info) {
   }
 }
 
+async function maybeGetRootsFromStorage(storage, length) {
+  try {
+    return length ? await MerkleTree.getRootsFromStorage(storage, length) : []
+  } catch (err) {
+    if (err.code !== 'INVALID_OPERATION') throw err
+    // If INVALID_OPERATION, a root is likely missing so load empty roots to try to recover
+    return []
+  }
+}
+
 function noop() {}
 
 async function getCoreInfo(storage) {

package/lib/fully-remote-proof.js

@@ -65,7 +65,9 @@ async function verify(storage, buffer, { referrer = null } = {}) {
 
   rx.tryFlush()
 
-  const roots = await Promise.all(rootPromises)
+  let roots = await Promise.all(rootPromises)
+  if (roots.some(isNull)) roots = []
+
   const length = head ? head.length : 0
 
   if (!auth.manifest || !auth.manifest.signers.length) return null
@@ -100,10 +102,10 @@ async function verify(storage, buffer, { referrer = null } = {}) {
   return result
 }
 
-async function proof(sender, { index, block = null } = {}) {
+async function proof(sender, { index, block = null, upgrade = null } = {}) {
   const proof = await sender.proof({
     block: block ? { index, nodes: 0 } : null,
-    upgrade: { start: 0, length: sender.length }
+    upgrade: upgrade ? upgrade : { start: 0, length: sender.length }
   })
 
   if (block) proof.block.value = block
@@ -122,3 +124,7 @@ async function proof(sender, { index, block = null } = {}) {
 
   return state.buffer
 }
+
+function isNull(x) {
+  return x === null
+}

package/lib/merkle-tree.js

@@ -231,6 +231,11 @@ class MerkleTreeBatch {
 
   commit(tx) {
     if (tx === undefined) throw INVALID_OPERATION('No database batch was passed')
+
+    if (this.session.core._repairMode) {
+      throw ASSERTION('Cannot commit while repair mode is on')
+    }
+
     if (!this.commitable()) {
       throw INVALID_OPERATION('Tree was modified during batch, refusing to commit')
     }
@@ -768,6 +773,18 @@ class MerkleTree {
     rx.tryFlush()
     return Promise.all([offset, size])
   }
+
+  static upgradeLength(proof) {
+    const extra = proof.additionalNodes
+
+    if (extra.length) {
+      const last = extra[extra.length - 1]
+      const index = flat.rightSpan(last.index)
+      return index / 2 + 1
+    }
+
+    return proof.start + proof.length
+  }
 }
 
 module.exports = {

package/lib/replicator.js

@@ -197,7 +197,7 @@ class Attachable {
 }
 
 class BlockRequest extends Attachable {
-  constructor(replicator, tracker, index, priority) {
+  constructor(replicator, tracker, index, priority, force) {
     super(replicator)
 
     this.index = index
@@ -206,6 +206,7 @@ class BlockRequest extends Attachable {
     this.queued = false
     this.hotswap = null
     this.tracker = tracker
+    this.force = force
   }
 
   _unref() {
@@ -409,11 +410,11 @@ class BlockTracker {
     return this._map.get(index) || null
   }
 
-  add(index, priority) {
+  add(index, priority, force) {
     let b = this._map.get(index)
     if (b) return b
 
-    b = new BlockRequest(this._replicator, this, index, priority)
+    b = new BlockRequest(this._replicator, this, index, priority, force)
     this._map.set(index, b)
 
     return b
@@ -674,6 +675,9 @@ class Peer {
   }
 
   sendSync() {
+    // Skip syncs if in repair mode
+    if (this.core._repairMode) return
+
     if (this.syncsProcessing !== 0) {
       this.needsSync = true
       return
@@ -996,55 +1000,92 @@ class Peer {
       priority: 0
     }
 
-    const remoteLength = Math.max(this.remoteLength, this.pushedLength)
+    await this.replicator._txLock.lock()
+    try {
+      const remoteLength = Math.max(this.remoteLength, this.pushedLength)
 
-    msg.block = {
-      index,
-      nodes: MerkleTree.maxMissingNodes(2 * index, remoteLength)
-    }
+      msg.block = {
+        index,
+        nodes: MerkleTree.maxMissingNodes(2 * index, remoteLength)
+      }
 
-    if (index >= remoteLength) {
-      msg.upgrade = {
-        start: remoteLength,
-        length: this.core.state.length - remoteLength
+      if (index >= remoteLength) {
+        msg.upgrade = {
+          start: remoteLength,
+          length: this.core.state.length - remoteLength
+        }
       }
-    }
 
-    let req = null
-    const batch = this.core.storage.read()
-    try {
-      req = await this._getProof(batch, msg, true)
-    } catch (err) {
-      this.replicator._oninvalidrequest(err, msg, this)
-      return
-    }
+      let req = null
+      const batch = this.core.storage.read()
+      try {
+        req = await this._getProof(batch, msg, true)
+      } catch (err) {
+        this.replicator._oninvalidrequest(err, msg, this)
+        return
+      }
 
-    if (req === null) return
-    batch.tryFlush()
+      if (req === null) return
+      batch.tryFlush()
 
-    await this._fulfillRequest(req, true)
+      await this._fulfillRequest(req, true)
+    } finally {
+      this.replicator._txLock.unlock()
+    }
   }
 
   async _handleRequest(msg) {
-    const batch = this.core.storage.read()
+    const locked = !!msg.upgrade && this.remoteAllowPush
 
-    // TODO: could still be answerable if (index, fork) is an ancestor of the current fork
-    const req =
-      msg.fork === this.core.state.fork
-        ? await this._getProof(batch, msg, false)
-        : new ProofRequest(msg, null, null, null)
+    if (locked) {
+      // ensure contig push proofs
+      await this.replicator._txLock.lock()
+      const end = msg.upgrade.start + msg.upgrade.length
 
-    if (req === null) {
-      this.wireNoData.send({ request: msg.id, reason: INVALID_REQUEST })
-      return
+      if (this.pushedLength < end) {
+        msg.upgrade.start = this.pushedLength
+        msg.upgrade.length = end - this.pushedLength
+      }
     }
 
-    batch.tryFlush()
+    try {
+      const batch = this.core.storage.read()
+
+      // TODO: could still be answerable if (index, fork) is an ancestor of the current fork
+      const req =
+        msg.fork === this.core.state.fork
+          ? await this._getProof(batch, msg, false)
+          : new ProofRequest(msg, null, null, null)
 
-    await this._fulfillRequest(req, false)
+      if (req === null) {
+        this.wireNoData.send({ request: msg.id, reason: INVALID_REQUEST })
+        return
+      }
+
+      batch.tryFlush()
+
+      await this._fulfillRequest(req, false)
+    } finally {
+      if (locked) this.replicator._txLock.unlock()
+    }
   }
 
   async _fulfillRequest(req, pushing) {
+    if (this.core._repairMode) {
+      // if cancelled do not reply
+      if (this.remoteRequests.get(req.msg.id) !== req.msg) {
+        return
+      }
+
+      // sync from now on, so safe to delete from the map
+      this.remoteRequests.delete(req.msg.id)
+
+      // If cutting off the fulfillment, ensure promise is caught.
+      // While repairing merkle tree nodes can be `nil` so can throw.
+      req.fulfill().catch(safetyCatch)
+      return
+    }
+
     const proof = await req.fulfill()
 
     if (!pushing) {
@@ -1089,7 +1130,7 @@ class Peer {
     }
 
     if (proof.upgrade) {
-      const remoteLength = proof.upgrade.start + proof.upgrade.length
+      const remoteLength = MerkleTree.upgradeLength(proof.upgrade)
       if (remoteLength > this.pushedLength) this.pushedLength = remoteLength
     }
 
@@ -1150,13 +1191,13 @@ class Peer {
   }
 
   async ondata(data) {
-    if (data.request !== 0) return this._handleData(data)
+    if (data.request !== 0 && !data.upgrade) return this._handleData(data)
 
-    await this.replicator._pushLock.lock()
+    await this.replicator._rxLock.lock()
     try {
       await this._handleData(data)
     } finally {
-      this.replicator._pushLock.unlock()
+      this.replicator._rxLock.unlock()
     }
   }
 
@@ -1176,6 +1217,12 @@ class Peer {
       return
     }
 
+    // Skip accepting data if in repair mode to avoid corruption
+    if (this.core._repairMode) {
+      this.replicator._onnodata(this, req, NOT_AVAILABLE)
+      return
+    }
+
     if (req === null && reorg === false && data.block) {
       // mark this as inflight to avoid parallel requests
       this.replicator._markInflight(data.block.index)
@@ -1452,6 +1499,8 @@ class Peer {
 
   _requestManifest() {
     const req = this._makeRequest(false, 0, 0)
+    if (req === null) return
+
     this._send(req)
   }
 
@@ -1465,15 +1514,17 @@ class Peer {
 
     // do the normal checks
     if (!this._remoteHasBlock(index)) return null
-    if (!this._canRequest(index)) return null
+    if (!this._canRequest(index, false)) return null
 
     // atm we only ever do one upgrade request in parallel, if that changes
     // then we should add some check here for inflights to avoid over requesting
-    const b = this.replicator._blocks.add(index, PRIORITY.NORMAL)
+    const b = this.replicator._blocks.add(index, PRIORITY.NORMAL, false)
     return b
   }
 
   _requestUpgrade(u) {
+    if (this.replicator.pushOnly) return false
+
     const req = this._makeRequest(true, 0, 0)
     if (req === null) return false
 
@@ -1488,6 +1539,7 @@ class Peer {
   _requestSeek(s) {
     // if replicator is updating the seeks etc, bail and wait for it to drain
     if (this.replicator._updatesPending > 0) return false
+    if (this.replicator.pushOnly) return false
 
     const { length, fork } = this.core.state
 
@@ -1518,14 +1570,14 @@ class Peer {
 
       if (this._remoteHasBlock(index) === false) continue
       if (this.core.bitfield.get(index) === true) continue
-      if (!this._canRequest(index)) continue
+      if (!this._canRequest(index, false)) continue
 
       // Check if this block is currently inflight - if so pick another
       const b = this.replicator._blocks.get(index)
       if (b !== null && b.inflight.length > 0) continue
 
       // Block is not inflight, but we only want the hash, check if that is inflight
-      const h = this.replicator._hashes.add(index, PRIORITY.NORMAL)
+      const h = this.replicator._hashes.add(index, PRIORITY.NORMAL, false)
       if (h.inflight.length > 0) continue
 
       const req = this._makeRequest(false, h.priority, index + 1)
@@ -1549,9 +1601,13 @@ class Peer {
     return false
   }
 
-  _canRequest(index) {
+  _canRequest(index, force) {
     if (!(index >= 0)) throw ASSERTION('bad index to _canRequest: ' + index)
 
+    if (!force && this.replicator.pushOnly) {
+      return false
+    }
+
     if (this.remoteLength >= this.core.state.length) {
       return true
     }
@@ -1572,6 +1628,8 @@ class Peer {
   }
 
   _sendBlockRequest(req, b) {
+    if (this.core._repairMode) return
+
     req.block = { index: b.index, nodes: 0 }
     this.replicator._markInflight(b.index)
 
@@ -1588,7 +1646,7 @@ class Peer {
       return false
     }
 
-    if (!this._canRequest(b.index)) return false
+    if (!this._canRequest(b.index, b.force)) return false
 
     const req = this._makeRequest(b.index >= length, b.priority, b.index + 1)
     if (req === null) return false
@@ -1636,9 +1694,9 @@ class Peer {
   }
 
   _requestRangeBlock(index, length) {
-    if (this.core.bitfield.get(index) === true || !this._canRequest(index)) return false
+    if (this.core.bitfield.get(index) === true || !this._canRequest(index, false)) return false
 
-    const b = this.replicator._blocks.add(index, PRIORITY.NORMAL)
+    const b = this.replicator._blocks.add(index, PRIORITY.NORMAL, false)
     if (b.inflight.length > 0) {
       this.missingBlocks.set(index, false) // in case we missed some states just set them ondemand, nbd
       return false
@@ -1789,8 +1847,10 @@ class Peer {
 
   _requestForkProof(f) {
     if (!this.remoteLength) return
+    if (this.replicator.pushOnly) return
 
     const req = this._makeRequest(false, 0, 0)
+    if (req === null) return
 
     req.upgrade = { start: 0, length: this.remoteLength }
     req.manifest = !this.core.header.manifest
@@ -1801,6 +1861,7 @@ class Peer {
 
   _requestForkRange(f) {
     if (f.fork !== this.remoteFork || f.batch.want === null) return false
+    if (this.replicator.pushOnly) return false
 
     const end = Math.min(f.batch.want.end, this.remoteLength)
     if (end < f.batch.want.start) return false
@@ -1815,6 +1876,7 @@ class Peer {
       if (this._remoteHasBlock(index) === false) continue
 
       const req = this._makeRequest(false, 0, 0)
+      if (req === null) return false
 
       req.hash = { index: 2 * index, nodes: f.batch.want.nodes }
 
@@ -1901,6 +1963,7 @@ module.exports = class Replicator {
     this.findingPeers = 0 // updatable from the outside
     this.destroyed = false
     this.downloading = false
+    this.pushOnly = false
     this.activeSessions = 0
 
     this.hotswaps = new HotswapQueue()
@@ -1937,7 +2000,9 @@ module.exports = class Replicator {
     this._reorgs = []
     this._ranges = []
 
-    this._pushLock = new Mutex()
+    this._rxLock = new Mutex()
+    this._txLock = new Mutex()
+
     this._hadPeers = false
     this._active = 0
     this._ifAvailable = 0
@@ -1988,6 +2053,11 @@ module.exports = class Replicator {
     return this.downloading
   }
 
+  setPushOnly(pushOnly) {
+    if (pushOnly === this.pushOnly) return
+    this.pushOnly = pushOnly
+  }
+
   setDownloading(downloading) {
     const allowPush = this.isAllowingPush()
     const linger = this._notDownloadingLinger
@@ -2124,8 +2194,8 @@ module.exports = class Replicator {
     return ref
   }
 
-  addBlock(session, index) {
-    const b = this._blocks.add(index, PRIORITY.HIGH)
+  addBlock(session, index, force) {
+    const b = this._blocks.add(index, PRIORITY.HIGH, force)
     const ref = b.attach(session)
 
     this._queueBlock(b)

package/lib/session-state.js

@@ -4,7 +4,7 @@ const assert = require('nanoassert')
 const flat = require('flat-tree')
 const quickbit = require('quickbit-universal')
 
-const { INVALID_OPERATION, INVALID_SIGNATURE } = require('hypercore-errors')
+const { INVALID_OPERATION, INVALID_SIGNATURE, ASSERTION } = require('hypercore-errors')
 
 const Mutex = require('./mutex')
 const Bitfield = require('./bitfield')
@@ -30,7 +30,7 @@ module.exports = class SessionState {
     // merkle state
     this.roots = treeInfo.roots.length ? treeInfo.roots : []
     this.fork = treeInfo.fork || 0
-    this.length = MerkleTree.span(this.roots) / 2
+    this.length = this.roots.length ? MerkleTree.span(this.roots) / 2 : treeInfo.length
     this.byteLength = MerkleTree.size(this.roots)
     this.prologue = treeInfo.prologue || null
     this.signature = treeInfo.signature || null
@@ -917,6 +917,10 @@ module.exports = class SessionState {
       'Can only commit into default state'
     )
 
+    if (this.core._repairMode) {
+      throw ASSERTION('Cannot commit while repair mode is on')
+    }
+
     let srcLocked = false
     await this.mutex.lock()
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.23.1",
+  "version": "11.25.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {