hypercore 10.4.1 → 10.5.1

package/README.md

@@ -107,7 +107,7 @@ const blockIfFast = await core.get(43, { timeout: 5000 })
 const blockLocal = await core.get(44, { wait: false })
 ```
 
-Additional options include
+`options` include:
 
 ``` js
 {
@@ -118,6 +118,10 @@ Additional options include
 }
 ```
 
+#### `const has = await core.has(start, [end])`
+
+Check if the core has all blocks between `start` and `end`.
+
 #### `const updated = await core.update()`
 
 Wait for the core to try and find a signed update to it's length.
@@ -163,7 +167,7 @@ for await (const data of fullStream) {
 }
 ```
 
-Additional options include:
+`options` include:
 
 ``` js
 {
@@ -238,7 +242,7 @@ To cancel downloading a range simply destroy the range instance.
 range.destroy()
 ```
 
-#### `const info = await core.info()`
+#### `const info = await core.info([options])`
 
 Get information about this core, such as its total size in bytes.
 
@@ -252,7 +256,21 @@ Info {
   contiguousLength: 16,
   byteLength: 742,
   fork: 0,
-  padding: 8
+  padding: 8,
+  storage: {
+    oplog: 8192, 
+    tree: 4096, 
+    blocks: 4096, 
+    bitfield: 4096 
+  }
+}
+```
+
+`options` include:
+
+```js
+{
+  storage: false // get storage estimates in bytes, disabled by default
 }
 ```
 

package/index.js

@@ -324,7 +324,8 @@ module.exports = class Hypercore extends EventEmitter {
       crypto: this.crypto,
       legacy: opts.legacy,
       auth: opts.auth,
-      onupdate: this._oncoreupdate.bind(this)
+      onupdate: this._oncoreupdate.bind(this),
+      onconflict: this._oncoreconflict.bind(this)
     })
 
     if (opts.userData) {
@@ -377,13 +378,13 @@ module.exports = class Hypercore extends EventEmitter {
     return prev.length !== next.length || prev.fork !== next.fork
   }
 
-  close () {
+  close (err) {
     if (this.closing) return this.closing
-    this.closing = this._close()
+    this.closing = this._close(err || null)
     return this.closing
   }
 
-  async _close () {
+  async _close (err) {
     await this.opening
 
     const i = this.sessions.indexOf(this)
@@ -403,19 +404,23 @@ module.exports = class Hypercore extends EventEmitter {
 
     if (this.replicator !== null) {
       this.replicator.findingPeers -= this._findingPeers
-      this.replicator.clearRequests(this.activeRequests)
+      this.replicator.clearRequests(this.activeRequests, err)
     }
 
     this._findingPeers = 0
 
     if (this.sessions.length) {
       // if this is the last session and we are auto closing, trigger that first to enforce error handling
-      if (this.sessions.length === 1 && this.autoClose) await this.sessions[0].close()
+      if (this.sessions.length === 1 && this.autoClose) await this.sessions[0].close(err)
       // emit "fake" close as this is a session
       this.emit('close', false)
       return
     }
 
+    if (this.replicator !== null) {
+      this.replicator.destroy()
+    }
+
     await this.core.close()
 
     this.emit('close', true)
@@ -510,6 +515,18 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
+  async _oncoreconflict (proof, from) {
+    await this.replicator.onconflict(from)
+
+    for (const s of this.sessions) s.emit('conflict', proof.upgrade.length, proof.fork, proof)
+
+    const err = new Error('Two conflicting signatures exist for length ' + proof.upgrade.length)
+
+    const all = []
+    for (const s of this.sessions) all.push(s.close(err))
+    await Promise.allSettled(all)
+  }
+
   _oncoreupdate (status, bitfield, value, from) {
     if (status !== 0) {
       const truncatedNonSparse = (status & 0b1000) !== 0
@@ -614,10 +631,10 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
-  async info () {
+  async info (opts) {
     if (this.opened === false) await this.opening
 
-    return Info.from(this)
+    return Info.from(this, opts)
   }
 
   async update (opts) {
@@ -666,10 +683,15 @@ module.exports = class Hypercore extends EventEmitter {
     return req.promise
   }
 
-  async has (index) {
+  async has (start, end = start + 1) {
     if (this.opened === false) await this.opening
 
-    return this.core.bitfield.get(index)
+    const length = end - start
+    if (length <= 0) return false
+    if (length === 1) return this.core.bitfield.get(start)
+
+    const i = this.core.bitfield.firstUnset(start)
+    return i === -1 || i >= end
   }
 
   async get (index, opts) {
@@ -804,13 +826,13 @@ module.exports = class Hypercore extends EventEmitter {
       }
     }
 
-    return await this.core.append(buffers, this.auth, { preappend })
+    return this.core.append(buffers, this.auth, { preappend })
   }
 
   async treeHash (length) {
     if (length === undefined) {
       await this.ready()
-      length = this.core.length
+      length = this.core.tree.length
     }
 
     const roots = await this.core.tree.getRoots(length)

package/lib/core.js

@@ -9,8 +9,9 @@ const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = req
 const m = require('./messages')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate) {
+  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate, onconflict) {
     this.onupdate = onupdate
+    this.onconflict = onconflict
     this.header = header
     this.crypto = crypto
     this.oplog = oplog
@@ -158,7 +159,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop)
+    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop, opts.onconflict || noop)
   }
 
   _shouldFlush () {
@@ -437,6 +438,27 @@ module.exports = class Core {
     return verifies[0] !== null
   }
 
+  async checkConflict (proof, from) {
+    if (this.tree.length < proof.upgrade.length || proof.fork !== this.tree.fork) {
+      // out of date this proof - ignore for now
+      return false
+    }
+
+    const batch = this.tree.verifyFullyRemote(proof)
+
+    if (!batch.signature || !this._signed(batch, batch.hash())) {
+      throw INVALID_SIGNATURE('Proof contains an invalid signature with no input from us')
+    }
+
+    const remoteTreeHash = this.crypto.tree(proof.upgrade.nodes)
+    const localTreeHash = this.crypto.tree(await this.tree.getRoots(proof.upgrade.length))
+
+    if (b4a.equals(localTreeHash, remoteTreeHash)) return false
+
+    await this.onconflict(proof)
+    return true
+  }
+
   async verify (proof, from) {
     // We cannot apply "other forks" atm.
     // We should probably still try and they are likely super similar for non upgrades

package/lib/errors.js

@@ -32,6 +32,18 @@ module.exports = class HypercoreError extends Error {
     return new HypercoreError(msg, 'INVALID_CAPABILITY', HypercoreError.INVALID_CAPABILITY)
   }
 
+  static INVALID_CHECKSUM (msg = 'Invalid checksum') {
+    return new HypercoreError(msg, 'INVALID_CHECKSUM', HypercoreError.INVALID_CHECKSUM)
+  }
+
+  static INVALID_OPERATION (msg) {
+    return new HypercoreError(msg, 'INVALID_OPERATION', HypercoreError.INVALID_OPERATION)
+  }
+
+  static INVALID_PROOF (msg = 'Proof not verifiable') {
+    return new HypercoreError(msg, 'INVALID_PROOF', HypercoreError.INVALID_PROOF)
+  }
+
   static SNAPSHOT_NOT_AVAILABLE (msg = 'Snapshot is not available') {
     return new HypercoreError(msg, 'SNAPSHOT_NOT_AVAILABLE', HypercoreError.SNAPSHOT_NOT_AVAILABLE)
   }
@@ -47,4 +59,12 @@ module.exports = class HypercoreError extends Error {
   static SESSION_CLOSED (msg = 'Session is closed') {
     return new HypercoreError(msg, 'SESSION_CLOSED', HypercoreError.SESSION_CLOSED)
   }
+
+  static OPLOG_CORRUPT (msg = 'Oplog file appears corrupt or out of date') {
+    return new HypercoreError(msg, 'OPLOG_CORRUPT', HypercoreError.OPLOG_CORRUPT)
+  }
+
+  static INVALID_OPLOG_VERSION (msg = 'Invalid header version') {
+    return new HypercoreError(msg, 'INVALID_OPLOG_VERSION', HypercoreError.INVALID_OPLOG_VERSION)
+  }
 }

package/lib/info.js

@@ -7,9 +7,10 @@ module.exports = class Info {
     this.byteLength = opts.byteLength || 0
     this.fork = opts.fork || 0
     this.padding = opts.padding || 0
+    this.storage = opts.storage || null
   }
 
-  static async from (session) {
+  static async from (session, opts = {}) {
     return new Info({
       key: session.key,
       discoveryKey: session.discoveryKey,
@@ -17,7 +18,36 @@ module.exports = class Info {
       contiguousLength: session.contiguousLength,
       byteLength: session.byteLength,
       fork: session.fork,
-      padding: session.padding
+      padding: session.padding,
+      storage: opts.storage ? await this.storage(session) : null
     })
   }
+
+  static async storage (session) {
+    const { oplog, tree, blocks, bitfield } = session.core
+    try {
+      return {
+        oplog: await bytesUsed(oplog.storage),
+        tree: await bytesUsed(tree.storage),
+        blocks: await bytesUsed(blocks.storage),
+        bitfield: await bytesUsed(bitfield.storage)
+      }
+    } catch {
+      return null
+    }
+
+    function bytesUsed (file) {
+      return new Promise((resolve, reject) => {
+        file.stat((err, st) => {
+          if (err) {
+            resolve(0) // prob just file not found (TODO, improve)
+          } else if (typeof st.blocks !== 'number') {
+            reject(new Error('cannot determine bytes used'))
+          } else {
+            resolve(st.blocks * 512)
+          }
+        })
+      })
+    }
+  }
 }

package/lib/merkle-tree.js

@@ -4,6 +4,7 @@ const c = require('compact-encoding')
 const Xache = require('xache')
 const b4a = require('b4a')
 const caps = require('./caps')
+const { INVALID_PROOF, INVALID_CHECKSUM, INVALID_OPERATION, BAD_ARGUMENT } = require('./errors')
 
 const BLANK_HASH = b4a.alloc(32)
 const OLD_TREE = b4a.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
@@ -26,12 +27,12 @@ class NodeQueue {
     }
 
     if (this.i >= this.nodes.length) {
-      throw new Error('Expected node ' + index + ', got (nil)')
+      throw INVALID_OPERATION('Expected node ' + index + ', got (nil)')
     }
 
     const node = this.nodes[this.i++]
     if (node.index !== index) {
-      throw new Error('Expected node ' + index + ', got node ' + node.index)
+      throw INVALID_OPERATION('Expected node ' + index + ', got node ' + node.index)
     }
 
     this.length--
@@ -109,7 +110,7 @@ class MerkleTreeBatch {
   }
 
   commit () {
-    if (!this.commitable()) throw new Error('Tree was modified during batch, refusing to commit')
+    if (!this.commitable()) throw INVALID_OPERATION('Tree was modified during batch, refusing to commit')
 
     if (this.upgraded) this._commitUpgrade()
 
@@ -423,7 +424,7 @@ module.exports = class MerkleTree {
 
     if (node !== undefined) {
       if (node.hash === BLANK_HASH) {
-        if (error) throw new Error('Could not load node: ' + index)
+        if (error) throw INVALID_OPERATION('Could not load node: ' + index)
         return Promise.resolve(null)
       }
       return Promise.resolve(node)
@@ -560,7 +561,7 @@ module.exports = class MerkleTree {
     }
 
     if (!verifyUpgrade(proof, unverified, batch)) {
-      throw new Error('Fork proof not verifiable')
+      throw INVALID_PROOF('Fork proof not verifiable')
     }
 
     for (const root of batch.roots) {
@@ -580,6 +581,27 @@ module.exports = class MerkleTree {
     return batch
   }
 
+  verifyFullyRemote (proof) {
+    // TODO: impl this less hackishly
+    const batch = new MerkleTreeBatch(this)
+
+    batch.fork = proof.fork
+    batch.roots = []
+    batch.length = 0
+    batch.ancestors = 0
+    batch.byteLength = 0
+
+    let unverified = verifyTree(proof, this.crypto, batch.nodes)
+
+    if (proof.upgrade) {
+      if (verifyUpgrade(proof, unverified, batch)) {
+        unverified = null
+      }
+    }
+
+    return batch
+  }
+
   async verify (proof) {
     const batch = new MerkleTreeBatch(this)
 
@@ -594,7 +616,7 @@ module.exports = class MerkleTree {
     if (unverified) {
       const verified = await this.get(unverified.index)
       if (!b4a.equals(verified.hash, unverified.hash)) {
-        throw new Error('Invalid checksum at node ' + unverified.index)
+        throw INVALID_CHECKSUM('Invalid checksum at node ' + unverified.index)
       }
     }
 
@@ -613,10 +635,10 @@ module.exports = class MerkleTree {
     const node = normalizeIndexed(block, hash)
 
     if (from >= to || to > head) {
-      throw new Error('Invalid upgrade')
+      throw INVALID_OPERATION('Invalid upgrade')
     }
     if (seek && upgrade && node !== null && node.index >= from) {
-      throw new Error('Cannot both do a seek and block/hash request when upgrading')
+      throw INVALID_OPERATION('Cannot both do a seek and block/hash request when upgrading')
     }
 
     let subTree = head
@@ -713,7 +735,7 @@ module.exports = class MerkleTree {
   async byteRange (index) {
     const head = 2 * this.length
     if (((index & 1) === 0 ? index : flat.rightSpan(index)) >= head) {
-      throw new Error('Index is out of bounds')
+      throw BAD_ARGUMENT('Index is out of bounds')
     }
     return [await this.byteOffset(index), (await this.get(index)).size]
   }
@@ -862,7 +884,7 @@ function verifyUpgrade ({ fork, upgrade }, blockRoot, batch) {
     const node = extra[i++]
 
     while (node.index !== ite.index) {
-      if (ite.factor === 2) throw new Error('Unexpected node: ' + node.index)
+      if (ite.factor === 2) throw INVALID_OPERATION('Unexpected node: ' + node.index)
       ite.leftChild()
     }
 
@@ -923,14 +945,14 @@ async function seekTrustedTree (tree, root, bytes) {
 async function seekUntrustedTree (tree, root, bytes) {
   const offset = await tree.byteOffset(root)
 
-  if (offset > bytes) throw new Error('Invalid seek')
+  if (offset > bytes) throw INVALID_OPERATION('Invalid seek')
   if (offset === bytes) return root
 
   bytes -= offset
 
   const node = await tree.get(root)
 
-  if (node.size <= bytes) throw new Error('Invalid seek')
+  if (node.size <= bytes) throw INVALID_OPERATION('Invalid seek')
 
   return seekTrustedTree(tree, root, bytes)
 }
@@ -1061,7 +1083,7 @@ function nodesToRoot (index, nodes, head) {
 
   for (let i = 0; i < nodes; i++) {
     ite.parent()
-    if (ite.contains(head)) throw new Error('Nodes is out of bounds')
+    if (ite.contains(head)) throw BAD_ARGUMENT('Nodes is out of bounds')
   }
 
   return ite.index

package/lib/messages.js

@@ -1,5 +1,6 @@
 const c = require('compact-encoding')
 const b4a = require('b4a')
+const { INVALID_OPLOG_VERSION } = require('./errors')
 
 const EMPTY = b4a.alloc(0)
 
@@ -637,7 +638,7 @@ oplog.header = {
     const version = c.uint.decode(state)
 
     if (version !== 0) {
-      throw new Error('Invalid header version. Expected 0, got ' + version)
+      throw INVALID_OPLOG_VERSION('Invalid header version. Expected 0, got ' + version)
     }
 
     return {

package/lib/oplog.js

@@ -1,6 +1,7 @@
 const cenc = require('compact-encoding')
 const b4a = require('b4a')
 const { crc32 } = require('crc-universal')
+const { OPLOG_CORRUPT } = require('./errors')
 
 module.exports = class Oplog {
   constructor (storage, { pageSize = 4096, headerEncoding = cenc.raw, entryEncoding = cenc.raw } = {}) {
@@ -77,7 +78,7 @@ module.exports = class Oplog {
       this._headers[1] = 0
 
       if (buffer.byteLength >= this._entryOffset) {
-        throw new Error('Oplog file appears corrupt or out of date')
+        throw OPLOG_CORRUPT()
       }
       return result
     }

package/lib/replicator.js

@@ -250,6 +250,8 @@ class Peer {
     this.protomux = protomux
     this.remotePublicKey = this.stream.remotePublicKey
 
+    this.paused = false
+
     this.session = session
 
     this.channel = channel
@@ -518,7 +520,32 @@ class Peer {
     })
   }
 
+  _checkIfConflict (err) {
+    this.paused = true
+
+    const length = Math.min(this.core.tree.length, this.remoteLength)
+    if (length === 0) throw err
+
+    this.wireRequest.send({
+      id: 0, // TODO: use an more explicit id for this eventually...
+      fork: this.remoteFork,
+      block: null,
+      hash: null,
+      seek: null,
+      upgrade: {
+        start: 0,
+        length
+      }
+    })
+  }
+
   async ondata (data) {
+    // always allow a fork conflict proof to be sent
+    if (data.request === 0 && data.upgrade && data.upgrade.start === 0) {
+      if (await this.core.checkConflict(data, this)) return
+      this.paused = false
+    }
+
     const req = data.request > 0 ? this.replicator._inflight.get(data.request) : null
     const reorg = data.fork > this.core.tree.fork
 
@@ -542,8 +569,11 @@ class Peer {
         return
       }
     } catch (err) {
+      safetyCatch(err)
+      // might be a fork, verify
+      this._checkIfConflict(err)
       this.replicator._onnodata(this, req)
-      throw err
+      return
     } finally {
       this.dataProcessing--
     }
@@ -597,6 +627,25 @@ class Peer {
     this.replicator.updatePeer(this)
   }
 
+  async _onconflict () {
+    this.protomux.cork()
+    if (this.remoteLength > 0 && this.core.tree.fork === this.remoteFork) {
+      await this.onrequest({
+        id: 0,
+        fork: this.core.tree.fork,
+        block: null,
+        hash: null,
+        seek: null,
+        upgrade: {
+          start: 0,
+          length: Math.min(this.core.tree.length, this.remoteLength)
+        }
+      })
+    }
+    this.channel.close()
+    this.protomux.uncork()
+  }
+
   _makeRequest (needsUpgrade) {
     if (needsUpgrade === true && this.replicator._shouldUpgrade(this) === false) {
       return null
@@ -901,6 +950,15 @@ module.exports = class Replicator {
     if (this._ranges.length !== 0 || this._seeks.length !== 0) this._updateNonPrimary()
   }
 
+  // Called externally when a conflict has been detected and verified
+  async onconflict (from) {
+    const all = []
+    for (const peer of this.peers) {
+      all.push(peer._onconflict())
+    }
+    await Promise.allSettled(all)
+  }
+
   addUpgrade (session) {
     if (this._upgrade !== null) {
       const ref = this._upgrade.attach(session)
@@ -973,10 +1031,10 @@ module.exports = class Replicator {
     ref.context.detach(ref, null)
   }
 
-  clearRequests (session) {
+  clearRequests (session, err = null) {
     while (session.length > 0) {
       const ref = session[session.length - 1]
-      ref.context.detach(ref, null)
+      ref.context.detach(ref, err)
     }
 
     this.updateAll()
@@ -1418,7 +1476,7 @@ module.exports = class Replicator {
   }
 
   _updatePeer (peer) {
-    if (peer.inflight >= peer.maxInflight) {
+    if (peer.paused || peer.inflight >= peer.maxInflight) {
       return false
     }
 
@@ -1444,7 +1502,7 @@ module.exports = class Replicator {
   }
 
   _updatePeerNonPrimary (peer) {
-    if (peer.inflight >= peer.maxInflight) {
+    if (peer.paused || peer.inflight >= peer.maxInflight) {
       return false
     }
 
@@ -1523,6 +1581,13 @@ module.exports = class Replicator {
     })
   }
 
+  destroy () {
+    for (const peer of this.peers) {
+      peer.protomux.unpair({ protocol: 'hypercore/alpha', id: this.discoveryKey })
+      peer.channel.close()
+    }
+  }
+
   _makePeer (protomux, session) {
     if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) return onnochannel()
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.4.1",
+  "version": "10.5.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -35,7 +35,7 @@
   "dependencies": {
     "@hyperswarm/secret-stream": "^6.0.0",
     "b4a": "^1.1.0",
-    "big-sparse-array": "^1.0.2",
+    "big-sparse-array": "^1.0.3",
     "compact-encoding": "^2.11.0",
     "crc-universal": "^1.0.2",
     "events": "^3.3.0",
@@ -55,7 +55,7 @@
   "devDependencies": {
     "brittle": "^3.0.0",
     "hyperswarm": "^4.3.0",
-    "random-access-memory": "^6.0.0",
+    "random-access-memory": "^6.1.0",
     "random-access-memory-overlay": "^3.0.0",
     "standard": "^17.0.0",
     "tmp-promise": "^3.0.2"