hypercore 10.30.4 → 10.31.0

package/index.js

@@ -391,7 +391,8 @@ module.exports = class Hypercore extends EventEmitter {
       eagerUpgrade: true,
       allowFork: opts.allowFork !== false,
       onpeerupdate: this._onpeerupdate.bind(this),
-      onupload: this._onupload.bind(this)
+      onupload: this._onupload.bind(this),
+      oninvalid: this._oninvalid.bind(this)
     })
 
     this.replicator.findingPeers += this._findingPeers
@@ -589,7 +590,7 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   get contiguousLength () {
-    return this.core === null ? 0 : this.core.header.hints.contiguousLength
+    return this.core === null ? 0 : Math.min(this.core.tree.length, this.core.header.hints.contiguousLength)
   }
 
   get contiguousByteLength () {
@@ -624,6 +625,12 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
+  _oninvalid (err, req, res, from) {
+    for (let i = 0; i < this.sessions.length; i++) {
+      this.sessions[i].emit('verification-error', err, req, res, from)
+    }
+  }
+
   async _oncoreconflict (proof, from) {
     await this.replicator.onconflict(from)
 
@@ -800,14 +807,15 @@ module.exports = class Hypercore extends EventEmitter {
     return true
   }
 
-  batch ({ checkout = -1, autoClose = true, session = true } = {}) {
-    return new Batch(session ? this.session() : this, checkout, autoClose)
+  batch ({ checkout = -1, autoClose = true, session = true, restore = false } = {}) {
+    return new Batch(session ? this.session() : this, checkout, autoClose, restore)
   }
 
   async seek (bytes, opts) {
     if (this.opened === false) await this.opening
 
-    const s = this.core.tree.seek(bytes, this.padding)
+    const tree = (opts && opts.tree) || this.core.tree
+    const s = tree.seek(bytes, this.padding)
 
     const offset = await s.update()
     if (offset) return offset
@@ -887,7 +895,8 @@ module.exports = class Hypercore extends EventEmitter {
     let block
 
     if (this.core.bitfield.get(index)) {
-      block = this.core.blocks.get(index)
+      const tree = (opts && opts.tree) || this.core.tree
+      block = this.core.blocks.get(index, tree)
 
       if (this.cache) this.cache.set(index, block)
     } else {

package/lib/batch.js

@@ -4,7 +4,7 @@ const c = require('compact-encoding')
 const b4a = require('b4a')
 
 module.exports = class HypercoreBatch extends EventEmitter {
-  constructor (session, checkoutLength, autoClose) {
+  constructor (session, checkoutLength, autoClose, restore) {
     super()
 
     this.session = session
@@ -14,6 +14,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
     this.closing = null
     this.writable = true // always writable...
     this.autoClose = autoClose
+    this.restore = restore
     this.fork = 0
 
     this._appends = []
@@ -41,7 +42,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
   }
 
   get indexedLength () {
-    return this._sessionLength
+    return Math.min(this._sessionLength, this.session.core === null ? 0 : this.session.core.tree.length)
   }
 
   get indexedByteLength () {
@@ -77,9 +78,18 @@ module.exports = class HypercoreBatch extends EventEmitter {
       this._sessionByteLength = batch.byteLength
       this._sessionBatch = batch
     } else {
-      this._sessionLength = this.session.length
-      this._sessionByteLength = this.session.byteLength
-      this._sessionBatch = this.session.createTreeBatch()
+      const last = this.restore ? this.session.core.bitfield.findFirst(false, this.session.length) : 0
+
+      if (last > this.session.length) {
+        const batch = await this.session.core.tree.restoreBatch(last)
+        this._sessionLength = batch.length
+        this._sessionByteLength = batch.byteLength - this.session.padding * batch.length
+        this._sessionBatch = batch
+      } else {
+        this._sessionLength = this.session.length
+        this._sessionByteLength = this.session.byteLength
+        this._sessionBatch = this.session.createTreeBatch()
+      }
     }
 
     this._appendsActual = this.session.encryption ? [] : this._appends
@@ -128,11 +138,11 @@ module.exports = class HypercoreBatch extends EventEmitter {
     return info
   }
 
-  async seek (bytes, opts) {
+  async seek (bytes, opts = {}) {
     if (this.opened === false) await this.opening
     if (this.closing) throw SESSION_CLOSED()
 
-    if (bytes < this._sessionByteLength) return await this.session.seek(bytes, opts)
+    if (bytes < this._sessionByteLength) return await this.session.seek(bytes, { ...opts, tree: this._sessionBatch })
 
     bytes -= this._sessionByteLength
 
@@ -154,7 +164,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
     if (this.closing) throw SESSION_CLOSED()
 
     const length = this._sessionLength
-    if (index < length) return this.session.get(index, opts)
+    if (index < length) return this.session.get(index, { ...opts, tree: this._sessionBatch })
 
     const buffer = this._appends[index - length] || null
     if (!buffer) throw BLOCK_NOT_AVAILABLE()
@@ -173,6 +183,11 @@ module.exports = class HypercoreBatch extends EventEmitter {
     }
   }
 
+  async restoreBatch (length) {
+    if (this.opened === false) await this.opening
+    return this.session.core.tree.restoreBatch(length)
+  }
+
   createTreeBatch (length, blocks = []) {
     if (!length && length !== 0) length = this.length + blocks.length
 
@@ -293,10 +308,10 @@ module.exports = class HypercoreBatch extends EventEmitter {
     if (this.opened === false) await this.opening
     if (this.closing) throw SESSION_CLOSED()
 
-    const { length = this.length, keyPair = this.session.keyPair, signature = null } = opts
+    const { length = this.length, keyPair = this.session.keyPair, signature = null, pending = !signature && !keyPair } = opts
 
     while (this._flushing) await this._flushing
-    this._flushing = this._flush(length, keyPair, signature)
+    this._flushing = this._flush(length, keyPair, signature, pending)
 
     let flushed = false
 
@@ -311,12 +326,9 @@ module.exports = class HypercoreBatch extends EventEmitter {
     return flushed
   }
 
-  async _flush (length, keyPair, signature) { // TODO: make this safe to interact with a parallel truncate...
+  async _flush (length, keyPair, signature, pending) { // TODO: make this safe to interact with a parallel truncate...
     if (this._sessionBatch.fork !== this.session.fork) return false // no truncs supported atm
 
-    const flushingLength = Math.min(length - this._sessionLength, this._appends.length)
-    if (flushingLength <= 0) return true
-
     if (this.session.replicator._upgrade) {
       for (const req of this.session.replicator._upgrade.inflight) {
         // yield to the remote inflight upgrade, TODO: if the remote upgrade fails, retry flushing...
@@ -326,17 +338,28 @@ module.exports = class HypercoreBatch extends EventEmitter {
       }
     }
 
+    const flushingLength = Math.min(length - this._sessionLength, this._appends.length)
+    if (flushingLength <= 0) {
+      if (this._sessionLength > this.core.tree.length && length > this.core.tree.length && !pending) {
+        const batch = await this.restoreBatch(length)
+        const info = await this.core.insertBatch(batch, [], { keyPair, signature, pending, treeLength: length })
+        return info !== null
+      }
+      return true
+    }
+
     const batch = this.createTreeBatch(this._sessionLength + flushingLength)
     if (batch === null) return false
 
-    const info = await this.core.insertBatch(batch, this._appendsActual, { keyPair, signature })
+    const info = await this.core.insertBatch(batch, this._appendsActual, { keyPair, signature, pending, treeLength: this._sessionLength })
     if (info === null) return false
 
     const delta = info.byteLength - this._sessionByteLength
+    const newBatch = info.length !== this.session.length ? await this.restoreBatch(info.length) : this.session.createTreeBatch()
 
     this._sessionLength = info.length
     this._sessionByteLength = info.byteLength
-    this._sessionBatch = this.session.createTreeBatch()
+    this._sessionBatch = newBatch
 
     const same = this._appends === this._appendsActual
 

package/lib/block-store.js

@@ -6,8 +6,9 @@ module.exports = class BlockStore {
     this.tree = tree
   }
 
-  async get (i) {
-    const [offset, size] = await this.tree.byteRange(2 * i)
+  async get (i, tree) {
+    if (!tree) tree = this.tree
+    const [offset, size] = await tree.byteRange(2 * i)
     return this._read(offset, size)
   }
 

package/lib/core.js

@@ -451,7 +451,7 @@ module.exports = class Core {
     })
   }
 
-  async insertBatch (batch, values, { signature, keyPair = this.header.keyPair } = {}) {
+  async insertBatch (batch, values, { signature, keyPair = this.header.keyPair, pending = false, treeLength = batch.treeLength } = {}) {
     await this._mutex.lock()
 
     try {
@@ -471,29 +471,30 @@ module.exports = class Core {
         }
       }
 
-      const offset = batch.treeLength
-      const adding = batch.length - batch.treeLength
+      const adding = batch.length - treeLength
 
       batch.upgraded = batch.length > this.tree.length
       batch.treeLength = this.tree.length
       batch.ancestors = this.tree.length
-      if (batch.upgraded) batch.signature = signature || this.verifier.sign(batch, keyPair)
+      if (batch.upgraded && !pending) batch.signature = signature || this.verifier.sign(batch, keyPair)
 
       let byteOffset = batch.byteLength
       for (let i = 0; i < adding; i++) byteOffset -= values[i].byteLength
 
+      if (pending === true) batch.upgraded = false
+
       const entry = {
         userData: null,
         treeNodes: batch.nodes,
         treeUpgrade: batch.upgraded ? batch : null,
         bitfield: {
           drop: false,
-          start: offset,
+          start: treeLength,
           length: adding
         }
       }
 
-      await this.blocks.putBatch(offset, adding > values.length ? values.slice(0, adding) : values, byteOffset)
+      await this.blocks.putBatch(treeLength, adding > values.length ? values.slice(0, adding) : values, byteOffset)
       await this.oplog.append([entry], false)
 
       this.bitfield.setRange(entry.bitfield.start, entry.bitfield.length, true)
@@ -506,7 +507,7 @@ module.exports = class Core {
       }
 
       const status = (batch.upgraded ? 0b0001 : 0) | updateContig(this.header, entry.bitfield, this.bitfield)
-      this.onupdate(status, entry.bitfield, null, null)
+      if (!pending) this.onupdate(status, entry.bitfield, null, null)
 
       if (this._shouldFlush()) await this._flushOplog()
     } finally {

package/lib/merkle-tree.js

@@ -87,7 +87,7 @@ class MerkleTreeBatch {
     return caps.treeSignableCompat(this.hash(), this.length, this.fork, noHeader)
   }
 
-  get (index) {
+  get (index, error) {
     if (index >= this.length * 2) {
       return null
     }
@@ -96,11 +96,7 @@ class MerkleTreeBatch {
       if (n.index === index) return n
     }
 
-    if (index < this.treeLength * 2) {
-      return this.tree.get(index)
-    }
-
-    return null
+    return this.tree.get(index, error)
   }
 
   proof ({ block, hash, seek, upgrade }) {
@@ -203,40 +199,17 @@ class MerkleTreeBatch {
     this.tree.signature = this.signature
   }
 
-  // TODO: this is the only async method on the batch, so unsure if it should go here
-  // this is important so you know where to right data without committing the batch
-  // so we'll keep it here for now.
-
-  async byteOffset (index) {
-    if (2 * this.tree.length === index) return this.tree.byteLength
-
-    const ite = flat.iterator(index)
-
-    let treeOffset = 0
-    let isRight = false
-    let parent = null
-
-    for (const node of this.nodes) {
-      if (node.index === ite.index) {
-        if (isRight && parent) treeOffset += node.size - parent.size
-        parent = node
-        isRight = ite.isRight()
-        ite.parent()
-      }
-    }
-
-    const r = this.roots.indexOf(parent)
-    if (r > -1) {
-      for (let i = 0; i < r; i++) {
-        treeOffset += this.roots[i].size
-      }
-
-      return treeOffset
-    }
+  seek (bytes, padding) {
+    return new ByteSeeker(this, bytes, padding)
+  }
 
-    const byteOffset = await this.tree.byteOffset(parent ? parent.index : index)
+  byteRange (index) {
+    return getByteRange(this, index)
+  }
 
-    return byteOffset + treeOffset
+  byteOffset (index) {
+    if (index === 2 * this.tree.length) return this.tree.byteLength
+    return getByteOffset(this, index)
   }
 }
 
@@ -337,10 +310,6 @@ class ByteSeeker {
     this.end = bytes < size ? tree.length : 0
   }
 
-  nodes () {
-    return this.tree.nodes(this.start * 2)
-  }
-
   async _seek (bytes) {
     if (!bytes) return [0, 0]
 
@@ -357,6 +326,7 @@ class ByteSeeker {
 
       while ((ite.index & 1) !== 0) {
         const l = await this.tree.get(ite.leftChild(), false)
+
         if (l) {
           const size = getUnpaddedSize(l, this.padding, ite)
 
@@ -415,6 +385,22 @@ module.exports = class MerkleTree {
     return new MerkleTreeBatch(this)
   }
 
+  async restoreBatch (length) {
+    const batch = new MerkleTreeBatch(this)
+    if (length === this.length) return batch
+
+    const roots = await this.getRoots(length)
+
+    batch.roots = roots
+    batch.length = length
+    batch.byteLength = 0
+    batch.ancestors = length
+
+    for (const node of roots) batch.byteLength += node.size
+
+    return batch
+  }
+
   seek (bytes, padding) {
     return new ByteSeeker(this, bytes, padding)
   }
@@ -706,42 +692,12 @@ module.exports = class MerkleTree {
     return cnt
   }
 
-  async byteRange (index) {
-    const head = 2 * this.length
-    if (((index & 1) === 0 ? index : flat.rightSpan(index)) >= head) {
-      throw BAD_ARGUMENT('Index is out of bounds')
-    }
-    return [await this.byteOffset(index), (await this.get(index)).size]
+  byteRange (index) {
+    return getByteRange(this, index)
   }
 
-  async byteOffset (index) {
-    if (index === 2 * this.length) return this.byteLength
-    if ((index & 1) === 1) index = flat.leftSpan(index)
-
-    let head = 0
-    let offset = 0
-
-    for (const node of this.roots) { // all async ticks happen once we find the root so safe
-      head += 2 * ((node.index - head) + 1)
-
-      if (index >= head) {
-        offset += node.size
-        continue
-      }
-
-      const ite = flat.iterator(node.index)
-
-      while (ite.index !== index) {
-        if (index < ite.index) {
-          ite.leftChild()
-        } else {
-          offset += (await this.get(ite.leftChild())).size
-          ite.sibling()
-        }
-      }
-
-      return offset
-    }
+  byteOffset (index) {
+    return getByteOffset(this, index)
   }
 
   static async open (storage, opts = {}) {
@@ -766,6 +722,44 @@ module.exports = class MerkleTree {
   }
 }
 
+async function getByteRange (tree, index) {
+  const head = 2 * tree.length
+  if (((index & 1) === 0 ? index : flat.rightSpan(index)) >= head) {
+    throw BAD_ARGUMENT('Index is out of bounds')
+  }
+  return [await tree.byteOffset(index), (await tree.get(index)).size]
+}
+
+async function getByteOffset (tree, index) {
+  if (index === 2 * tree.length) return tree.byteLength
+  if ((index & 1) === 1) index = flat.leftSpan(index)
+
+  let head = 0
+  let offset = 0
+
+  for (const node of tree.roots) { // all async ticks happen once we find the root so safe
+    head += 2 * ((node.index - head) + 1)
+
+    if (index >= head) {
+      offset += node.size
+      continue
+    }
+
+    const ite = flat.iterator(node.index)
+
+    while (ite.index !== index) {
+      if (index < ite.index) {
+        ite.leftChild()
+      } else {
+        offset += (await tree.get(ite.leftChild())).size
+        ite.sibling()
+      }
+    }
+
+    return offset
+  }
+}
+
 // All the methods needed for proof verification
 
 function verifyTree ({ block, hash, seek }, crypto, nodes) {

package/lib/replicator.js

@@ -408,7 +408,7 @@ class Peer {
 
     this.sendSync()
 
-    const contig = this.core.header.hints.contiguousLength
+    const contig = Math.min(this.core.tree.length, this.core.header.hints.contiguousLength)
     if (contig > 0) {
       this.broadcastRange(0, contig, false)
 
@@ -636,11 +636,11 @@ class Peer {
     this.wireCancel.send({ request: id })
   }
 
-  _checkIfConflict (err) {
+  _checkIfConflict () {
     this.paused = true
 
     const length = Math.min(this.core.tree.length, this.remoteLength)
-    if (length === 0) throw err
+    if (length === 0) return // pause and ignore
 
     this.wireRequest.send({
       id: 0, // TODO: use an more explicit id for this eventually...
@@ -675,7 +675,14 @@ class Peer {
       this.replicator._removeInflight(req.id)
     }
 
-    if (reorg === true) return this.replicator._onreorgdata(this, req, data)
+    try {
+      if (reorg === true) return await this.replicator._onreorgdata(this, req, data)
+    } catch (err) {
+      safetyCatch(err)
+      this.paused = true
+      this.replicator.oninvalid(err, req, data, this)
+      return
+    }
 
     this.dataProcessing++
 
@@ -690,9 +697,10 @@ class Peer {
 
       if (err.code !== 'INVALID_OPERATION') {
         // might be a fork, verify
-        this._checkIfConflict(err)
+        this._checkIfConflict()
       }
       this.replicator._onnodata(this, req)
+      this.replicator.oninvalid(err, req, data, this)
       return
     } finally {
       this.dataProcessing--
@@ -1104,7 +1112,7 @@ class Peer {
 }
 
 module.exports = class Replicator {
-  constructor (core, key, { eagerUpgrade = true, allowFork = true, onpeerupdate = noop, onupload = noop } = {}) {
+  constructor (core, key, { eagerUpgrade = true, allowFork = true, onpeerupdate = noop, onupload = noop, oninvalid = noop } = {}) {
     this.key = key
     this.discoveryKey = core.crypto.discoveryKey(key)
     this.core = core
@@ -1112,6 +1120,7 @@ module.exports = class Replicator {
     this.allowFork = allowFork
     this.onpeerupdate = onpeerupdate
     this.onupload = onupload
+    this.oninvalid = oninvalid
     this.ondownloading = null // optional external hook for monitoring downloading status
     this.peers = []
     this.findingPeers = 0 // updateable from the outside

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.30.4",
+  "version": "10.31.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {