hypercore 10.24.4 → 10.24.6

package/index.js

@@ -629,7 +629,7 @@ module.exports = class Hypercore extends EventEmitter {
         this.replicator.ontruncate(bitfield.start)
       }
 
-      if ((status & 0b0011) !== 0) {
+      if ((status & 0b10011) !== 0) {
         this.replicator.onupgrade()
       }
 
@@ -958,12 +958,8 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   async append (blocks, opts = {}) {
-    if (this._batch && this !== this._batch.session) throw BATCH_UNFLUSHED()
-
-    // if a batched append, run unlocked as it already locked...
-    const lock = !this._batch
-
     if (this.opened === false) await this.opening
+    if (this._batch) throw BATCH_UNFLUSHED()
 
     const { keyPair = this.keyPair, signature = null } = opts
     const writable = !this._readonly && !!(signature || (keyPair && keyPair.secretKey))
@@ -982,7 +978,7 @@ module.exports = class Hypercore extends EventEmitter {
       }
     }
 
-    return this.core.append(buffers, { lock, keyPair, signature, preappend })
+    return this.core.append(buffers, { keyPair, signature, preappend })
   }
 
   async treeHash (length) {

package/lib/batch.js

@@ -1,7 +1,6 @@
 const { BLOCK_NOT_AVAILABLE, SESSION_CLOSED } = require('hypercore-errors')
 const EventEmitter = require('events')
 const c = require('compact-encoding')
-const b4a = require('b4a')
 
 module.exports = class HypercoreBatch extends EventEmitter {
   constructor (session, autoClose) {
@@ -76,7 +75,6 @@ module.exports = class HypercoreBatch extends EventEmitter {
   async update (opts) {
     if (this.opened === false) await this.ready()
     await this.session.update(opts)
-    await this._catchup()
   }
 
   setUserData (key, value, opts) {
@@ -253,64 +251,26 @@ module.exports = class HypercoreBatch extends EventEmitter {
     return flushed
   }
 
-  async _catchup () {
-    if (this.core.tree.length === this._sessionLength) return true
-
-    const batchLength = this._sessionLength + this._appends.length
-
-    if (this.core.tree.length > batchLength) return false
-
-    const b = this.createTreeBatch()
-
-    for (const root of this.core.tree.roots) {
-      const batchRoot = await b.get(root.index)
-
-      if (batchRoot === null) continue // in the shared tree
-      if (batchRoot.size !== root.size || !b4a.equals(batchRoot.hash, root.hash)) {
-        // TODO: type this error
-        throw new Error('Batch is uncommitable due to a conflict at ' + root.index)
-      }
-    }
-
-    const offset = this.core.tree.length - this._sessionLength
-    for (let i = 0; i < offset; i++) this._byteLength -= this._appends[i].byteLength
-
-    await this.core.insertValuesUnsafe(this._appends.slice(0, offset), this._sessionLength, this._sessionByteLength, b.nodes)
-
-    this._sessionLength = this.session.length
-    this._sessionByteLength = this.session.byteLength
-    this._sessionBatch = this.session.createTreeBatch()
-    this._appends = this._appends.slice(offset)
-
-    return true
-  }
-
   async _flush (length, keyPair, signature) { // TODO: make this safe to interact with a parallel truncate...
-    if (this._appends.length === 0) return true
-    if (!(await this._catchup())) return false
+    const flushingLength = Math.min(length - this._sessionLength, this._appends.length)
+    if (flushingLength <= 0) return true
 
-    await this.core._mutex.lock()
+    const batch = this.createTreeBatch(this._sessionLength + flushingLength)
+    if (batch === null) return false
 
-    try {
-      const flushingLength = Math.min(length - this._sessionLength, this._appends.length)
-      if (flushingLength <= 0) return true
+    const info = await this.core.insertBatch(batch, this._appends, { keyPair, signature })
+    if (info === null) return false
 
-      const blocks = flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends
+    const delta = info.byteLength - this._sessionByteLength
 
-      const info = await this.session.append(blocks, { keyPair, signature })
-      const delta = info.byteLength - this._sessionByteLength
-
-      this._sessionLength = info.length
-      this._sessionByteLength = info.byteLength
-      this._sessionBatch = this.session.createTreeBatch()
+    this._sessionLength = info.length
+    this._sessionByteLength = info.byteLength
+    this._sessionBatch = this.session.createTreeBatch()
 
-      this._appends = this._appends.slice(flushingLength)
-      this._byteLength -= delta
+    this._appends = this._appends.slice(flushingLength)
+    this._byteLength -= delta
 
-      this.emit('flush')
-    } finally {
-      this.core._mutex.unlock()
-    }
+    this.emit('flush')
 
     return true
   }

package/lib/core.js

@@ -432,40 +432,71 @@ module.exports = class Core {
     })
   }
 
-  // assumes mutex is already acquried
-  async insertValuesUnsafe (values, offset, byteOffset, treeNodes) {
-    if (!values.length) return
+  async insertBatch (batch, values, { signature, keyPair = this.header.keyPair } = {}) {
+    await this._mutex.lock()
 
-    const bitfield = {
-      drop: false,
-      start: offset,
-      end: offset + values.length
-    }
+    try {
+      // upsert compat manifest
+      if (this.verifier === null && keyPair) this.setManifest(null, keyPair)
 
-    await this.blocks.putBatch(offset, values, byteOffset)
-    await this.oplog.append({
-      userData: null,
-      treeNodes,
-      treeUpgrade: null,
-      bitfield
-    }, false)
+      if (this.tree.fork !== batch.fork) return null
 
-    this.bitfield.setRange(bitfield.start, values.length, true)
-    for (const node of treeNodes) this.tree.unflushed.set(node.index, node)
+      if (this.tree.length > batch.treeLength) {
+        if (this.tree.length > batch.length) return null // TODO: partial commit in the future if possible
 
-    const status = updateContig(this.header, bitfield, this.bitfield)
-    this.onupdate(status, bitfield, null, null)
+        for (const root of this.tree.roots) {
+          const batchRoot = await batch.get(root.index)
+          if (batchRoot.size !== root.size || !b4a.equals(batchRoot.hash, root.hash)) {
+            return null
+          }
+        }
+      }
 
-    if (this._shouldFlush()) await this._flushOplog()
-  }
+      const offset = batch.treeLength
+      const adding = batch.length - batch.treeLength
 
-  async append (values, { lock, signature, keyPair = this.header.keyPair, preappend } = {}) {
-    if (lock) await this._mutex.lock()
+      batch.upgraded = batch.length > this.tree.length
+      batch.treeLength = this.tree.length
+      if (batch.upgraded) batch.signature = signature || this.verifier.sign(batch, keyPair)
 
-    // upsert compat manifest
-    if (this.verifier === null && keyPair) this.setManifest(null, keyPair)
+      let byteOffset = batch.byteLength
+      for (let i = 0; i < adding; i++) byteOffset -= values[i].byteLength
+
+      const entry = {
+        userData: null,
+        treeNodes: batch.nodes,
+        treeUpgrade: batch.upgraded ? batch : null,
+        bitfield: {
+          drop: false,
+          start: offset,
+          length: adding
+        }
+      }
+
+      await this.blocks.putBatch(offset, adding > values.length ? values.slice(0, adding) : values, byteOffset)
+      await this.oplog.append([entry], false)
+
+      this.bitfield.setRange(entry.bitfield.start, entry.bitfield.length, true)
+      batch.commit()
+
+      const status = (batch.upgraded ? 0b0001 : 0) | updateContig(this.header, entry.bitfield, this.bitfield)
+      this.onupdate(status, entry.bitfield, null, null)
+
+      if (this._shouldFlush()) await this._flushOplog()
+    } finally {
+      this._mutex.unlock()
+    }
+
+    return { length: batch.length, byteLength: batch.byteLength }
+  }
+
+  async append (values, { signature, keyPair = this.header.keyPair, preappend } = {}) {
+    await this._mutex.lock()
 
     try {
+      // upsert compat manifest
+      if (this.verifier === null && keyPair) this.setManifest(null, keyPair)
+
       if (preappend) await preappend(values)
 
       if (!values.length) {
@@ -506,7 +537,7 @@ module.exports = class Core {
 
       return { length: batch.length, byteLength }
     } finally {
-      if (lock) this._mutex.unlock()
+      this._mutex.unlock()
     }
   }
 

package/lib/merkle-tree.js

@@ -1188,6 +1188,12 @@ async function generateProof (tree, block, hash, seek, upgrade) {
   const from = upgrade ? upgrade.start * 2 : 0
   const to = upgrade ? from + upgrade.length * 2 : head
   const node = normalizeIndexed(block, hash)
+
+  const result = { fork, block: null, hash: null, seek: null, upgrade: null, manifest: null }
+
+  // can't do anything as we have no data...
+  if (head === 0) return result
+
   if (from >= to || to > head) {
     throw INVALID_OPERATION('Invalid upgrade')
   }
@@ -1218,7 +1224,6 @@ async function generateProof (tree, block, hash, seek, upgrade) {
   }
 
   const [pNode, pSeek, pUpgrade, pAdditional] = await settleProof(p)
-  const result = { fork, block: null, hash: null, seek: null, upgrade: null, manifest: null }
 
   if (block) {
     result.block = {

package/lib/mutex.js

@@ -10,7 +10,7 @@ module.exports = class Mutex {
   }
 
   lock () {
-    if (this.destroyed) return Promise.reject(this._destroyError)
+    if (this.destroyed) return Promise.reject(this._destroyError || new Error('Mutex has been destroyed'))
     if (this.locked) return new Promise(this._enqueue)
     this.locked = true
     return Promise.resolve()
@@ -28,7 +28,7 @@ module.exports = class Mutex {
     if (!this._destroying) this._destroying = this.locked ? this.lock().catch(() => {}) : Promise.resolve()
 
     this.destroyed = true
-    this._destroyError = err || new Error('Mutex has been destroyed')
+    if (err) this._destroyError = err
 
     if (err) {
       while (this._queue.length) this._queue.shift()[1](err)

package/lib/replicator.js

@@ -384,7 +384,7 @@ class Peer {
       remoteLength: this.core.tree.fork === this.remoteFork ? this.remoteLength : 0,
       canUpgrade: this.canUpgrade,
       uploading: true,
-      downloading: this.replicator.downloading,
+      downloading: this.replicator.isDownloading(),
       hasManifest: !!this.core.header.manifest && this.core.compat === false
     })
   }
@@ -582,9 +582,9 @@ class Peer {
     if (msg.fork === this.core.tree.fork) {
       try {
         proof = await this._getProof(msg)
-      } catch (err) { // TODO: better error handling here, ie custom errors
+      } catch (err) {
         safetyCatch(err)
-        if (this.replicator.core.closed) throw err // just an extra safety check...
+        if (isCriticalError(err)) throw err
       }
     }
 
@@ -619,7 +619,7 @@ class Peer {
     if (!exists) return
 
     this.inflight--
-    this.replicator._inflight.remove(id)
+    this.replicator._removeInflight(id)
 
     this.wireCancel.send({ request: id })
   }
@@ -660,7 +660,7 @@ class Peer {
     if (req !== null) {
       if (req.peer !== this) return
       this.inflight--
-      this.replicator._inflight.remove(req.id)
+      this.replicator._removeInflight(req.id)
     }
 
     if (reorg === true) return this.replicator._onreorgdata(this, req, data)
@@ -674,6 +674,8 @@ class Peer {
       }
     } catch (err) {
       safetyCatch(err)
+      if (this.core.closed && !isCriticalError(err)) return
+
       // might be a fork, verify
       this._checkIfConflict(err)
       this.replicator._onnodata(this, req)
@@ -695,7 +697,7 @@ class Peer {
     if (req === null || req.peer !== this) return
 
     this.inflight--
-    this.replicator._inflight.remove(req.id)
+    this.replicator._removeInflight(req.id)
     this.replicator._onnodata(this, req)
   }
 
@@ -1072,10 +1074,15 @@ module.exports = class Replicator {
     }
   }
 
+  isDownloading () {
+    return this.downloading || !this._inflight.idle
+  }
+
   setDownloading (downloading) {
     if (this.downloading === downloading) return
     this.downloading = downloading
-    for (const peer of this.peers) peer.sendSync()
+    if (!downloading && this.isDownloading()) return
+    for (const peer of this.peers) peer.signalUpgrade()
   }
 
   cork () {
@@ -1319,6 +1326,13 @@ module.exports = class Replicator {
     this.onpeerupdate(true, peer)
   }
 
+  _removeInflight (id) {
+    this._inflight.remove(id)
+    if (this.isDownloading() === false) {
+      for (const peer of this.peers) peer.signalUpgrade()
+    }
+  }
+
   _removePeer (peer) {
     this.peers.splice(this.peers.indexOf(peer), 1)
 
@@ -1904,6 +1918,11 @@ function destroyRequestTimeout (req) {
   }
 }
 
+function isCriticalError (err) {
+  // TODO: expose .critical or similar on the hypercore errors that are critical (if all not are)
+  return err.name === 'HypercoreError'
+}
+
 function onwireopen (m, c) {
   return c.userData.onopen(m)
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.24.4",
+  "version": "10.24.6",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {