hypercore 10.24.1 → 10.24.2

package/index.js

@@ -135,7 +135,7 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   static key (manifest, { compat } = {}) {
-    return compat ? manifest.signer.publicKey : manifestHash(manifest)
+    return compat ? manifest.signer.publicKey : manifestHash(createManifest(manifest))
   }
 
   static discoveryKey (key) {
@@ -960,6 +960,9 @@ module.exports = class Hypercore extends EventEmitter {
   async append (blocks, opts = {}) {
     if (this._batch && this !== this._batch.session) throw BATCH_UNFLUSHED()
 
+    // if a batched append, run unlocked as it already locked...
+    const lock = !this._batch
+
     if (this.opened === false) await this.opening
 
     const { keyPair = this.keyPair, signature = null } = opts
@@ -979,7 +982,7 @@ module.exports = class Hypercore extends EventEmitter {
       }
     }
 
-    return this.core.append(buffers, { keyPair, signature, preappend })
+    return this.core.append(buffers, { lock, keyPair, signature, preappend })
   }
 
   async treeHash (length) {

package/lib/batch.js

@@ -1,6 +1,7 @@
 const { BLOCK_NOT_AVAILABLE, SESSION_CLOSED } = require('hypercore-errors')
 const EventEmitter = require('events')
 const c = require('compact-encoding')
+const b4a = require('b4a')
 
 module.exports = class HypercoreBatch extends EventEmitter {
   constructor (session, autoClose) {
@@ -19,6 +20,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
     this._byteLength = 0
     this._sessionLength = 0
     this._sessionByteLength = 0
+    this._sessionBatch = null
     this._flushing = null
 
     this.opening = this.ready().catch(noop)
@@ -65,6 +67,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
     if (this.opened) return
     this._sessionLength = this.session.length
     this._sessionByteLength = this.session.byteLength
+    this._sessionBatch = this.session.createTreeBatch()
     this.fork = this.session.fork
     this.opened = true
     this.emit('ready')
@@ -149,7 +152,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
     if (!length && length !== 0) length = this.length + blocks.length
 
     const maxLength = this.length + blocks.length
-    const b = this.session.createTreeBatch()
+    const b = this._sessionBatch.clone()
     const len = Math.min(length, this.length)
 
     if (len < this._sessionLength || length > maxLength) return null
@@ -185,6 +188,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
       await this.session.truncate(newLength, { fork, force: true, ...opts })
       this._sessionLength = this.session.length
       this._sessionByteLength = this.session.byteLength
+      this._sessionBatch = this.session.createTreeBatch()
     } else {
       for (let i = newLength - length; i < this._appends.length; i++) this._byteLength -= this._appends[i].byteLength
       this._appends.length = newLength - length
@@ -230,34 +234,83 @@ module.exports = class HypercoreBatch extends EventEmitter {
     if (this.opened === false) await this.opening
     if (this.closing) throw SESSION_CLOSED()
 
-    const { length = this._appends.length, keyPair = this.session.keyPair, signature = null } = opts
+    const { length = this.length, keyPair = this.session.keyPair, signature = null } = opts
 
     while (this._flushing) await this._flushing
     this._flushing = this._flush(length, keyPair, signature)
 
+    let flushed = false
+
     try {
-      await this._flushing
+      flushed = await this._flushing
     } finally {
       this._flushing = null
     }
 
     if (this.autoClose) await this.close()
+
+    return flushed
+  }
+
+  async _catchup () {
+    if (this.core.tree.length === this._sessionLength) return true
+
+    const batchLength = this._sessionLength + this._appends.length
+
+    if (this.core.tree.length > batchLength) return false
+
+    const b = this.createTreeBatch()
+
+    for (const root of this.core.tree.roots) {
+      const batchRoot = await b.get(root.index)
+
+      if (batchRoot === null) continue // in the shared tree
+      if (batchRoot.size !== root.size || !b4a.equals(batchRoot.hash, root.hash)) {
+        // TODO: type this error
+        throw new Error('Batch is uncommitable due to a conflict at ' + root.index)
+      }
+    }
+
+    const offset = this._sessionLength + this._appends.length - this.core.tree.length
+    for (let i = 0; i < offset; i++) this._byteLength -= this._appends[i]
+
+    this._sessionLength = this.session.length
+    this._sessionByteLength = this.session.byteLength
+    this._sessionBatch = this.session.createTreeBatch()
+
+    if (offset) this._appends = this._appends.slice(this._appends.length - offset)
+
+    return true
   }
 
   async _flush (length, keyPair, signature) { // TODO: make this safe to interact with a parallel truncate...
-    if (this._appends.length === 0) return
+    if (this._appends.length === 0) return true
+    if (!(await this._catchup())) return false
+
+    await this.core._mutex.lock()
 
-    const flushingLength = Math.min(length, this._appends.length)
-    const blocks = flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends
-    const info = await this.session.append(blocks, { keyPair, signature })
-    const delta = info.byteLength - this._sessionByteLength
+    try {
+      const flushingLength = Math.min(length - this._sessionLength, this._appends.length)
+      if (flushingLength <= 0) return true
 
-    this._sessionLength = info.length
-    this._sessionByteLength = info.byteLength
-    this._appends = this._appends.slice(flushingLength)
-    this._byteLength -= delta
+      const blocks = flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends
+
+      const info = await this.session.append(blocks, { keyPair, signature })
+      const delta = info.byteLength - this._sessionByteLength
+
+      this._sessionLength = info.length
+      this._sessionByteLength = info.byteLength
+      this._sessionBatch = this.session.createTreeBatch()
+
+      this._appends = this._appends.slice(flushingLength)
+      this._byteLength -= delta
+
+      this.emit('flush')
+    } finally {
+      this.core._mutex.unlock()
+    }
 
-    this.emit('flush')
+    return true
   }
 
   close () {

package/lib/core.js

@@ -432,8 +432,8 @@ module.exports = class Core {
     })
   }
 
-  async append (values, { signature, keyPair = this.header.keyPair, preappend } = {}) {
-    await this._mutex.lock()
+  async append (values, { lock, signature, keyPair = this.header.keyPair, preappend } = {}) {
+    if (lock) await this._mutex.lock()
 
     // upsert compat manifest
     if (this.verifier === null && keyPair) this.setManifest(null, keyPair)
@@ -479,7 +479,7 @@ module.exports = class Core {
 
       return { length: batch.length, byteLength }
     } finally {
-      this._mutex.unlock()
+      if (lock) this._mutex.unlock()
     }
   }
 

package/lib/merkle-tree.js

@@ -92,14 +92,14 @@ class MerkleTreeBatch {
       return null
     }
 
-    if (index < this.treeLength * 2) {
-      return this.tree.get(index)
-    }
-
     for (const n of this.nodes) {
       if (n.index === index) return n
     }
 
+    if (index < this.treeLength * 2) {
+      return this.tree.get(index)
+    }
+
     return null
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.24.1",
+  "version": "10.24.2",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {