hypercore 10.24.5 → 10.24.7

package/index.js

@@ -85,6 +85,7 @@ module.exports = class Hypercore extends EventEmitter {
     this._snapshot = null
     this._batch = opts._batch || null
     this._findingPeers = 0
+    this._active = opts.active !== false
 
     this.opening = this._openSession(key, storage, opts)
     this.opening.catch(noop)
@@ -323,6 +324,8 @@ module.exports = class Hypercore extends EventEmitter {
     // It's required so that corestore can load a name from userData before 'ready' is emitted.
     if (opts._preready) await opts._preready(this)
 
+    this.replicator.updateActivity(this._active ? 1 : 0, this)
+
     this.opened = true
     this.emit('ready')
   }
@@ -441,6 +444,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (this.replicator !== null) {
       this.replicator.findingPeers -= this._findingPeers
       this.replicator.clearRequests(this.activeRequests, err)
+      this.replicator.updateActivity(this._active ? -1 : 0, this)
     }
 
     this._findingPeers = 0
@@ -530,7 +534,7 @@ module.exports = class Hypercore extends EventEmitter {
   _attachToMuxerOpened (mux, useSession) {
     // If the user wants to, we can make this replication run in a session
     // that way the core wont close "under them" during replication
-    const session = useSession ? this.session() : null
+    const session = useSession ? this.session({ active: false }) : null
     this.replicator.attachTo(mux, session)
   }
 
@@ -958,12 +962,8 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   async append (blocks, opts = {}) {
-    if (this._batch && this !== this._batch.session) throw BATCH_UNFLUSHED()
-
-    // if a batched append, run unlocked as it already locked...
-    const lock = !this._batch
-
     if (this.opened === false) await this.opening
+    if (this._batch) throw BATCH_UNFLUSHED()
 
     const { keyPair = this.keyPair, signature = null } = opts
     const writable = !this._readonly && !!(signature || (keyPair && keyPair.secretKey))
@@ -982,7 +982,7 @@ module.exports = class Hypercore extends EventEmitter {
       }
     }
 
-    return this.core.append(buffers, { lock, keyPair, signature, preappend })
+    return this.core.append(buffers, { keyPair, signature, preappend })
   }
 
   async treeHash (length) {

package/lib/batch.js

@@ -1,7 +1,6 @@
 const { BLOCK_NOT_AVAILABLE, SESSION_CLOSED } = require('hypercore-errors')
 const EventEmitter = require('events')
 const c = require('compact-encoding')
-const b4a = require('b4a')
 
 module.exports = class HypercoreBatch extends EventEmitter {
   constructor (session, autoClose) {
@@ -76,7 +75,6 @@ module.exports = class HypercoreBatch extends EventEmitter {
   async update (opts) {
     if (this.opened === false) await this.ready()
     await this.session.update(opts)
-    await this._catchup()
   }
 
   setUserData (key, value, opts) {
@@ -253,64 +251,26 @@ module.exports = class HypercoreBatch extends EventEmitter {
     return flushed
   }
 
-  async _catchup () {
-    if (this.core.tree.length === this._sessionLength) return true
-
-    const batchLength = this._sessionLength + this._appends.length
-
-    if (this.core.tree.length > batchLength) return false
-
-    const b = this.createTreeBatch()
-
-    for (const root of this.core.tree.roots) {
-      const batchRoot = await b.get(root.index)
-
-      if (batchRoot === null) continue // in the shared tree
-      if (batchRoot.size !== root.size || !b4a.equals(batchRoot.hash, root.hash)) {
-        // TODO: type this error
-        throw new Error('Batch is uncommitable due to a conflict at ' + root.index)
-      }
-    }
-
-    const offset = this.core.tree.length - this._sessionLength
-    for (let i = 0; i < offset; i++) this._byteLength -= this._appends[i].byteLength
-
-    await this.core.insertValuesUnsafe(this._appends.slice(0, offset), this._sessionLength, this._sessionByteLength, b.nodes)
-
-    this._sessionLength = this.session.length
-    this._sessionByteLength = this.session.byteLength
-    this._sessionBatch = this.session.createTreeBatch()
-    this._appends = this._appends.slice(offset)
-
-    return true
-  }
-
   async _flush (length, keyPair, signature) { // TODO: make this safe to interact with a parallel truncate...
-    if (this._appends.length === 0) return true
-    if (!(await this._catchup())) return false
+    const flushingLength = Math.min(length - this._sessionLength, this._appends.length)
+    if (flushingLength <= 0) return true
 
-    await this.core._mutex.lock()
+    const batch = this.createTreeBatch(this._sessionLength + flushingLength)
+    if (batch === null) return false
 
-    try {
-      const flushingLength = Math.min(length - this._sessionLength, this._appends.length)
-      if (flushingLength <= 0) return true
+    const info = await this.core.insertBatch(batch, this._appends, { keyPair, signature })
+    if (info === null) return false
 
-      const blocks = flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends
+    const delta = info.byteLength - this._sessionByteLength
 
-      const info = await this.session.append(blocks, { keyPair, signature })
-      const delta = info.byteLength - this._sessionByteLength
-
-      this._sessionLength = info.length
-      this._sessionByteLength = info.byteLength
-      this._sessionBatch = this.session.createTreeBatch()
+    this._sessionLength = info.length
+    this._sessionByteLength = info.byteLength
+    this._sessionBatch = this.session.createTreeBatch()
 
-      this._appends = this._appends.slice(flushingLength)
-      this._byteLength -= delta
+    this._appends = this._appends.slice(flushingLength)
+    this._byteLength -= delta
 
-      this.emit('flush')
-    } finally {
-      this.core._mutex.unlock()
-    }
+    this.emit('flush')
 
     return true
   }

package/lib/core.js

@@ -432,40 +432,71 @@ module.exports = class Core {
     })
   }
 
-  // assumes mutex is already acquried
-  async insertValuesUnsafe (values, offset, byteOffset, treeNodes) {
-    if (!values.length) return
+  async insertBatch (batch, values, { signature, keyPair = this.header.keyPair } = {}) {
+    await this._mutex.lock()
 
-    const bitfield = {
-      drop: false,
-      start: offset,
-      end: offset + values.length
-    }
+    try {
+      // upsert compat manifest
+      if (this.verifier === null && keyPair) this.setManifest(null, keyPair)
 
-    await this.blocks.putBatch(offset, values, byteOffset)
-    await this.oplog.append({
-      userData: null,
-      treeNodes,
-      treeUpgrade: null,
-      bitfield
-    }, false)
+      if (this.tree.fork !== batch.fork) return null
 
-    this.bitfield.setRange(bitfield.start, values.length, true)
-    for (const node of treeNodes) this.tree.unflushed.set(node.index, node)
+      if (this.tree.length > batch.treeLength) {
+        if (this.tree.length > batch.length) return null // TODO: partial commit in the future if possible
 
-    const status = updateContig(this.header, bitfield, this.bitfield)
-    this.onupdate(status, bitfield, null, null)
+        for (const root of this.tree.roots) {
+          const batchRoot = await batch.get(root.index)
+          if (batchRoot.size !== root.size || !b4a.equals(batchRoot.hash, root.hash)) {
+            return null
+          }
+        }
+      }
 
-    if (this._shouldFlush()) await this._flushOplog()
-  }
+      const offset = batch.treeLength
+      const adding = batch.length - batch.treeLength
 
-  async append (values, { lock, signature, keyPair = this.header.keyPair, preappend } = {}) {
-    if (lock) await this._mutex.lock()
+      batch.upgraded = batch.length > this.tree.length
+      batch.treeLength = this.tree.length
+      if (batch.upgraded) batch.signature = signature || this.verifier.sign(batch, keyPair)
 
-    // upsert compat manifest
-    if (this.verifier === null && keyPair) this.setManifest(null, keyPair)
+      let byteOffset = batch.byteLength
+      for (let i = 0; i < adding; i++) byteOffset -= values[i].byteLength
+
+      const entry = {
+        userData: null,
+        treeNodes: batch.nodes,
+        treeUpgrade: batch.upgraded ? batch : null,
+        bitfield: {
+          drop: false,
+          start: offset,
+          length: adding
+        }
+      }
+
+      await this.blocks.putBatch(offset, adding > values.length ? values.slice(0, adding) : values, byteOffset)
+      await this.oplog.append([entry], false)
+
+      this.bitfield.setRange(entry.bitfield.start, entry.bitfield.length, true)
+      batch.commit()
+
+      const status = (batch.upgraded ? 0b0001 : 0) | updateContig(this.header, entry.bitfield, this.bitfield)
+      this.onupdate(status, entry.bitfield, null, null)
+
+      if (this._shouldFlush()) await this._flushOplog()
+    } finally {
+      this._mutex.unlock()
+    }
+
+    return { length: batch.length, byteLength: batch.byteLength }
+  }
+
+  async append (values, { signature, keyPair = this.header.keyPair, preappend } = {}) {
+    await this._mutex.lock()
 
     try {
+      // upsert compat manifest
+      if (this.verifier === null && keyPair) this.setManifest(null, keyPair)
+
       if (preappend) await preappend(values)
 
       if (!values.length) {
@@ -506,7 +537,7 @@ module.exports = class Core {
 
       return { length: batch.length, byteLength }
     } finally {
-      if (lock) this._mutex.unlock()
+      this._mutex.unlock()
     }
   }
 

package/lib/replicator.js

@@ -1046,7 +1046,7 @@ module.exports = class Replicator {
     this.findingPeers = 0 // updateable from the outside
     this.destroyed = false
     this.downloading = true
-    this.sessions = 0
+    this.activeSessions = 0
 
     this._attached = new Set()
     this._inflight = new InflightTracker()
@@ -1074,15 +1074,29 @@ module.exports = class Replicator {
     }
   }
 
+  updateActivity (inc, session) {
+    this.activeSessions += inc
+    this.setDownloading(this.activeSessions !== 0, session)
+  }
+
   isDownloading () {
     return this.downloading || !this._inflight.idle
   }
 
-  setDownloading (downloading) {
+  setDownloading (downloading, session) {
     if (this.downloading === downloading) return
     this.downloading = downloading
     if (!downloading && this.isDownloading()) return
+
     for (const peer of this.peers) peer.signalUpgrade()
+
+    if (downloading) { // restart channel if needed...
+      for (const protomux of this._attached) {
+        if (!protomux.stream.handshakeHash) continue
+        if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) continue
+        this._makePeer(protomux, session && session.session())
+      }
+    }
   }
 
   cork () {
@@ -1779,12 +1793,10 @@ module.exports = class Replicator {
   }
 
   _closeSession (session) {
-    if (!session.closing) this.sessions--
     session.close().catch(noop)
   }
 
   attachTo (protomux, session) {
-    if (session) this.sessions++
     const makePeer = this._makePeer.bind(this, protomux, session)
 
     this._attached.add(protomux)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.24.5",
+  "version": "10.24.7",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {