hypercore 10.19.1 → 10.20.1

package/index.js

@@ -82,6 +82,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.onwait = opts.onwait || null
     this.wait = opts.wait !== false
     this.timeout = opts.timeout || 0
+    this._clone = opts.clone || null
     this._readonly = opts.writable === false
 
     this.closing = null
@@ -306,6 +307,14 @@ module.exports = class Hypercore extends EventEmitter {
         const s = this.sessions[i]
         if (s !== this) s._passCapabilities(this)
       }
+
+      // copy state over
+      if (this._clone) {
+        const { from, upgrade } = this._clone
+        await from.opening
+        await this.core.copyFrom(from.core, upgrade)
+        this._clone = null
+      }
     }
 
     if (!this.auth) this.auth = this.core.defaultAuth
@@ -458,6 +467,50 @@ module.exports = class Hypercore extends EventEmitter {
     this.emit('close', true)
   }
 
+  clone (storage, opts = {}) {
+    // TODO: current limitation is no forking
+    if ((opts.fork && opts.fork !== 0) || this.fork !== 0) {
+      throw BAD_ARGUMENT('Cannot clone a fork')
+    }
+
+    const key = opts.key === undefined ? opts.keyPair ? null : this.key : opts.key
+    const keyPair = (opts.auth || opts.keyPair === undefined) ? null : opts.keyPair
+
+    let auth = this.core.defaultAuth
+    if (opts.auth) {
+      auth = opts.auth
+    } else if (opts.sign && keyPair) {
+      auth = Core.createAuth(this.crypto, keyPair, opts)
+    } else if (opts.sign) {
+      // TODO: dangerous to just update sign?
+      auth.sign = opts.sign
+    } else if (keyPair && keyPair.secretKey) {
+      auth = Core.createAuth(this.crypto, keyPair)
+    }
+
+    const upgrade = opts.upgrade === undefined ? null : opts.upgrade
+
+    const sparse = opts.sparse === false ? false : this.sparse
+    const wait = opts.wait === false ? false : this.wait
+    const onwait = opts.onwait === undefined ? this.onwait : opts.onwait
+    const timeout = opts.timeout === undefined ? this.timeout : opts.timeout
+
+    const Clz = this.constructor
+    return new Clz(storage, key, {
+      ...opts,
+      sparse,
+      wait,
+      onwait,
+      timeout,
+      auth,
+      overwrite: true,
+      clone: {
+        from: this,
+        upgrade
+      }
+    })
+  }
+
   replicate (isInitiator, opts = {}) {
     // Only limitation here is that ondiscoverykey doesn't work atm when passing a muxer directly,
     // because it doesn't really make a lot of sense.
@@ -787,6 +840,7 @@ module.exports = class Hypercore extends EventEmitter {
     const cleared = (opts && opts.diff) ? { blocks: 0 } : null
 
     if (start >= end) return cleared
+    if (start >= this.length) return cleared
 
     await this.core.clear(start, end, cleared)
 

package/lib/core.js

@@ -22,6 +22,7 @@ module.exports = class Core {
     this.bitfield = bitfield
     this.defaultAuth = auth
     this.truncating = 0
+    this.closed = false
 
     this._maxOplogSize = 65536
     this._autoFlush = 1
@@ -175,6 +176,70 @@ module.exports = class Core {
     return false
   }
 
+  async copyFrom (src, signature, auth = this.defaultAuth) {
+    this._mutex.lock()
+
+    try {
+      let pos = src.bitfield.firstSet(0)
+
+      while (pos >= 0) {
+        const segmentStart = pos
+        const segmentEnd = src.bitfield.firstUnset(pos)
+
+        if (segmentStart < 0) break
+
+        const segment = []
+
+        while (pos < segmentEnd) {
+          const val = await src.blocks.get(pos++)
+          segment.push(val)
+        }
+
+        const [offset] = await src.tree.byteRange(2 * segmentStart)
+        await this.blocks.putBatch(segmentStart, segment, offset)
+
+        this.bitfield.setRange(segmentStart, segmentEnd, true)
+
+        pos = src.bitfield.firstSet(segmentEnd + 1)
+      }
+
+      // TODO: is it an issue to move the nodes directly?
+      // TODO: make flat iterator that computes needed nodes
+
+      for (let i = 0; i <= src.tree.length * 2; i++) {
+        const node = await src.tree.get(i, false)
+        if (node === null) continue
+
+        await this.tree.addNode(node)
+      }
+
+      await this.tree.flush()
+
+      this.tree.fork = src.tree.fork
+      this.tree.roots = [...src.tree.roots]
+      this.tree.length = src.tree.length
+      this.tree.byteLength = src.tree.byteLength
+      this.tree.signature = null // must provide signature
+
+      this.tree.signature = signature || auth.sign(this.tree.signable())
+
+      if (signature && !this._signed(this.tree)) {
+        // TODO: how to handle signature failure?
+        this.tree.signature = null
+        throw INVALID_SIGNATURE('Clone was provided with an invalid signature')
+      }
+
+      this.header.tree.length = this.tree.length
+      this.header.tree.rootHash = this.tree.hash()
+      this.header.tree.signature = this.tree.signature
+      this.header.userData = src.header.userData // should copy?
+
+      await this._flushOplog()
+    } finally {
+      this._mutex.unlock()
+    }
+  }
+
   async _flushOplog () {
     // TODO: the apis using this, actually do not need to wait for the bitfields, tree etc to flush
     // as their mutations are already stored in the oplog. We could potentially just run this in the
@@ -271,8 +336,8 @@ module.exports = class Core {
       if (start >= end || start >= this.tree.length) return
 
       const offset = await this.tree.byteOffset(start * 2)
-      const [byteEnd, byteEndLength] = await this.tree.byteRange((end - 1) * 2)
-      const length = (byteEnd + byteEndLength) - offset
+      const endOffset = await this.tree.byteOffset(end * 2)
+      const length = endOffset - offset
 
       const before = cleared ? await Info.bytesUsed(this.blocks.storage) : null
 
@@ -575,6 +640,7 @@ module.exports = class Core {
   }
 
   async close () {
+    this.closed = true
     await this._mutex.destroy()
     await Promise.allSettled([
       this.oplog.close(),

package/lib/merkle-tree.js

@@ -714,6 +714,7 @@ module.exports = class MerkleTree {
   }
 
   async byteOffset (index) {
+    if (index === 2 * this.length) return this.byteLength
     if ((index & 1) === 1) index = flat.leftSpan(index)
 
     let head = 0

package/lib/replicator.js

@@ -560,6 +560,7 @@ class Peer {
         proof = await this._getProof(msg)
       } catch (err) { // TODO: better error handling here, ie custom errors
         safetyCatch(err)
+        if (this.replicator.core.closed) throw err // just an extra safety check...
       }
     }
 
@@ -1004,6 +1005,7 @@ module.exports = class Replicator {
     this.onupload = onupload
     this.peers = []
     this.findingPeers = 0 // updateable from the outside
+    this.destroyed = false
 
     this._attached = new Set()
     this._inflight = new InflightTracker()
@@ -1711,7 +1713,7 @@ module.exports = class Replicator {
     protomux.stream.opened.then((opened) => {
       this._ifAvailable--
 
-      if (opened) makePeer()
+      if (opened && !this.destroyed) makePeer()
       else if (session) session.close().catch(noop)
       this._checkUpgradeIfAvailable()
     })
@@ -1725,6 +1727,7 @@ module.exports = class Replicator {
   }
 
   destroy () {
+    this.destroyed = true
     for (const peer of this.peers) {
       this.detachFrom(peer.protomux)
       peer.channel.close()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.19.1",
+  "version": "10.20.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {
@@ -64,8 +64,8 @@
     "range-parser": "^1.2.1",
     "speedometer": "^1.1.0",
     "standard": "^17.0.0",
+    "test-tmp": "^1.0.2",
     "tiny-byte-size": "^1.1.0",
-    "tmp-promise": "^3.0.2",
     "udx-native": "^1.6.1"
   }
 }