hypercore 10.24.10 → 10.25.0

package/index.js

@@ -511,6 +511,9 @@ module.exports = class Hypercore extends EventEmitter {
     // because it doesn't really make a lot of sense.
     if (Protomux.isProtomux(isInitiator)) return this._attachToMuxer(isInitiator, opts)
 
+    // if same stream is passed twice, ignore the 2nd one before we make sessions etc
+    if (isStream(isInitiator) && this._isAttached(isInitiator)) return isInitiator
+
     const protocolStream = Hypercore.createProtocolStream(isInitiator, opts)
     const noiseStream = protocolStream.noiseStream
     const protocol = noiseStream.userData
@@ -521,6 +524,10 @@ module.exports = class Hypercore extends EventEmitter {
     return protocolStream
   }
 
+  _isAttached (stream) {
+    return stream.userData && this.replicator && this.replicator.attached(stream.userData)
+  }
+
   _attachToMuxer (mux, useSession) {
     if (this.opened) {
       this._attachToMuxerOpened(mux, useSession)
@@ -771,9 +778,9 @@ module.exports = class Hypercore extends EventEmitter {
     return true
   }
 
-  batch ({ autoClose = true, session = true } = {}) {
+  batch ({ checkout = -1, autoClose = true, session = true } = {}) {
     if (this._batch !== null) throw BATCH_ALREADY_EXISTS()
-    const batch = new Batch(session ? this.session() : this, autoClose)
+    const batch = new Batch(session ? this.session() : this, checkout, autoClose)
     for (const session of this.sessions) session._batch = batch
     return batch
   }

package/lib/batch.js

@@ -3,7 +3,7 @@ const EventEmitter = require('events')
 const c = require('compact-encoding')
 
 module.exports = class HypercoreBatch extends EventEmitter {
-  constructor (session, autoClose) {
+  constructor (session, checkoutLength, autoClose) {
     super()
 
     this.session = session
@@ -16,6 +16,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
     this.fork = 0
 
     this._appends = []
+    this._checkoutLength = checkoutLength
     this._byteLength = 0
     this._sessionLength = 0
     this._sessionByteLength = 0
@@ -64,9 +65,21 @@ module.exports = class HypercoreBatch extends EventEmitter {
   async ready () {
     await this.session.ready()
     if (this.opened) return
-    this._sessionLength = this.session.length
-    this._sessionByteLength = this.session.byteLength
-    this._sessionBatch = this.session.createTreeBatch()
+
+    if (this._checkoutLength !== -1) {
+      const batch = await this.session.core.tree.truncate(this._checkoutLength, this.session.fork)
+      batch.upgraded = false
+      batch.treeLength = batch.ancestors = this._checkoutLength
+      if (this.opened) return
+      this._sessionLength = batch.length
+      this._sessionByteLength = batch.byteLength
+      this._sessionBatch = batch
+    } else {
+      this._sessionLength = this.session.length
+      this._sessionByteLength = this.session.byteLength
+      this._sessionBatch = this.session.createTreeBatch()
+    }
+
     this.fork = this.session.fork
     this.opened = true
     this.emit('ready')
@@ -77,6 +90,10 @@ module.exports = class HypercoreBatch extends EventEmitter {
     await this.session.update(opts)
   }
 
+  treeHash () {
+    return this._sessionBatch.hash()
+  }
+
   setUserData (key, value, opts) {
     return this.session.setUserData(key, value, opts)
   }

package/lib/core.js

@@ -217,20 +217,21 @@ module.exports = class Core {
     return false
   }
 
-  async copyFrom (src, signature) {
+  async copyFrom (src, signature, { length = src.tree.length } = {}) {
     await this._mutex.lock()
 
     try {
-      let pos = src.bitfield.firstSet(0)
+      let pos = 0
 
-      while (pos >= 0) {
-        const segmentStart = pos
-        const segmentEnd = src.bitfield.firstUnset(pos)
+      while (pos < length) {
+        const segmentStart = maximumSegmentStart(pos, src.bitfield, this.bitfield)
+        const segmentEnd = minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield)
 
-        if (segmentStart < 0) break
+        if (segmentStart >= length || segmentStart < 0) break
 
         const segment = []
 
+        pos = segmentStart
         while (pos < segmentEnd) {
           const val = await src.blocks.get(pos++)
           segment.push(val)
@@ -241,13 +242,10 @@ module.exports = class Core {
 
         this.bitfield.setRange(segmentStart, segmentEnd, true)
 
-        pos = src.bitfield.firstSet(segmentEnd + 1)
+        pos = segmentEnd + 1
       }
 
-      // TODO: is it an issue to move the nodes directly?
-      // TODO: make flat iterator that computes needed nodes
-
-      for (let i = 0; i <= src.tree.length * 2; i++) {
+      for (let i = 0; i < length * 2; i++) {
         const node = await src.tree.get(i, false)
         if (node === null) continue
 
@@ -256,25 +254,35 @@ module.exports = class Core {
 
       await this.tree.flush()
 
-      this.tree.fork = src.tree.fork
-      this.tree.roots = [...src.tree.roots]
-      this.tree.length = src.tree.length
-      this.tree.byteLength = src.tree.byteLength
-
-      try {
-        const batch = this.tree.batch()
-        batch.signature = signature
-        this._verifyBatchUpgrade(batch, this.header.manifest)
-        this.tree.signature = signature
-      } catch (err) {
-        this.tree.signature = null
-        // TODO: how to handle signature failure?
-        throw err
+      if (length > this.tree.length) {
+        this.tree.fork = src.tree.fork
+        this.tree.roots = [...src.tree.roots]
+        this.tree.length = src.tree.length
+        this.tree.byteLength = src.tree.byteLength
+
+        if (length < this.tree.length) {
+          const batch = await src.tree.truncate(length)
+          this.tree.roots = [...batch.roots]
+          this.tree.length = batch.length
+          this.tree.byteLength = batch.byteLength
+        }
+
+        try {
+          const batch = this.tree.batch()
+          batch.signature = signature
+          this._verifyBatchUpgrade(batch, this.header.manifest)
+          this.tree.signature = signature
+        } catch (err) {
+          this.tree.signature = null
+          // TODO: how to handle signature failure?
+          throw err
+        }
+
+        this.header.tree.length = this.tree.length
+        this.header.tree.rootHash = this.tree.hash()
+        this.header.tree.signature = this.tree.signature
       }
 
-      this.header.tree.length = this.tree.length
-      this.header.tree.rootHash = this.tree.hash()
-      this.header.tree.signature = this.tree.signature
       this.header.userData = src.header.userData // should copy?
 
       await this._flushOplog()
@@ -457,6 +465,7 @@ module.exports = class Core {
 
       batch.upgraded = batch.length > this.tree.length
       batch.treeLength = this.tree.length
+      batch.ancestors = this.tree.length
       if (batch.upgraded) batch.signature = signature || this.verifier.sign(batch, keyPair)
 
       let byteOffset = batch.byteLength
@@ -897,3 +906,21 @@ async function flushHeader (oplog, bigHeader, header) {
 }
 
 function noop () {}
+
+function maximumSegmentStart (start, src, dst) {
+  const a = src.firstSet(start)
+  const b = dst.firstUnset(start)
+
+  if (a === -1) return -1
+  if (b === -1) return a
+  return a < b ? b : a
+}
+
+function minimumSegmentEnd (start, src, dst) {
+  const a = src.firstUnset(start)
+  const b = dst.firstSet(start)
+
+  if (a === -1) return -1
+  if (b === -1) return a
+  return a < b ? a : b
+}

package/lib/replicator.js

@@ -1,6 +1,7 @@
 const b4a = require('b4a')
 const safetyCatch = require('safety-catch')
 const RandomIterator = require('random-array-iterator')
+const flatTree = require('flat-tree')
 const ReceiverQueue = require('./receiver-queue')
 const RemoteBitfield = require('./remote-bitfield')
 const { REQUEST_CANCELLED, REQUEST_TIMEOUT, INVALID_CAPABILITY, SNAPSHOT_NOT_AVAILABLE } = require('hypercore-errors')
@@ -439,6 +440,16 @@ class Peer {
     }
   }
 
+  closeIfIdle () {
+    if (this.remoteDownloading === false && this.replicator.isDownloading() === false) {
+      // idling, shut it down...
+      this.channel.close()
+      return true
+    }
+
+    return false
+  }
+
   async onsync ({ fork, length, remoteLength, canUpgrade, uploading, downloading, hasManifest }) {
     const lengthChanged = length !== this.remoteLength
     const sameFork = fork === this.core.tree.fork
@@ -451,11 +462,7 @@ class Peer {
     this.remoteDownloading = downloading
     this.remoteHasManifest = hasManifest
 
-    if (this.remoteDownloading === false && this.replicator.isDownloading() === false) {
-      // idling, shut it down...
-      this.channel.close()
-      return
-    }
+    if (this.closeIfIdle()) return
 
     this.lengthAcked = sameFork ? remoteLength : 0
     this.syncsProcessing++
@@ -583,7 +590,7 @@ class Peer {
         proof = await this._getProof(msg)
       } catch (err) {
         safetyCatch(err)
-        if (isCriticalError(err)) throw err
+        if (msg.fork === this.core.tree.fork && isCriticalError(err)) throw err
       }
     }
 
@@ -830,6 +837,7 @@ class Peer {
 
       if (this.remoteBitfield.get(index) === false) continue
       if (this.core.bitfield.get(index) === true) continue
+      if (!this._hasTreeParent(index)) continue
 
       // Check if this block is currently inflight - if so pick another
       const b = this.replicator._blocks.get(index)
@@ -860,6 +868,38 @@ class Peer {
     return this.remoteBitfield.get(b.index)
   }
 
+  _hasTreeParent (index) {
+    if (this.remoteLength >= this.core.tree.length) return true
+
+    const ite = flatTree.iterator(index * 2)
+
+    let span = 2
+    let length = 0
+
+    while (true) {
+      ite.parent()
+
+      const left = (ite.index - ite.factor / 2 + 1) / 2
+      length = left + span
+
+      // if larger than local AND larger than remote - they share the root so its ok
+      if (length > this.core.tree.length) {
+        if (length > this.remoteLength) return true
+        break
+      }
+
+      // its less than local but larger than remote so skip it
+      if (length > this.remoteLength) break
+
+      span *= 2
+      const first = this.core.bitfield.findFirst(true, left)
+      if (first > -1 && first < length) return true
+    }
+
+    // TODO: push to async queue and check against our local merkle tree if we actually can request this block
+    return false
+  }
+
   _requestBlock (b) {
     const { length, fork } = this.core.tree
 
@@ -867,6 +907,9 @@ class Peer {
       this._maybeWant(b.index)
       return false
     }
+    if (!this._hasTreeParent(b.index)) {
+      return false
+    }
 
     const req = this._makeRequest(b.index >= length, b.priority)
     if (req === null) return false
@@ -916,6 +959,10 @@ class Peer {
         continue
       }
 
+      if (!this._hasTreeParent(index)) {
+        continue
+      }
+
       const b = this.replicator._blocks.add(index, PRIORITY.NORMAL)
 
       if (b.inflight.length > 0) {
@@ -1041,6 +1088,7 @@ module.exports = class Replicator {
     this.allowFork = allowFork
     this.onpeerupdate = onpeerupdate
     this.onupload = onupload
+    this.ondownloading = null // optional external hook for monitoring downloading status
     this.peers = []
     this.findingPeers = 0 // updateable from the outside
     this.destroyed = false
@@ -1095,7 +1143,11 @@ module.exports = class Replicator {
         if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) continue
         this._makePeer(protomux, session && session.session({ active: false }))
       }
+    } else {
+      for (const peer of this.peers) peer.closeIfIdle()
     }
+
+    if (this.ondownloading !== null && downloading) this.ondownloading()
   }
 
   cork () {
@@ -1795,6 +1847,10 @@ module.exports = class Replicator {
     session.close().catch(noop)
   }
 
+  attached (protomux) {
+    return this._attached.has(protomux)
+  }
+
   attachTo (protomux, session) {
     const makePeer = this._makePeer.bind(this, protomux, session)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.24.10",
+  "version": "10.25.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {