hypercore 10.24.11 → 10.25.1

package/README.md

@@ -283,15 +283,15 @@ A range can have the following properties:
 }
 ```
 
-To download the full core continously (often referred to as non sparse mode) do
+To download the full core continuously (often referred to as non sparse mode) do
 
 ``` js
-// Note that this will never be consider downloaded as the range
+// Note that this will never be considered downloaded as the range
 // will keep waiting for new blocks to be appended.
 core.download({ start: 0, end: -1 })
 ```
 
-To downloaded a discrete range of blocks pass a list of indices.
+To download a discrete range of blocks pass a list of indices.
 
 ```js
 core.download({ blocks: [4, 9, 7] })
@@ -434,7 +434,7 @@ How much padding is applied to each block of this core? Will be `0` unless block
 
 Create a replication stream. You should pipe this to another Hypercore instance.
 
-The `isInitiator` argument is a boolean indicating whether you are the iniatior of the connection (ie the client)
+The `isInitiator` argument is a boolean indicating whether you are the initiator of the connection (ie the client)
 or if you are the passive part (ie the server).
 
 If you are using a P2P swarm like [Hyperswarm](https://github.com/hyperswarm/hyperswarm) you can know this by checking if the swarm connection is a client socket or server socket. In Hyperswarm you can check that using the [client property on the peer details object](https://github.com/hyperswarm/hyperswarm#swarmonconnection-socket-details--)

package/index.js

@@ -211,8 +211,8 @@ module.exports = class Hypercore extends EventEmitter {
 
   session (opts = {}) {
     if (this.closing) {
-      // This makes the closing logic alot easier. If this turns out to be a problem
-      // in practive, open an issue and we'll try to make a solution for it.
+      // This makes the closing logic a lot easier. If this turns out to be a problem
+      // in practice, open an issue and we'll try to make a solution for it.
       throw SESSION_CLOSED('Cannot make sessions on a closing core')
     }
 
@@ -511,6 +511,9 @@ module.exports = class Hypercore extends EventEmitter {
     // because it doesn't really make a lot of sense.
     if (Protomux.isProtomux(isInitiator)) return this._attachToMuxer(isInitiator, opts)
 
+    // if same stream is passed twice, ignore the 2nd one before we make sessions etc
+    if (isStream(isInitiator) && this._isAttached(isInitiator)) return isInitiator
+
     const protocolStream = Hypercore.createProtocolStream(isInitiator, opts)
     const noiseStream = protocolStream.noiseStream
     const protocol = noiseStream.userData
@@ -521,6 +524,10 @@ module.exports = class Hypercore extends EventEmitter {
     return protocolStream
   }
 
+  _isAttached (stream) {
+    return stream.userData && this.replicator && this.replicator.attached(stream.userData)
+  }
+
   _attachToMuxer (mux, useSession) {
     if (this.opened) {
       this._attachToMuxerOpened(mux, useSession)
@@ -771,9 +778,9 @@ module.exports = class Hypercore extends EventEmitter {
     return true
   }
 
-  batch ({ autoClose = true, session = true } = {}) {
+  batch ({ checkout = -1, autoClose = true, session = true } = {}) {
     if (this._batch !== null) throw BATCH_ALREADY_EXISTS()
-    const batch = new Batch(session ? this.session() : this, autoClose)
+    const batch = new Batch(session ? this.session() : this, checkout, autoClose)
     for (const session of this.sessions) session._batch = batch
     return batch
   }
@@ -1093,7 +1100,7 @@ function preappend (blocks) {
 
 function ensureEncryption (core, opts) {
   if (!opts.encryptionKey) return
-  // Only override the block encryption if its either not already set or if
+  // Only override the block encryption if it's either not already set or if
   // the caller provided a different key.
   if (core.encryption && b4a.equals(core.encryption.key, opts.encryptionKey)) return
   core.encryption = new BlockEncryption(opts.encryptionKey, core.key)

package/lib/batch.js

@@ -1,9 +1,10 @@
 const { BLOCK_NOT_AVAILABLE, SESSION_CLOSED } = require('hypercore-errors')
 const EventEmitter = require('events')
 const c = require('compact-encoding')
+const b4a = require('b4a')
 
 module.exports = class HypercoreBatch extends EventEmitter {
-  constructor (session, autoClose) {
+  constructor (session, checkoutLength, autoClose) {
     super()
 
     this.session = session
@@ -16,6 +17,8 @@ module.exports = class HypercoreBatch extends EventEmitter {
     this.fork = 0
 
     this._appends = []
+    this._appendsActual = null
+    this._checkoutLength = checkoutLength
     this._byteLength = 0
     this._sessionLength = 0
     this._sessionByteLength = 0
@@ -64,9 +67,22 @@ module.exports = class HypercoreBatch extends EventEmitter {
   async ready () {
     await this.session.ready()
     if (this.opened) return
-    this._sessionLength = this.session.length
-    this._sessionByteLength = this.session.byteLength
-    this._sessionBatch = this.session.createTreeBatch()
+
+    if (this._checkoutLength !== -1) {
+      const batch = await this.session.core.tree.truncate(this._checkoutLength, this.session.fork)
+      batch.upgraded = false
+      batch.treeLength = batch.ancestors = this._checkoutLength
+      if (this.opened) return
+      this._sessionLength = batch.length
+      this._sessionByteLength = batch.byteLength
+      this._sessionBatch = batch
+    } else {
+      this._sessionLength = this.session.length
+      this._sessionByteLength = this.session.byteLength
+      this._sessionBatch = this.session.createTreeBatch()
+    }
+
+    this._appendsActual = this.session.encryption ? [] : this._appends
     this.fork = this.session.fork
     this.opened = true
     this.emit('ready')
@@ -77,6 +93,10 @@ module.exports = class HypercoreBatch extends EventEmitter {
     await this.session.update(opts)
   }
 
+  treeHash () {
+    return this._sessionBatch.hash()
+  }
+
   setUserData (key, value, opts) {
     return this.session.setUserData(key, value, opts)
   }
@@ -157,13 +177,13 @@ module.exports = class HypercoreBatch extends EventEmitter {
     if (len < this._sessionLength || length > maxLength) return null
 
     for (let i = 0; i < len - this._sessionLength; i++) {
-      b.append(this._appends[i])
+      b.append(this._appendsActual[i])
     }
 
     if (len < this.length) return b
 
     for (let i = 0; i < length - len; i++) {
-      b.append(blocks[i])
+      b.append(this._appendsActual === this._appends ? blocks[i] : this._encrypt(b.length + i, blocks[i]))
     }
 
     return b
@@ -215,11 +235,16 @@ module.exports = class HypercoreBatch extends EventEmitter {
 
     if (session.encodeBatch === null) {
       for (let i = 0; i < blocks.length; i++) {
-        const buffer = session._encode(session.valueEncoding, blocks[i])
+        const buffer = this._encode(session.valueEncoding, blocks[i])
         buffers[i] = buffer
         this._byteLength += buffer.byteLength
       }
     }
+    if (this._appends !== this._appendsActual) {
+      for (let i = 0; i < buffers.length; i++) {
+        this._appendsActual.push(this._encrypt(this._sessionLength + this._appendsActual.length, buffers[i]))
+      }
+    }
 
     this._appends.push(...buffers)
 
@@ -229,6 +254,35 @@ module.exports = class HypercoreBatch extends EventEmitter {
     return info
   }
 
+  _encode (enc, val) {
+    const state = { start: 0, end: 0, buffer: null }
+
+    if (b4a.isBuffer(val)) {
+      if (state.start === 0) return val
+      state.end += val.byteLength
+    } else if (enc) {
+      enc.preencode(state, val)
+    } else {
+      val = b4a.from(val)
+      if (state.start === 0) return val
+      state.end += val.byteLength
+    }
+
+    state.buffer = b4a.allocUnsafe(state.end)
+
+    if (enc) enc.encode(state, val)
+    else state.buffer.set(val, state.start)
+
+    return state.buffer
+  }
+
+  _encrypt (index, buffer) {
+    const block = b4a.allocUnsafe(buffer.byteLength + 8)
+    block.set(buffer, 8)
+    this.session.encryption.encrypt(index, block, this.fork)
+    return block
+  }
+
   async flush (opts = {}) {
     if (this.opened === false) await this.opening
     if (this.closing) throw SESSION_CLOSED()
@@ -258,7 +312,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
     const batch = this.createTreeBatch(this._sessionLength + flushingLength)
     if (batch === null) return false
 
-    const info = await this.core.insertBatch(batch, this._appends, { keyPair, signature })
+    const info = await this.core.insertBatch(batch, this._appendsActual, { keyPair, signature })
     if (info === null) return false
 
     const delta = info.byteLength - this._sessionByteLength

package/lib/core.js

@@ -217,20 +217,21 @@ module.exports = class Core {
     return false
   }
 
-  async copyFrom (src, signature) {
+  async copyFrom (src, signature, { length = src.tree.length } = {}) {
     await this._mutex.lock()
 
     try {
-      let pos = src.bitfield.firstSet(0)
+      let pos = 0
 
-      while (pos >= 0) {
-        const segmentStart = pos
-        const segmentEnd = src.bitfield.firstUnset(pos)
+      while (pos < length) {
+        const segmentStart = maximumSegmentStart(pos, src.bitfield, this.bitfield)
+        const segmentEnd = minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield)
 
-        if (segmentStart < 0) break
+        if (segmentStart >= length || segmentStart < 0) break
 
         const segment = []
 
+        pos = segmentStart
         while (pos < segmentEnd) {
           const val = await src.blocks.get(pos++)
           segment.push(val)
@@ -241,13 +242,10 @@ module.exports = class Core {
 
         this.bitfield.setRange(segmentStart, segmentEnd, true)
 
-        pos = src.bitfield.firstSet(segmentEnd + 1)
+        pos = segmentEnd + 1
       }
 
-      // TODO: is it an issue to move the nodes directly?
-      // TODO: make flat iterator that computes needed nodes
-
-      for (let i = 0; i <= src.tree.length * 2; i++) {
+      for (let i = 0; i < length * 2; i++) {
         const node = await src.tree.get(i, false)
         if (node === null) continue
 
@@ -256,25 +254,35 @@ module.exports = class Core {
 
       await this.tree.flush()
 
-      this.tree.fork = src.tree.fork
-      this.tree.roots = [...src.tree.roots]
-      this.tree.length = src.tree.length
-      this.tree.byteLength = src.tree.byteLength
-
-      try {
-        const batch = this.tree.batch()
-        batch.signature = signature
-        this._verifyBatchUpgrade(batch, this.header.manifest)
-        this.tree.signature = signature
-      } catch (err) {
-        this.tree.signature = null
-        // TODO: how to handle signature failure?
-        throw err
+      if (length > this.tree.length) {
+        this.tree.fork = src.tree.fork
+        this.tree.roots = [...src.tree.roots]
+        this.tree.length = src.tree.length
+        this.tree.byteLength = src.tree.byteLength
+
+        if (length < this.tree.length) {
+          const batch = await src.tree.truncate(length)
+          this.tree.roots = [...batch.roots]
+          this.tree.length = batch.length
+          this.tree.byteLength = batch.byteLength
+        }
+
+        try {
+          const batch = this.tree.batch()
+          batch.signature = signature
+          this._verifyBatchUpgrade(batch, this.header.manifest)
+          this.tree.signature = signature
+        } catch (err) {
+          this.tree.signature = null
+          // TODO: how to handle signature failure?
+          throw err
+        }
+
+        this.header.tree.length = this.tree.length
+        this.header.tree.rootHash = this.tree.hash()
+        this.header.tree.signature = this.tree.signature
       }
 
-      this.header.tree.length = this.tree.length
-      this.header.tree.rootHash = this.tree.hash()
-      this.header.tree.signature = this.tree.signature
       this.header.userData = src.header.userData // should copy?
 
       await this._flushOplog()
@@ -457,6 +465,7 @@ module.exports = class Core {
 
       batch.upgraded = batch.length > this.tree.length
       batch.treeLength = this.tree.length
+      batch.ancestors = this.tree.length
       if (batch.upgraded) batch.signature = signature || this.verifier.sign(batch, keyPair)
 
       let byteOffset = batch.byteLength
@@ -897,3 +906,21 @@ async function flushHeader (oplog, bigHeader, header) {
 }
 
 function noop () {}
+
+function maximumSegmentStart (start, src, dst) {
+  const a = src.firstSet(start)
+  const b = dst.firstUnset(start)
+
+  if (a === -1) return -1
+  if (b === -1) return a
+  return a < b ? b : a
+}
+
+function minimumSegmentEnd (start, src, dst) {
+  const a = src.firstUnset(start)
+  const b = dst.firstSet(start)
+
+  if (a === -1) return -1
+  if (b === -1) return a
+  return a < b ? a : b
+}

package/lib/replicator.js

@@ -346,6 +346,7 @@ class Peer {
   }
 
   broadcastRange (start, length, drop) {
+    if (drop) this.skipList.setRange(start, length, false)
     this.wireRange.send({
       drop,
       start,
@@ -440,6 +441,16 @@ class Peer {
     }
   }
 
+  closeIfIdle () {
+    if (this.remoteDownloading === false && this.replicator.isDownloading() === false) {
+      // idling, shut it down...
+      this.channel.close()
+      return true
+    }
+
+    return false
+  }
+
   async onsync ({ fork, length, remoteLength, canUpgrade, uploading, downloading, hasManifest }) {
     const lengthChanged = length !== this.remoteLength
     const sameFork = fork === this.core.tree.fork
@@ -452,11 +463,7 @@ class Peer {
     this.remoteDownloading = downloading
     this.remoteHasManifest = hasManifest
 
-    if (this.remoteDownloading === false && this.replicator.isDownloading() === false) {
-      // idling, shut it down...
-      this.channel.close()
-      return
-    }
+    if (this.closeIfIdle()) return
 
     this.lengthAcked = sameFork ? remoteLength : 0
     this.syncsProcessing++
@@ -1082,6 +1089,7 @@ module.exports = class Replicator {
     this.allowFork = allowFork
     this.onpeerupdate = onpeerupdate
     this.onupload = onupload
+    this.ondownloading = null // optional external hook for monitoring downloading status
     this.peers = []
     this.findingPeers = 0 // updateable from the outside
     this.destroyed = false
@@ -1136,7 +1144,11 @@ module.exports = class Replicator {
         if (protomux.opened({ protocol: 'hypercore/alpha', id: this.discoveryKey })) continue
         this._makePeer(protomux, session && session.session({ active: false }))
       }
+    } else {
+      for (const peer of this.peers) peer.closeIfIdle()
     }
+
+    if (this.ondownloading !== null && downloading) this.ondownloading()
   }
 
   cork () {
@@ -1836,6 +1848,10 @@ module.exports = class Replicator {
     session.close().catch(noop)
   }
 
+  attached (protomux) {
+    return this._attached.has(protomux)
+  }
+
   attachTo (protomux, session) {
     const makePeer = this._makePeer.bind(this, protomux, session)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.24.11",
+  "version": "10.25.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {