hypercore 11.20.2 → 11.21.0

package/README.md

@@ -762,7 +762,6 @@ Create an encrypted noise stream with a protomux instance attached used for Hype
 
 ```
 {
-  keepAlive: true, // Whether to keep the stream alive
   ondiscoverykey: () => {}, // A handler for when a discovery key is set over the stream for corestore management
 }
 ```

package/index.js

@@ -146,7 +146,7 @@ class Hypercore extends EventEmitter {
     if (!noiseStream.userData) {
       const protocol = Protomux.from(noiseStream)
 
-      if (opts.keepAlive !== false) {
+      if (opts.keepAlive !== false && noiseStream.keepAlive === 0) {
         noiseStream.setKeepAlive(5000)
       }
       noiseStream.userData = protocol
@@ -938,7 +938,7 @@ class Hypercore extends EventEmitter {
     const isDefault = this.state === this.core.state
     const defaultKeyPair = this.state.name === null ? this.keyPair : null
 
-    const { keyPair = defaultKeyPair, signature = null, maxLength } = opts
+    const { keyPair = defaultKeyPair, signature = null, maxLength, postappend = null } = opts
     const writable =
       !isDefault || !!signature || !!(keyPair && keyPair.secretKey) || opts.writable === true
 
@@ -966,7 +966,7 @@ class Hypercore extends EventEmitter {
       }
     }
 
-    return this.state.append(buffers, { keyPair, signature, preappend, maxLength })
+    return this.state.append(buffers, { keyPair, signature, preappend, postappend, maxLength })
   }
 
   async signable(length = -1, fork = -1) {
@@ -1149,6 +1149,8 @@ function initOnce(session, storage, key, opts) {
     eagerUpgrade: opts.eagerUpgrade !== false,
     notDownloadingLinger: opts.notDownloadingLinger,
     allowFork: opts.allowFork !== false,
+    allowPush: !!opts.allowPush,
+    alwaysLatestBlock: !!opts.allowLatestBlock,
     inflightRange: opts.inflightRange,
     compat: opts.compat === true,
     force: opts.force,

package/lib/core.js

@@ -441,6 +441,7 @@ module.exports = class Core {
     const verifier =
       this.verifier ||
       new Verifier(this.header.key, Verifier.createManifest(manifest), { legacy: this._legacy })
+
     if (!verifier.verify(batch, batch.signature)) {
       throw INVALID_SIGNATURE('Proof contains an invalid signature', this.discoveryKey)
     }
@@ -451,10 +452,6 @@ module.exports = class Core {
   async _verifyExclusive({ batch, bitfield, value, manifest }) {
     manifest = this._verifyBatchUpgrade(batch, manifest)
 
-    if (!batch.commitable()) return false
-
-    if (this.preupdate !== null) await this.preupdate(batch, this.header.key)
-
     if (
       !(await this.state._verifyBlock(
         batch,
@@ -668,7 +665,14 @@ module.exports = class Core {
     if (proof.fork !== this.state.fork) return false
 
     const batch = await MerkleTree.verify(this.state, proof)
-    if (!batch.commitable()) return false
+
+    if (batch.upgraded && batch.length <= this.state.length) {
+      await batch.downgrade()
+    }
+
+    if (!batch.commitable()) {
+      return false
+    }
 
     const value = (proof.block && proof.block.value) || null
     const op = {

package/lib/merkle-tree.js

@@ -293,6 +293,32 @@ class MerkleTreeBatch {
     return offset
   }
 
+  async downgrade() {
+    if (!this.upgraded) return true
+
+    const rx = this.storage.read()
+    const nodePromises = []
+
+    for (const r of this.roots) {
+      nodePromises.push(rx.getTreeNode(r.index))
+    }
+
+    rx.tryFlush()
+
+    const nodes = await Promise.all(nodePromises)
+
+    for (let i = 0; i < this.roots.length; i++) {
+      const root = this.roots[i]
+      const node = nodes[i]
+
+      if (!node) return false
+      if (!b4a.equals(node.hash, root.hash)) return false
+    }
+
+    this.upgraded = false
+    return true
+  }
+
   async restore(length) {
     if (length === this.length) return this
 
@@ -684,6 +710,28 @@ class MerkleTree {
     return generateProof(session, rx, block, hash, seek, upgrade)
   }
 
+  static maxMissingNodes(index, length) {
+    const head = 2 * length
+    const ite = flat.iterator(index)
+
+    // See iterator.rightSpan()
+    const iteRightSpan = ite.index + ite.factor / 2 - 1
+    // If the index is not in the current tree, we do not know how many missing nodes there are...
+    if (iteRightSpan >= head) return 0
+
+    for (const r of flat.fullRoots(head)) {
+      if (r === index) return 0
+    }
+
+    let cnt = 0
+    while (!ite.contains(head)) {
+      cnt++
+      ite.parent()
+    }
+
+    return cnt
+  }
+
   static async missingNodes(session, index, length) {
     const head = 2 * length
     const ite = flat.iterator(index)

package/lib/messages.js

@@ -629,7 +629,8 @@ wire.sync = {
       (m.canUpgrade ? 1 : 0) |
         (m.uploading ? 2 : 0) |
         (m.downloading ? 4 : 0) |
-        (m.hasManifest ? 8 : 0)
+        (m.hasManifest ? 8 : 0) |
+        (m.allowPush ? 16 : 0)
     )
     c.uint.encode(state, m.fork)
     c.uint.encode(state, m.length)
@@ -645,7 +646,8 @@ wire.sync = {
       canUpgrade: (flags & 1) !== 0,
       uploading: (flags & 2) !== 0,
       downloading: (flags & 4) !== 0,
-      hasManifest: (flags & 8) !== 0
+      hasManifest: (flags & 8) !== 0,
+      allowPush: (flags & 16) !== 0
     }
   }
 }

package/lib/replicator.js

@@ -187,6 +187,11 @@ class RangeRequest extends Attachable {
     this.ifAvailable = ifAvailable
     this.blocks = blocks
     this.ranges = ranges
+    this.rangeIndex = ranges.push(this) - 1
+
+    if (this.end === -1) {
+      this.replicator._alwaysLatestBlock++
+    }
 
     // As passed by the user, immut
     this.userStart = start
@@ -194,10 +199,18 @@ class RangeRequest extends Attachable {
   }
 
   _unref() {
-    const i = this.ranges.indexOf(this)
-    if (i === -1) return
+    if (this.rangeIndex >= this.ranges.length) return
+    if (this.ranges[this.rangeIndex] !== this) return
+
     const h = this.ranges.pop()
-    if (i < this.ranges.length) this.ranges[i] = h
+    if (h !== this) {
+      this.ranges[this.rangeIndex] = h
+      h.rangeIndex = this.rangeIndex
+    }
+
+    if (this.end === -1) {
+      this.replicator._alwaysLatestBlock--
+    }
   }
 
   _cancel(r) {
@@ -371,7 +384,6 @@ class ProofRequest {
     const [proof, block] = await Promise.all([this.proof.settle(), this.block])
 
     if (this.manifest) proof.manifest = this.manifest
-
     if (!block && proof.block) return null
 
     if (block) proof.block.value = block
@@ -450,6 +462,8 @@ class Peer {
     this.remoteBitfield = new RemoteBitfield()
     this.missingBlocks = new RemoteBitfield()
 
+    this.pushing = false
+
     this.remoteFork = 0
     this.remoteLength = 0
     this.remoteCanUpgrade = false
@@ -457,6 +471,7 @@ class Peer {
     this.remoteDownloading = true
     this.remoteSynced = false
     this.remoteHasManifest = false
+    this.remoteAllowPush = false
     this.remoteRequests = new Map()
 
     this.segmentsWanted = new Set()
@@ -592,7 +607,8 @@ class Peer {
       canUpgrade: this.canUpgrade,
       uploading: true,
       downloading: this.replicator.isDownloading(),
-      hasManifest: !!this.core.header.manifest && this.core.compat === false
+      hasManifest: !!this.core.header.manifest && this.core.compat === false,
+      allowPush: this.replicator.allowPush
     })
     incrementTx(this.stats.wireSync, this.replicator.stats.wireSync)
   }
@@ -677,7 +693,18 @@ class Peer {
     return false
   }
 
-  async onsync({ fork, length, remoteLength, canUpgrade, uploading, downloading, hasManifest }) {
+  async onsync(msg) {
+    const {
+      fork,
+      length,
+      remoteLength,
+      canUpgrade,
+      uploading,
+      downloading,
+      hasManifest,
+      allowPush
+    } = msg
+
     const lengthChanged = length !== this.remoteLength
     const sameFork = fork === this.core.state.fork
 
@@ -688,6 +715,7 @@ class Peer {
     this.remoteUploading = uploading
     this.remoteDownloading = downloading
     this.remoteHasManifest = hasManifest
+    this.remoteAllowPush = allowPush
 
     if (this.closeIfIdle()) return
 
@@ -783,13 +811,13 @@ class Peer {
     }
   }
 
-  async _getProof(batch, msg) {
+  async _getProof(batch, msg, pushing) {
     let block = null
 
     if (msg.block) {
       const index = msg.block.index
 
-      if (msg.fork !== this.core.state.fork || !this.core.bitfield.get(index)) {
+      if (!pushing && (msg.fork !== this.core.state.fork || !this.core.bitfield.get(index))) {
         return new ProofRequest(msg, null, null, null)
       }
 
@@ -866,13 +894,50 @@ class Peer {
     this.receiverBusy = false
   }
 
+  async push(index) {
+    if (!this.remoteAllowPush) return
+    if (this.core.state.fork !== this.remoteFork) return
+    if (this.remoteBitfield.get(index)) return
+
+    const msg = {
+      id: 0,
+      fork: this.core.state.fork,
+      block: null,
+      hash: null,
+      seek: null,
+      upgrade: null,
+      manifest: this.remoteLength === 0,
+      priority: 0
+    }
+
+    msg.block = {
+      index,
+      nodes: MerkleTree.maxMissingNodes(2 * index, this.remoteLength)
+    }
+
+    if (index >= this.remoteLength) {
+      msg.upgrade = {
+        start: this.remoteLength,
+        length: this.core.state.length - this.remoteLength
+      }
+    }
+
+    const batch = this.core.storage.read()
+    const req = await this._getProof(batch, msg, true)
+    if (req === null) return
+
+    batch.tryFlush()
+
+    await this._fulfillRequest(req, true)
+  }
+
   async _handleRequest(msg) {
     const batch = this.core.storage.read()
 
     // TODO: could still be answerable if (index, fork) is an ancestor of the current fork
     const req =
       msg.fork === this.core.state.fork
-        ? await this._getProof(batch, msg)
+        ? await this._getProof(batch, msg, false)
         : new ProofRequest(msg, null, null, null)
 
     if (req === null) {
@@ -882,20 +947,25 @@ class Peer {
 
     batch.tryFlush()
 
-    await this._fulfillRequest(req)
+    await this._fulfillRequest(req, false)
   }
 
-  async _fulfillRequest(req) {
+  async _fulfillRequest(req, pushing) {
     const proof = await req.fulfill()
 
-    // if cancelled do not reply
-    if (this.remoteRequests.get(req.msg.id) !== req.msg) {
+    if (!pushing) {
+      // if cancelled do not reply
+      if (this.remoteRequests.get(req.msg.id) !== req.msg) {
+        return
+      }
+
+      // sync from now on, so safe to delete from the map
+      this.remoteRequests.delete(req.msg.id)
+    } else if (!this.remoteAllowPush) {
+      // if pushing but remote disabled it, just drop it
       return
     }
 
-    // sync from now on, so safe to delete from the map
-    this.remoteRequests.delete(req.msg.id)
-
     if (!this.isActive() && proof.block !== null) {
       return
     }
@@ -924,6 +994,15 @@ class Peer {
       this.replicator._onupload(proof.block.index, proof.block.value.byteLength, this)
     }
 
+    // TODO: we should prob move to a sep length prop for this. This is just quick and dirty
+    // to produce better push upgrade proofs as we can better predict what length the remote
+    // is going to be at.
+    if ((pushing || this.pushing) && proof.upgrade) {
+      this.pushing = true
+      const remoteLength = proof.upgrade.start + proof.upgrade.length
+      if (remoteLength > this.remoteLength) this.remoteLength = remoteLength
+    }
+
     this.wireData.send({
       request: req.msg.id,
       fork: req.msg.fork,
@@ -975,6 +1054,11 @@ class Peer {
     incrementTx(this.stats.wireRequest, this.replicator.stats.wireRequest)
   }
 
+  _unmarkInflightBlockRequest(req, data) {
+    if (isBlockRequest(req)) this.replicator._unmarkInflight(req.block.index)
+    else if (data.block && !req) this.replicator._unmarkInflight(data.block.index)
+  }
+
   async ondata(data) {
     // always allow a fork conflict proof to be sent
     if (data.request === 0 && data.upgrade && data.upgrade.start === 0) {
@@ -987,7 +1071,14 @@ class Peer {
 
     // no push atm, TODO: check if this satisfies another pending request
     // allow reorg pushes tho as those are not written to storage so we'll take all the help we can get
-    if (req === null && reorg === false) return
+    if (req === null && reorg === false && !this.replicator.allowPush) {
+      return
+    }
+
+    if (req === null && reorg === false && data.block) {
+      // mark this as inflight to avoid parallel requests
+      this.replicator._markInflight(data.block.index)
+    }
 
     if (req !== null) {
       if (req.peer !== this) return
@@ -998,7 +1089,7 @@ class Peer {
       if (reorg === true) return await this.replicator._onreorgdata(this, req, data)
     } catch (err) {
       safetyCatch(err)
-      if (isBlockRequest(req)) this.replicator._unmarkInflight(req.block.index)
+      this._unmarkInflightBlockRequest(req, data)
 
       this.paused = true
       this.replicator._oninvaliddata(err, req, data, this)
@@ -1009,13 +1100,13 @@ class Peer {
     if (isBlockRequest(req)) this.replicator._markProcessing(req.block.index)
 
     try {
-      if (!matchingRequest(req, data) || !(await this.core.verify(data, this))) {
+      if (!this.replicator._matchingRequest(req, data) || !(await this.core.verify(data, this))) {
         this.replicator._onnodata(this, req, 0)
         return
       }
     } catch (err) {
       safetyCatch(err)
-      if (isBlockRequest(req)) this.replicator._unmarkInflight(req.block.index)
+      this._unmarkInflightBlockRequest(req, data)
 
       if (err.code === 'WRITE_FAILED') {
         // For example, we don't want to keep pulling data when storage is full
@@ -1031,7 +1122,12 @@ class Peer {
         this._checkIfConflict()
       }
 
-      this.paused = true
+      // if push mode, we MIGHT get an occasional bad message
+      // no need to mega bail for that. TODO: rate limit instead as a general thing
+      if (!this.replicator.allowPush) {
+        this.paused = true
+      }
+
       this.replicator._onnodata(this, req, 0)
       this.replicator._oninvaliddata(err, req, data, this)
       return
@@ -1049,7 +1145,11 @@ class Peer {
 
   onnodata({ request, reason }) {
     const req = request > 0 ? this.replicator._inflight.get(request) : null
-    if (req === null || req.peer !== this) return
+
+    if (req === null || req.peer !== this) {
+      this.replicator.updateAll()
+      return
+    }
 
     this._onrequestroundtrip(req)
     this.replicator._onnodata(this, req, reason)
@@ -1256,11 +1356,32 @@ class Peer {
     this._send(req)
   }
 
+  _includeLastBlock() {
+    if (this.replicator._alwaysLatestBlock === 0) return null
+
+    const index = this.remoteLength - 1
+
+    // only valid if its an OOB get
+    if (index < this.core.state.length) return null
+
+    // do the normal checks
+    if (!this._remoteHasBlock(index)) return null
+    if (!this._canRequest(index)) return null
+
+    // atm we only ever do one upgrade request in parallel, if that changes
+    // then we should add some check here for inflights to avoid over requesting
+    const b = this.replicator._blocks.add(index, PRIORITY.NORMAL)
+    return b
+  }
+
   _requestUpgrade(u) {
     const req = this._makeRequest(true, 0, 0)
     if (req === null) return false
 
-    this._send(req)
+    const b = this._includeLastBlock()
+
+    if (b) this._sendBlockRequest(req, b)
+    else this._send(req)
 
     return true
   }
@@ -1597,12 +1718,15 @@ module.exports = class Replicator {
       notDownloadingLinger = NOT_DOWNLOADING_SLACK,
       eagerUpgrade = true,
       allowFork = true,
+      allowPush = false,
+      alwaysLatestBlock = false,
       inflightRange = null
     } = {}
   ) {
     this.core = core
     this.eagerUpgrade = eagerUpgrade
     this.allowFork = allowFork
+    this.allowPush = allowPush
     this.ondownloading = null // optional external hook for monitoring downloading status
     this.peers = []
     this.findingPeers = 0 // updatable from the outside
@@ -1635,6 +1759,7 @@ module.exports = class Replicator {
     this._blocks = new BlockTracker(this)
     this._hashes = new BlockTracker(this)
 
+    this._alwaysLatestBlock = alwaysLatestBlock ? 1 : 0
     this._queued = []
 
     this._seeks = []
@@ -1668,6 +1793,20 @@ module.exports = class Replicator {
     return this.downloading || !this._inflight.idle
   }
 
+  async push(index) {
+    const all = []
+    for (const peer of this.peers) {
+      all.push(peer.push(index))
+    }
+    await Promise.all(all)
+  }
+
+  setAllowPush(allowPush) {
+    if (allowPush === this.allowPush) return
+    this.allowPush = allowPush
+    for (const peer of this.peers) peer.sendSync()
+  }
+
   setDownloading(downloading) {
     clearTimeout(this._downloadingTimer)
 
@@ -1843,10 +1982,8 @@ module.exports = class Replicator {
     // Also auto compresses the range based on local bitfield
     clampRange(this.core, r)
 
-    this._ranges.push(r)
-
     if (r.end !== -1 && r.start >= r.end) {
-      this._resolveRangeRequest(r, this._ranges.length - 1)
+      this._resolveRangeRequest(r)
       return ref
     }
 
@@ -1886,6 +2023,25 @@ module.exports = class Replicator {
     if (cleared) this.updateAll()
   }
 
+  _matchingRequest(req, data) {
+    if (this.allowPush) {
+      return true
+    }
+    if (data.block !== null && (req.block === null || req.block.index !== data.block.index)) {
+      return false
+    }
+    if (data.hash !== null && (req.hash === null || req.hash.index !== data.hash.index)) {
+      return false
+    }
+    if (data.seek !== null && (req.seek === null || req.seek.bytes !== data.seek.bytes)) {
+      return false
+    }
+    if (data.upgrade !== null && req.upgrade === null) {
+      return false
+    }
+    return req.fork === data.fork
+  }
+
   _addUpgradeMaybe() {
     return this.eagerUpgrade === true ? this._addUpgrade() : this._upgrade
   }
@@ -2115,12 +2271,9 @@ module.exports = class Replicator {
     return true
   }
 
-  _resolveRangeRequest(req, index) {
-    const head = this._ranges.pop()
-
-    if (index < this._ranges.length) this._ranges[index] = head
-
+  _resolveRangeRequest(req) {
     req.resolve(true)
+    req.gc()
   }
 
   _clearInflightBlock(tracker, req) {
@@ -2143,7 +2296,7 @@ module.exports = class Replicator {
   }
 
   _clearInflightUpgrade(req) {
-    if (removeInflight(this._upgrade.inflight, req) === false) return
+    if (this._upgrade === null || removeInflight(this._upgrade.inflight, req) === false) return
     this._upgrade.gc()
   }
 
@@ -2182,7 +2335,8 @@ module.exports = class Replicator {
         clampRange(this.core, r)
 
         if (r.end !== -1 && r.start >= r.end) {
-          this._resolveRangeRequest(r, i--)
+          this._resolveRangeRequest(r)
+          i--
           if (len > this._ranges.length) len--
           if (this._ranges.length === MAX_RANGES) updateAll = true
         }
@@ -2231,7 +2385,8 @@ module.exports = class Replicator {
       const r = this._ranges[i]
 
       if (r.ifAvailable) {
-        this._resolveRangeRequest(r, i--)
+        this._resolveRangeRequest(r)
+        i--
       }
     }
   }
@@ -2269,7 +2424,10 @@ module.exports = class Replicator {
       }
     }
 
-    this._clearRequest(peer, req)
+    if (req) {
+      this._clearRequest(peer, req)
+    }
+
     this.updateAll()
   }
 
@@ -2322,7 +2480,10 @@ module.exports = class Replicator {
   }
 
   _ondata(peer, req, data) {
-    req.elapsed = Date.now() - req.timestamp
+    if (req) {
+      req.elapsed = Date.now() - req.timestamp
+    }
+
     if (data.block !== null) {
       this._resolveBlockRequest(this._blocks, data.block.index, data.block.value, req)
       this._ondownload(data.block.index, data.block.value.byteLength, peer, req)
@@ -2770,16 +2931,6 @@ module.exports = class Replicator {
   }
 }
 
-function matchingRequest(req, data) {
-  if (data.block !== null && (req.block === null || req.block.index !== data.block.index)) {
-    return false
-  }
-  if (data.hash !== null && (req.hash === null || req.hash.index !== data.hash.index)) return false
-  if (data.seek !== null && (req.seek === null || req.seek.bytes !== data.seek.bytes)) return false
-  if (data.upgrade !== null && req.upgrade === null) return false
-  return req.fork === data.fork
-}
-
 function removeHotswap(block) {
   if (block.hotswap === null) return false
   block.hotswap.ref.remove(block)

package/lib/session-state.js

@@ -325,7 +325,18 @@ module.exports = class SessionState {
     await this.mutex.lock()
 
     try {
-      if (!batch.commitable()) return false
+      if (batch.upgraded && batch.length <= this.length) {
+        await batch.downgrade()
+      }
+
+      if (!batch.commitable()) {
+        return false
+      }
+
+      if (this.core.preupdate !== null) {
+        await this.core.preupdate(batch, this.core.header.key)
+      }
+
       const tx = this.createWriteBatch()
       this.updating = true
 
@@ -551,7 +562,7 @@ module.exports = class SessionState {
     }
   }
 
-  async append(values, { signature, keyPair, preappend, maxLength = -1 } = {}) {
+  async append(values, { signature, keyPair, preappend, postappend, maxLength = -1 } = {}) {
     if (!keyPair && this.isDefault()) keyPair = this.core.header.keyPair
 
     await this.mutex.lock()
@@ -623,6 +634,8 @@ module.exports = class SessionState {
       this.byteLength = batch.byteLength
       this.signature = batch.signature
 
+      if (postappend) await postappend(values)
+
       this.onappend(tree, bitfield, flushed)
 
       if (this.core.hintsChanged && this.isDefault()) await this.core.flushHints()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.20.2",
+  "version": "11.21.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {