hypercore 11.34.1 → 11.35.1

package/index.js

@@ -23,7 +23,8 @@ const Download = require('./lib/download')
 const DefaultEncryption = require('./lib/default-encryption')
 const caps = require('./lib/caps')
 const Replicator = require('./lib/replicator')
-const { manifestHash, createManifest } = require('./lib/verifier')
+const Verifier = require('./lib/verifier')
+const { manifestHash, createManifest, encodeManifest } = Verifier
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const { MerkleTree } = require('./lib/merkle-tree')
 const { proof, verify } = require('./lib/fully-remote-proof')
@@ -35,7 +36,8 @@ const {
   SESSION_NOT_WRITABLE,
   SNAPSHOT_NOT_AVAILABLE,
   DECODING_ERROR,
-  REQUEST_CANCELLED
+  REQUEST_CANCELLED,
+  INVALID_CHECKSUM
 } = require('hypercore-errors')
 
 // Hypercore actually does not have any notion of max/min block sizes
@@ -130,11 +132,29 @@ class Hypercore extends EventEmitter {
 
   static key(manifest, { compat, version, namespace } = {}) {
     if (b4a.isBuffer(manifest)) {
-      manifest = { version, signers: [{ publicKey: manifest, namespace }] }
+      manifest =
+        manifest.byteLength === 32
+          ? { version, signers: [{ publicKey: manifest, namespace }] }
+          : this.parseManifest(manifest)
     }
     return compat ? manifest.signers[0].publicKey : manifestHash(createManifest(manifest))
   }
 
+  static parseManifest(manifest, key) {
+    const parsed = createManifest(manifest)
+    if (parsed === null) return null
+
+    if (parsed.quorum > parsed.signers.length) {
+      throw BAD_ARGUMENT('Quorum should not be higher than the number of signers')
+    }
+
+    if (key && !Verifier.isValidManifest(key, parsed)) {
+      throw INVALID_CHECKSUM('Manifest does not hash to provided key')
+    }
+
+    return parsed
+  }
+
   static discoveryKey(key) {
     return crypto.discoveryKey(key)
   }
@@ -453,7 +473,7 @@ class Hypercore extends EventEmitter {
     }
 
     if (opts.manifest && !this.core.header.manifest) {
-      await this.core.setManifest(createManifest(opts.manifest))
+      await this.core.setManifest(opts.manifest)
     }
 
     this.core.replicator.updateActivity(this._active ? 1 : 0)
@@ -606,6 +626,11 @@ class Hypercore extends EventEmitter {
     return this.core === null ? null : this.core.manifest
   }
 
+  getManifest({ raw = false } = {}) {
+    if (this.manifest === null) return null
+    return raw ? encodeManifest(this.manifest) : this.manifest
+  }
+
   get length() {
     if (this._snapshot) return this._snapshot.length
     return this.opened === false ? 0 : this.state.length

package/lib/core.js

@@ -259,15 +259,14 @@ module.exports = class Core {
     }
 
     if (opts.manifest) {
+      const manifest = Verifier.createManifest(opts.manifest)
+
       // if we provide a manifest and no key, verify that the stored key is the same
-      if (
-        !opts.key &&
-        !Verifier.isValidManifest(header.key, Verifier.createManifest(opts.manifest))
-      ) {
+      if (!opts.key && !Verifier.isValidManifest(header.key, manifest)) {
         throw STORAGE_CONFLICT('Manifest does not hash to provided key', this.discoveryKey)
       }
 
-      if (!header.manifest) header.manifest = opts.manifest
+      if (!header.manifest) header.manifest = manifest
     }
 
     if (opts.key && !b4a.equals(header.key, opts.key)) {
@@ -393,12 +392,14 @@ module.exports = class Core {
 
     try {
       if (manifest && this.header.manifest === null) {
+        manifest = Verifier.createManifest(manifest)
+
         if (!Verifier.isValidManifest(this.header.key, manifest)) {
           throw INVALID_CHECKSUM('Manifest hash does not match', this.discoveryKey)
         }
 
         const tx = this.state.createWriteBatch()
-        this._setManifest(tx, Verifier.createManifest(manifest), null)
+        this._setManifest(tx, manifest, null)
         await this.state.flush()
       }
     } finally {

package/lib/replicator.js

@@ -285,6 +285,10 @@ class RangeRequest extends Attachable {
       this.ranges[rangeIndex] = h
     }
 
+    if (!this.resolved && this.replicator._updatesRunning) {
+      this.replicator._updatesQueued = true
+    }
+
     if (this.end === -1) {
       this.replicator._alwaysLatestBlock--
     }
@@ -1541,8 +1545,8 @@ class Peer {
   }
 
   _requestSeek(s) {
-    // if replicator is updating the seeks etc, bail and wait for it to drain
-    if (this.replicator._updatesPending > 0) return false
+    // if replicator is updating the seeks, bail and wait for it to drain
+    if (this.replicator._updatingSeeks) return false
     if (this.replicator.pushOnly) return false
 
     const { length, fork } = this.core.state
@@ -2028,7 +2032,9 @@ module.exports = class Replicator {
     this._hadPeers = false
     this._active = 0
     this._ifAvailable = 0
-    this._updatesPending = 0
+    this._updatingSeeks = false
+    this._updatesRunning = false
+    this._updatesQueued = false
     this._applyingReorg = null
     this._manifestPeer = null
     this._notDownloadingLinger = notDownloadingLinger
@@ -2614,55 +2620,117 @@ module.exports = class Replicator {
     }
   }
 
+  _updateRanges(index, limit) {
+    index = Math.min(index, this._ranges.length - 1)
+
+    let checked = 0
+    let resolved = 0
+    let updateAll = false
+    let remaining = Math.min(limit, this._ranges.length)
+
+    while (remaining-- > 0 && this._ranges.length > 0) {
+      if (index >= this._ranges.length) index = 0
+
+      const r = this._ranges[index]
+
+      clampRange(this.core, r)
+      checked++
+
+      if (r.end !== -1 && r.start >= r.end) {
+        this._resolveRangeRequest(r)
+        resolved++
+        // Crossing into the capped window exposes another range to peer scheduling.
+        if (this._ranges.length === MAX_RANGES) updateAll = true
+      } else {
+        index++
+      }
+    }
+
+    index = this._ranges.length === 0 ? 0 : index % this._ranges.length
+
+    return { checked, resolved, index, updateAll }
+  }
+
   // "slow" updates here - async but not allowed to ever throw
   async _updateNonPrimary(updateAll) {
-    // Check if running, if so skip it and the running one will issue another update for us (debounce)
-    while (++this._updatesPending === 1) {
-      let len = Math.min(MAX_RANGES, this._ranges.length)
+    this._updatesQueued = true
+    if (this._updatesRunning) {
+      if (this._inflight.idle || updateAll) this.queueUpdateAll()
+      return
+    }
 
-      for (let i = 0; i < len; i++) {
-        const r = this._ranges[i]
+    this._updatesRunning = true
 
-        clampRange(this.core, r)
+    while (this._updatesQueued) {
+      this._updatesQueued = false
 
-        if (r.end !== -1 && r.start >= r.end) {
-          this._resolveRangeRequest(r)
-          i--
-          if (len > this._ranges.length) len--
-          if (this._ranges.length === MAX_RANGES) updateAll = true
-        }
-      }
+      let checkedSinceResolve = 0
+      let rangeIndex = 0
+      const drain = this._inflight.idle || updateAll
+      const checkedSeeks = new Set()
 
-      for (let i = 0; i < this._seeks.length; i++) {
-        const s = this._seeks[i]
+      while (true) {
+        const limit = Math.min(MAX_RANGES, this._ranges.length - checkedSinceResolve)
+        const {
+          checked,
+          resolved,
+          index,
+          updateAll: resultUpdateAll
+        } = this._updateRanges(rangeIndex, limit)
+        rangeIndex = index
 
-        let err = null
-        let res = null
+        if (resultUpdateAll) updateAll = true
 
-        try {
-          res = await s.seeker.update()
-        } catch (error) {
-          err = error
-        }
+        if (resolved > 0) checkedSinceResolve = 0
+        else checkedSinceResolve += checked
+
+        const continueRanges = checked > 0 && checkedSinceResolve < this._ranges.length && drain
+
+        this._updatingSeeks = true
+
+        let checkedSeek = false
+        for (const s of this._seeks.slice()) {
+          if (checkedSeeks.has(s) || !this._seeks.includes(s)) continue
+          checkedSeeks.add(s)
+          checkedSeek = true
+
+          let err = null
+          let res = null
+
+          try {
+            res = await s.seeker.update()
+          } catch (error) {
+            err = error
+          }
 
-        if (!res && !err) continue
+          const seekIndex = this._seeks.indexOf(s)
+          if (seekIndex === -1 || (!res && !err)) continue
 
-        if (i < this._seeks.length - 1) this._seeks[i] = this._seeks.pop()
-        else this._seeks.pop()
+          const h = this._seeks.pop()
+          if (h !== s) this._seeks[seekIndex] = h
 
-        i--
+          if (err) s.reject(err)
+          else s.resolve(res)
+        }
+
+        this._updatingSeeks = false
+
+        if (
+          (!continueRanges || (checkedSeek && this._seeks.length > 0)) &&
+          (this._inflight.idle || updateAll)
+        ) {
+          this.queueUpdateAll()
+        }
+        if (!continueRanges) break
 
-        if (err) s.reject(err)
-        else s.resolve(res)
+        await yieldToLoop()
+        if (this.destroyed) break
       }
 
-      // No additional updates scheduled - break
-      if (--this._updatesPending === 0) break
-      // Debounce the additional updates - continue
-      this._updatesPending = 0
+      if (this.destroyed) break
     }
 
-    if (this._inflight.idle || updateAll) this.queueUpdateAll()
+    this._updatesRunning = false
   }
 
   _clearRequest(peer, req) {
@@ -3398,6 +3466,10 @@ function incrementRx(stats1, stats2) {
 
 function noop() {}
 
+function yieldToLoop() {
+  return new Promise((resolve) => setImmediate(resolve))
+}
+
 function backoff(times) {
   const sleep = times < 2 ? 200 : times < 5 ? 500 : times < 40 ? 1000 : 5000
   return new Promise((resolve) => setTimeout(resolve, sleep))

package/lib/verifier.js

@@ -214,6 +214,8 @@ module.exports = class Verifier {
   static createManifest(inp) {
     if (!inp) return null
 
+    if (b4a.isBuffer(inp)) return Verifier.decodeManifest(inp)
+
     if (inp.quorum && inp.quorum < 0) throw BAD_ARGUMENT('Quorum cannot be negative')
     if (inp.quorum && inp.signers && inp.quorum > inp.signers.length) {
       throw BAD_ARGUMENT('Quorum should not be higher than the number of signers')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.34.1",
+  "version": "11.35.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {