blind-peer 3.2.0 → 3.5.0

package/index.js

@@ -12,10 +12,20 @@ const safetyCatch = require('safety-catch')
 const Wakeup = require('protomux-wakeup')
 const ScopeLock = require('scope-lock')
 const IdEnc = require('hypercore-id-encoding')
+const ProtomuxRpcClientPool = require('protomux-rpc-client-pool')
+const ProtomuxRpcClient = require('protomux-rpc-client')
+const {
+  AddCoreEncoding,
+  DeleteCoreEncoding,
+  RouterResolvePeersRequest,
+  RouterResolvePeersResponse
+} = require('blind-peer-encodings')
 
 const BlindPeerDB = require('./lib/db.js')
+const TopKWindow = require('./lib/top-k.js')
 
-const { AddCoreEncoding, DeleteCoreEncoding } = require('blind-peer-encodings')
+// Enable Small wants in Hypercore. Must be before anywhere that uses Hypercore
+Hypercore.enable(Hypercore.SMALL_WANTS)
 
 class CoreTracker {
   constructor(blindPeer, core) {
@@ -194,10 +204,13 @@ class BlindPeer extends ReadyResource {
       maxBytes = 100_000_000_000,
       enableGc = true,
       trustedPubKeys,
+      routerKey,
+      routerPoolOpts,
       port,
       announcingInterval = 100,
       wakeupGcTickTime = null,
-      replicationLagThreshold = 100
+      replicationLagThreshold = 100,
+      topK = {}
     } = {}
   ) {
     super()
@@ -224,6 +237,10 @@ class BlindPeer extends ReadyResource {
     this.announcedCores = new Map()
     this.replicationLagThreshold = replicationLagThreshold
 
+    this.routerKey = routerKey || null
+    this.routerPoolOpts = routerPoolOpts || {}
+    this.routerPool = null
+
     this.stats = {
       bytesGcd: 0,
       coresAdded: 0,
@@ -233,6 +250,17 @@ class BlindPeer extends ReadyResource {
       muxerPaired: 0,
       muxerErrors: 0
     }
+
+    const {
+      bucketCount = 6,
+      bucketTime = 10_000,
+      k = 5,
+      peerThreshold = 100,
+      referrerThreshold = 100
+    } = topK
+
+    this.topKByPeer = new TopKWindow(bucketCount, bucketTime, k, peerThreshold)
+    this.topKByReferrer = new TopKWindow(bucketCount, bucketTime, k, referrerThreshold)
   }
 
   get encryptionPublicKey() {
@@ -282,6 +310,14 @@ class BlindPeer extends ReadyResource {
     }
     this.swarm.on('connection', this._onconnection.bind(this))
 
+    if (this.routerKey) {
+      const rpcClient = new ProtomuxRpcClient(this.swarm.dht)
+      this.routerPool = new ProtomuxRpcClientPool([this.routerKey], rpcClient, this.routerPoolOpts)
+    }
+
+    await this.topKByPeer.ready()
+    await this.topKByReferrer.ready()
+
     this._announceCores().catch(safetyCatch) // announcing cores asynchronously
     this.flushInterval = setInterval(this.flush.bind(this), 10_000)
   }
@@ -470,6 +506,23 @@ class BlindPeer extends ReadyResource {
     stream.on('close', () => core.close().catch(safetyCatch))
   }
 
+  async _resolvePeers(key) {
+    if (!this.routerPool) return
+
+    const result = await this.routerPool.makeRequest(
+      'resolve-peers',
+      { key },
+      {
+        requestEncoding: RouterResolvePeersRequest,
+        responseEncoding: RouterResolvePeersResponse
+      }
+    )
+
+    this.emit('resolve-peers', { key, result })
+
+    return result
+  }
+
   async _announceCores() {
     for await (const record of this.db.createAnnouncingCoresStream()) {
       if (this.closing) return
@@ -572,8 +625,14 @@ class BlindPeer extends ReadyResource {
 
   async _onaddcores(stream, request) {
     this.stats.addCoresRx++
-    const priority = Math.min(request.priority, 1) // 2 is reserved for trusted peers
+
     const { cores, referrer } = request
+    if (referrer) {
+      this.topKByReferrer.hit(IdEnc.normalize(referrer))
+    }
+    this.topKByPeer.hit(IdEnc.normalize(stream.remotePublicKey))
+
+    const priority = Math.min(request.priority, 1) // 2 is reserved for trusted peers
 
     if (request.announce !== false && !this._isTrustedPeer(stream.remotePublicKey)) {
       // Note: we can't use the original downgrade-announce event because that assumes a 'record' object
@@ -647,6 +706,13 @@ class BlindPeer extends ReadyResource {
       const discoveryKey = core.discoveryKey
       await core.close()
       await this._onwakeup(discoveryKey, muxer)
+
+      // TODO: will process the result in V2
+      // TODO: handle no referrer
+      this._resolvePeers(referrer).catch((err) => {
+        safetyCatch(err)
+        this.emit('resolve-peers-error', { key: referrer, error: err })
+      })
     }
 
     for (const r of recordsToAdd) this.emit('add-new-core', r, true, stream)
@@ -710,7 +776,13 @@ class BlindPeer extends ReadyResource {
   }
 
   async _close() {
+    if (this.routerPool) {
+      await this.routerPool.destroy()
+      await this.routerPool.statelessRpc.close()
+    }
     clearInterval(this.flushInterval)
+    await this.topKByPeer.close()
+    await this.topKByReferrer.close()
     if (this.ownsWakeup) this.wakeup.destroy()
     if (this.ownsSwarm) await this.swarm.destroy()
     await this.flush()
@@ -819,6 +891,70 @@ class BlindPeer extends ReadyResource {
         this.set(self.stats.addCoresRx)
       }
     })
+    new promClient.Gauge({
+      name: 'blind_peer_add_cores_top5_by_remote_key',
+      help: 'The total number of requests from the top 5 peers in the last minute',
+      collect() {
+        this.set(self.topKByPeer.topKSum())
+      }
+    })
+    new promClient.Gauge({
+      name: 'blind_peer_add_cores_top5_by_referrer',
+      help: 'The total number of requests from the top 5 referrers in the last minute',
+      collect() {
+        this.set(self.topKByReferrer.topKSum())
+      }
+    })
+    if (self.rocks.stats) {
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_gets',
+        help: 'The amount of get ops from RocksDB',
+        collect() {
+          this.set(self.rocks.stats.gets)
+        }
+      })
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_puts',
+        help: 'The amount of put ops to RocksDB',
+        collect() {
+          this.set(self.rocks.stats.puts)
+        }
+      })
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_deletes',
+        help: 'The amount of delete ops from RocksDB',
+        collect() {
+          this.set(self.rocks.stats.deletes)
+        }
+      })
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_range_deletes',
+        help: 'The amount of range delete ops from RocksDB',
+        collect() {
+          this.set(self.rocks.stats.rangeDeletes)
+        }
+      })
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_read_batches',
+        help: 'The amount of read batches from RocksDB',
+        collect() {
+          this.set(self.rocks.stats.readBatches)
+        }
+      })
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_write_batches',
+        help: 'The amount of write batches to RocksDB',
+        collect() {
+          this.set(self.rocks.stats.writeBatches)
+        }
+      })
+    }
   }
 }
 

package/lib/top-k.js

@@ -0,0 +1,94 @@
+const ReadyResource = require('ready-resource')
+
+/**
+ * Tracks the most frequent keys within a rolling time window.
+ */
+class TopKWindow extends ReadyResource {
+  /**
+   * @param {number} [bucketCount=6]
+   * @param {number} [bucketTime=10_000]
+   * @param {number} [k=5]
+   * @param {number | null} [spikeThreshold=null]
+   */
+  constructor(bucketCount = 6, bucketTime = 10_000, k = 5, spikeThreshold = null) {
+    super()
+
+    this.bucketCount = bucketCount
+    this.bucketTime = bucketTime
+    this.k = k
+    this.spikeThreshold = spikeThreshold
+
+    /** @type {Map<string, number>[]} */
+    this._buckets = Array.from({ length: bucketCount }, () => new Map())
+    this._index = 0
+    this.topK = []
+    this._timer = null
+  }
+
+  /**
+   * @param {string} key
+   */
+  hit(key) {
+    const bucket = this._buckets[this._index]
+    bucket.set(key, (bucket.get(key) || 0) + 1)
+  }
+
+  /**
+   * @returns {number}
+   */
+  topKSum() {
+    let sum = 0
+    for (const { count } of this.topK) {
+      sum += count
+    }
+    return sum
+  }
+
+  async _open() {
+    this._timer = setInterval(this._rotate.bind(this), this.bucketTime)
+  }
+
+  async _close() {
+    if (this._timer !== null) {
+      clearInterval(this._timer)
+      this._timer = null
+    }
+
+    this.topK = []
+    for (const bucket of this._buckets) {
+      bucket.clear()
+    }
+  }
+
+  _rotate() {
+    // compute the top-k and cache
+    const totals = new Map()
+    for (const bucket of this._buckets) {
+      for (const [key, count] of bucket) {
+        totals.set(key, (totals.get(key) || 0) + count)
+      }
+    }
+
+    this.topK = [...totals.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, this.k)
+      .map(([key, count]) => ({ key, count }))
+
+    if (this.spikeThreshold !== null) {
+      // Deliberately emit only for cached top-k entries to keep spike volume bounded.
+      for (const { key, count } of this.topK) {
+        if (count >= this.spikeThreshold) {
+          this.emit('spike', key, count)
+        }
+      }
+    }
+
+    // rotate current bucket and clear the old bucket
+    this._index = (this._index + 1) % this.bucketCount
+    this._buckets[this._index].clear()
+
+    this.emit('rotated')
+  }
+}
+
+module.exports = TopKWindow

package/package.json

@@ -1,22 +1,25 @@
 {
   "name": "blind-peer",
-  "version": "3.2.0",
+  "version": "3.5.0",
   "description": "Blind peers help keep hypercores available",
   "main": "index.js",
   "dependencies": {
     "autobase": "^7.0.18",
     "b4a": "^1.6.7",
-    "blind-peer-encodings": "^3.1.1",
+    "blind-peer-encodings": "^3.2.0",
     "blind-peer-muxer": "^1.0.0",
     "compact-encoding": "^2.16.0",
     "corestore": "^7.4.4",
-    "hypercore": "^11.0.12",
+    "hypercore": "^11.26.0",
     "hypercore-crypto": "^3.5.0",
     "hypercore-id-encoding": "^1.3.0",
     "hyperdb": "^5.0.0",
+    "hyperdht": "^6.29.0",
     "hyperschema": "^1.10.3",
     "hyperswarm": "^4.13.1",
     "protomux-rpc": "^1.7.1",
+    "protomux-rpc-client": "^2.1.0",
+    "protomux-rpc-client-pool": "^2.1.0",
     "protomux-wakeup": "^2.9.0",
     "ready-resource": "^1.1.2",
     "repl-swarm": "^2.3.0",
@@ -28,14 +31,15 @@
     "bare-events": "^2.8.2",
     "bare-process": "^4.2.2",
     "bare-prom-client": "^15.1.3",
-    "blind-peering": "^2.0.0",
+    "blind-peer-router": "^0.2.2",
+    "blind-peering": "^2.0.1",
     "brittle": "^3.7.0",
     "debounceify": "^1.1.0",
-    "hyperdht": "^6.20.1",
+    "lunte": "^1.6.0",
     "prettier": "^3.6.2",
     "prettier-config-holepunch": "^2.0.0",
-    "test-tmp": "^1.3.0",
-    "which-runtime": "^1.3.2"
+    "protomux-rpc-router": "^1.2.0",
+    "test-tmp": "^1.3.0"
   },
   "files": [
     "index.js",
@@ -44,7 +48,7 @@
   "scripts": {
     "format": "prettier --write .",
     "format:check": "prettier --check .",
-    "test": "npm run format:check && brittle test/*.js",
+    "test": "npm run format:check && lunte &&  brittle test/*.js",
     "test:bare": "bare test/test.js"
   },
   "repository": {
@@ -61,6 +65,10 @@
     "events": {
       "bare": "bare-events",
       "default": "events"
+    },
+    "prom-client": {
+      "bare": "bare-prom-client",
+      "default": "prom-client"
     }
   }
 }