blind-peer 3.4.0 → 3.6.0

package/index.js

@@ -12,14 +12,25 @@ const safetyCatch = require('safety-catch')
 const Wakeup = require('protomux-wakeup')
 const ScopeLock = require('scope-lock')
 const IdEnc = require('hypercore-id-encoding')
+const ProtomuxRpcClientPool = require('protomux-rpc-client-pool')
+const ProtomuxRpcClient = require('protomux-rpc-client')
+const rrp = require('resolve-reject-promise')
+const IpBanList = require('ip-ban-list')
+const {
+  ADMIN_CHANNEL_ID,
+  AdminQueryTopKEncoding,
+  AddCoreEncoding,
+  DeleteCoreEncoding,
+  RouterResolvePeersRequest,
+  RouterResolvePeersResponse
+} = require('blind-peer-encodings')
 
 const BlindPeerDB = require('./lib/db.js')
+const TopKWindow = require('./lib/top-k.js')
 
 // Enable Small wants in Hypercore. Must be before anywhere that uses Hypercore
 Hypercore.enable(Hypercore.SMALL_WANTS)
 
-const { AddCoreEncoding, DeleteCoreEncoding } = require('blind-peer-encodings')
-
 class CoreTracker {
   constructor(blindPeer, core) {
     this.blindPeer = blindPeer
@@ -197,10 +208,16 @@ class BlindPeer extends ReadyResource {
       maxBytes = 100_000_000_000,
       enableGc = true,
       trustedPubKeys,
+      routerKey,
+      routerPoolOpts,
+      ipBanListKeys = [],
+      banTimeout = 16_000,
       port,
       announcingInterval = 100,
       wakeupGcTickTime = null,
-      replicationLagThreshold = 100
+      replicationLagThreshold = 100,
+      topK = {},
+      adminRouter = null
     } = {}
   ) {
     super()
@@ -208,6 +225,10 @@ class BlindPeer extends ReadyResource {
     this.rocks = typeof rocks === 'string' ? new RocksDB(rocks) : rocks
     this.store = store || new Corestore(this.rocks, { active: false })
     this.swarm = swarm || null
+    const ipBanNs = this.store.namespace('ip-ban-lists')
+    this.ipBanLists = ipBanListKeys.map((key) => new IpBanList(ipBanNs, { key }))
+    this.banTimeout = banTimeout
+
     this._port = port || 0
     this.announcingInterval = announcingInterval
     this.trustedPubKeys = new Set()
@@ -227,6 +248,11 @@ class BlindPeer extends ReadyResource {
     this.announcedCores = new Map()
     this.replicationLagThreshold = replicationLagThreshold
 
+    this.routerKey = routerKey || null
+    this.routerPoolOpts = routerPoolOpts || {}
+    this.routerPool = null
+    this.adminRouter = adminRouter
+
     this.stats = {
       bytesGcd: 0,
       coresAdded: 0,
@@ -236,6 +262,35 @@ class BlindPeer extends ReadyResource {
       muxerPaired: 0,
       muxerErrors: 0
     }
+
+    const {
+      bucketCount = 6,
+      bucketTime = 10_000,
+      k = 5,
+      peerThreshold = 100,
+      referrerThreshold = 100
+    } = topK
+
+    this.topKByPeer = new TopKWindow(bucketCount, bucketTime, k, peerThreshold)
+    this.topKByReferrer = new TopKWindow(bucketCount, bucketTime, k, referrerThreshold)
+    this.topKByIp = new TopKWindow(bucketCount, bucketTime, k, null) // we do not want to expose the ip spike
+
+    if (this.adminRouter) {
+      this.adminRouter.use({
+        onrequest: (ctx, next) => {
+          if (!this._isTrustedPeer(ctx.connection.remotePublicKey)) {
+            throw new Error('Only trusted peers can query top-k')
+          }
+
+          return next()
+        }
+      })
+      this.adminRouter.method(
+        'query-top-k',
+        AdminQueryTopKEncoding,
+        this._onadminquerytopk.bind(this)
+      )
+    }
   }
 
   get encryptionPublicKey() {
@@ -262,6 +317,15 @@ class BlindPeer extends ReadyResource {
     return this.trustedPubKeys.has(IdEnc.normalize(key))
   }
 
+  _onadminquerytopk() {
+    return {
+      version: 1,
+      ip: this.topKByIp.topK,
+      referrer: this.topKByReferrer.topK,
+      peerPublicKey: this.topKByPeer.topK
+    }
+  }
+
   async _open() {
     await this.store.ready()
 
@@ -285,6 +349,23 @@ class BlindPeer extends ReadyResource {
     }
     this.swarm.on('connection', this._onconnection.bind(this))
 
+    await Promise.all(
+      this.ipBanLists.map(async (banIpList) => {
+        await banIpList.ready()
+        this.swarm.join(banIpList.discoveryKey, { server: false, client: true })
+      })
+    )
+
+    if (this.routerKey) {
+      const rpcClient = new ProtomuxRpcClient(this.swarm.dht)
+      this.routerPool = new ProtomuxRpcClientPool([this.routerKey], rpcClient, this.routerPoolOpts)
+    }
+
+    await this.topKByPeer.ready()
+    await this.topKByReferrer.ready()
+    await this.topKByIp.ready()
+    if (this.adminRouter) await this.adminRouter.ready()
+
     this._announceCores().catch(safetyCatch) // announcing cores asynchronously
     this.flushInterval = setInterval(this.flush.bind(this), 10_000)
   }
@@ -432,6 +513,7 @@ class BlindPeer extends ReadyResource {
 
     rpc.respond('add-core', AddCoreEncoding, this._onaddcore.bind(this, conn))
     rpc.respond('delete-core', DeleteCoreEncoding, this._ondeletecore.bind(this, conn))
+    if (this.adminRouter) this.adminRouter.handleConnection(conn, ADMIN_CHANNEL_ID)
 
     const self = this
     BlindPeerMuxer.pair(conn, function () {
@@ -451,6 +533,36 @@ class BlindPeer extends ReadyResource {
     })
   }
 
+  _isBlocked(conn) {
+    // current behavior:
+    // as we do simple check ban
+    // bans can run in parallel across lists, but only the original banner can unban
+    for (const ipBanList of this.ipBanLists) {
+      if (ipBanList.isBanned(conn.rawStream.remoteHost)) {
+        return true
+      }
+    }
+    return false
+  }
+
+  async _timeoutThenThrow() {
+    const { promise, resolve } = rrp()
+
+    const done = () => {
+      clearTimeout(timer)
+      this.off('close', done)
+      resolve()
+    }
+
+    const timer = setTimeout(done, this.banTimeout)
+    timer.unref()
+
+    this.on('close', done)
+
+    await promise
+    throw new Error('Timed out')
+  }
+
   async _activateCore(stream, record) {
     this.stats.activations++
 
@@ -473,6 +585,23 @@ class BlindPeer extends ReadyResource {
     stream.on('close', () => core.close().catch(safetyCatch))
   }
 
+  async _resolvePeers(key) {
+    if (!this.routerPool) return
+
+    const result = await this.routerPool.makeRequest(
+      'resolve-peers',
+      { key },
+      {
+        requestEncoding: RouterResolvePeersRequest,
+        responseEncoding: RouterResolvePeersResponse
+      }
+    )
+
+    this.emit('resolve-peers', { key, result })
+
+    return result
+  }
+
   async _announceCores() {
     for await (const record of this.db.createAnnouncingCoresStream()) {
       if (this.closing) return
@@ -534,6 +663,10 @@ class BlindPeer extends ReadyResource {
 
   async _onaddcore(stream, record) {
     if (!this.opened) await this.ready()
+    if (this._isBlocked(stream)) {
+      this.emit('connection-banned', stream)
+      await this._timeoutThenThrow()
+    }
 
     record.priority = Math.min(record.priority, 1) // 2 is reserved for trusted peers
     if (record.announce !== false && !this._isTrustedPeer(stream.remotePublicKey)) {
@@ -574,9 +707,20 @@ class BlindPeer extends ReadyResource {
   }
 
   async _onaddcores(stream, request) {
+    if (this._isBlocked(stream)) {
+      this.emit('connection-banned', stream)
+      throw new Error('Timed out')
+    }
     this.stats.addCoresRx++
-    const priority = Math.min(request.priority, 1) // 2 is reserved for trusted peers
+
     const { cores, referrer } = request
+    if (referrer) {
+      this.topKByReferrer.hit(IdEnc.normalize(referrer))
+    }
+    this.topKByPeer.hit(IdEnc.normalize(stream.remotePublicKey))
+    this.topKByIp.hit(stream.rawStream.remoteHost)
+
+    const priority = Math.min(request.priority, 1) // 2 is reserved for trusted peers
 
     if (request.announce !== false && !this._isTrustedPeer(stream.remotePublicKey)) {
       // Note: we can't use the original downgrade-announce event because that assumes a 'record' object
@@ -650,6 +794,13 @@ class BlindPeer extends ReadyResource {
       const discoveryKey = core.discoveryKey
       await core.close()
       await this._onwakeup(discoveryKey, muxer)
+
+      // TODO: will process the result in V2
+      // TODO: handle no referrer
+      this._resolvePeers(referrer).catch((err) => {
+        safetyCatch(err)
+        this.emit('resolve-peers-error', { key: referrer, error: err })
+      })
     }
 
     for (const r of recordsToAdd) this.emit('add-new-core', r, true, stream)
@@ -668,6 +819,10 @@ class BlindPeer extends ReadyResource {
   }
 
   async _ondeletecore(stream, { key }) {
+    if (this._isBlocked(stream)) {
+      this.emit('connection-banned', stream)
+      await this._timeoutThenThrow()
+    }
     if (!this._isTrustedPeer(stream.remotePublicKey)) {
       this.emit('delete-blocked', stream, { key })
       throw new Error('Only trusted peers can delete cores')
@@ -713,7 +868,15 @@ class BlindPeer extends ReadyResource {
   }
 
   async _close() {
+    if (this.routerPool) {
+      await this.routerPool.destroy()
+      await this.routerPool.statelessRpc.close()
+    }
+    if (this.adminRouter) await this.adminRouter.close()
     clearInterval(this.flushInterval)
+    await this.topKByPeer.close()
+    await this.topKByReferrer.close()
+    await this.topKByIp.close()
     if (this.ownsWakeup) this.wakeup.destroy()
     if (this.ownsSwarm) await this.swarm.destroy()
     await this.flush()
@@ -822,6 +985,27 @@ class BlindPeer extends ReadyResource {
         this.set(self.stats.addCoresRx)
       }
     })
+    new promClient.Gauge({
+      name: 'blind_peer_add_cores_top5_by_remote_key',
+      help: 'The total number of requests from the top 5 peers in the last minute',
+      collect() {
+        this.set(self.topKByPeer.topKSum())
+      }
+    })
+    new promClient.Gauge({
+      name: 'blind_peer_add_cores_top5_by_referrer',
+      help: 'The total number of requests from the top 5 referrers in the last minute',
+      collect() {
+        this.set(self.topKByReferrer.topKSum())
+      }
+    })
+    new promClient.Gauge({
+      name: 'blind_peer_add_cores_top5_by_remote_ip',
+      help: 'The total number of requests from the top 5 remote IPs in the last minute',
+      collect() {
+        this.set(self.topKByIp.topKSum())
+      }
+    })
     if (self.rocks.stats) {
       new promClient.Gauge({
         // eslint-disable-line no-new

package/lib/top-k.js

@@ -0,0 +1,94 @@
+const ReadyResource = require('ready-resource')
+
+/**
+ * Tracks the most frequent keys within a rolling time window.
+ */
+class TopKWindow extends ReadyResource {
+  /**
+   * @param {number} [bucketCount=6]
+   * @param {number} [bucketTime=10_000]
+   * @param {number} [k=5]
+   * @param {number | null} [spikeThreshold=null]
+   */
+  constructor(bucketCount = 6, bucketTime = 10_000, k = 5, spikeThreshold = null) {
+    super()
+
+    this.bucketCount = bucketCount
+    this.bucketTime = bucketTime
+    this.k = k
+    this.spikeThreshold = spikeThreshold
+
+    /** @type {Map<string, number>[]} */
+    this._buckets = Array.from({ length: bucketCount }, () => new Map())
+    this._index = 0
+    this.topK = []
+    this._timer = null
+  }
+
+  /**
+   * @param {string} key
+   */
+  hit(key) {
+    const bucket = this._buckets[this._index]
+    bucket.set(key, (bucket.get(key) || 0) + 1)
+  }
+
+  /**
+   * @returns {number}
+   */
+  topKSum() {
+    let sum = 0
+    for (const { count } of this.topK) {
+      sum += count
+    }
+    return sum
+  }
+
+  _open() {
+    this._timer = setInterval(this._rotate.bind(this), this.bucketTime)
+  }
+
+  _close() {
+    if (this._timer !== null) {
+      clearInterval(this._timer)
+      this._timer = null
+    }
+
+    this.topK = []
+    for (const bucket of this._buckets) {
+      bucket.clear()
+    }
+  }
+
+  _rotate() {
+    // compute the top-k and cache
+    const totals = new Map()
+    for (const bucket of this._buckets) {
+      for (const [key, count] of bucket) {
+        totals.set(key, (totals.get(key) || 0) + count)
+      }
+    }
+
+    this.topK = [...totals.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, this.k)
+      .map(([key, count]) => ({ key, count }))
+
+    if (this.spikeThreshold !== null) {
+      // Deliberately emit only for cached top-k entries to keep spike volume bounded.
+      for (const { key, count } of this.topK) {
+        if (count >= this.spikeThreshold) {
+          this.emit('spike', key, count)
+        }
+      }
+    }
+
+    // rotate current bucket and clear the old bucket
+    this._index = (this._index + 1) % this.bucketCount
+    this._buckets[this._index].clear()
+
+    this.emit('rotated')
+  }
+}
+
+module.exports = TopKWindow

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "blind-peer",
-  "version": "3.4.0",
+  "version": "3.6.0",
   "description": "Blind peers help keep hypercores available",
   "main": "index.js",
   "dependencies": {
     "autobase": "^7.0.18",
     "b4a": "^1.6.7",
-    "blind-peer-encodings": "^3.1.1",
+    "blind-peer-encodings": "^3.3.0",
     "blind-peer-muxer": "^1.0.0",
     "compact-encoding": "^2.16.0",
     "corestore": "^7.4.4",
@@ -14,12 +14,17 @@
     "hypercore-crypto": "^3.5.0",
     "hypercore-id-encoding": "^1.3.0",
     "hyperdb": "^5.0.0",
+    "hyperdht": "^6.29.0",
     "hyperschema": "^1.10.3",
     "hyperswarm": "^4.13.1",
+    "ip-ban-list": "^0.3.0",
     "protomux-rpc": "^1.7.1",
+    "protomux-rpc-client": "^2.1.0",
+    "protomux-rpc-client-pool": "^2.1.0",
     "protomux-wakeup": "^2.9.0",
     "ready-resource": "^1.1.2",
     "repl-swarm": "^2.3.0",
+    "resolve-reject-promise": "^1.0.3",
     "rocksdb-native": "^3.1.6",
     "safety-catch": "^1.0.2",
     "scope-lock": "^1.2.4"
@@ -28,14 +33,17 @@
     "bare-events": "^2.8.2",
     "bare-process": "^4.2.2",
     "bare-prom-client": "^15.1.3",
-    "blind-peering": "^2.0.0",
+    "blind-peer-router": "^0.2.2",
+    "blind-peering": "^2.1.0",
     "brittle": "^3.7.0",
     "debounceify": "^1.1.0",
-    "hyperdht": "^6.20.1",
+    "graceful-goodbye": "^1.3.3",
+    "hyperdht-address": "^1.0.1",
+    "lunte": "^1.6.0",
     "prettier": "^3.6.2",
     "prettier-config-holepunch": "^2.0.0",
-    "test-tmp": "^1.3.0",
-    "which-runtime": "^1.3.2"
+    "protomux-rpc-router": "^1.2.0",
+    "test-tmp": "^1.3.0"
   },
   "files": [
     "index.js",
@@ -44,7 +52,8 @@
   "scripts": {
     "format": "prettier --write .",
     "format:check": "prettier --check .",
-    "test": "npm run format:check && brittle test/*.js",
+    "test": "npm run format:check && lunte &&  brittle test/*.js",
+    "test:netns": "brittle test-netns/*.test.js",
     "test:bare": "bare test/test.js"
   },
   "repository": {
@@ -61,6 +70,10 @@
     "events": {
       "bare": "bare-events",
       "default": "events"
+    },
+    "prom-client": {
+      "bare": "bare-prom-client",
+      "default": "prom-client"
     }
   }
 }