blind-peer 3.5.0 → 3.7.0

package/index.js

@@ -14,7 +14,11 @@ const ScopeLock = require('scope-lock')
 const IdEnc = require('hypercore-id-encoding')
 const ProtomuxRpcClientPool = require('protomux-rpc-client-pool')
 const ProtomuxRpcClient = require('protomux-rpc-client')
+const rrp = require('resolve-reject-promise')
+const IpBanList = require('ip-ban-list')
 const {
+  ADMIN_CHANNEL_ID,
+  AdminQueryTopKEncoding,
   AddCoreEncoding,
   DeleteCoreEncoding,
   RouterResolvePeersRequest,
@@ -206,11 +210,14 @@ class BlindPeer extends ReadyResource {
       trustedPubKeys,
       routerKey,
       routerPoolOpts,
+      ipBanListKeys = [],
+      banTimeout = 16_000,
       port,
       announcingInterval = 100,
       wakeupGcTickTime = null,
       replicationLagThreshold = 100,
-      topK = {}
+      topK = {},
+      adminRouter = null
     } = {}
   ) {
     super()
@@ -218,6 +225,10 @@ class BlindPeer extends ReadyResource {
     this.rocks = typeof rocks === 'string' ? new RocksDB(rocks) : rocks
     this.store = store || new Corestore(this.rocks, { active: false })
     this.swarm = swarm || null
+    const ipBanNs = this.store.namespace('ip-ban-lists')
+    this.ipBanLists = ipBanListKeys.map((key) => new IpBanList(ipBanNs, { key }))
+    this.banTimeout = banTimeout
+
     this._port = port || 0
     this.announcingInterval = announcingInterval
     this.trustedPubKeys = new Set()
@@ -240,6 +251,7 @@ class BlindPeer extends ReadyResource {
     this.routerKey = routerKey || null
     this.routerPoolOpts = routerPoolOpts || {}
     this.routerPool = null
+    this.adminRouter = adminRouter
 
     this.stats = {
       bytesGcd: 0,
@@ -261,6 +273,24 @@ class BlindPeer extends ReadyResource {
 
     this.topKByPeer = new TopKWindow(bucketCount, bucketTime, k, peerThreshold)
     this.topKByReferrer = new TopKWindow(bucketCount, bucketTime, k, referrerThreshold)
+    this.topKByIp = new TopKWindow(bucketCount, bucketTime, k, null) // we do not want to expose the ip spike
+
+    if (this.adminRouter) {
+      this.adminRouter.use({
+        onrequest: (ctx, next) => {
+          if (!this._isTrustedPeer(ctx.connection.remotePublicKey)) {
+            throw new Error('Only trusted peers can query top-k')
+          }
+
+          return next()
+        }
+      })
+      this.adminRouter.method(
+        'query-top-k',
+        AdminQueryTopKEncoding,
+        this._onadminquerytopk.bind(this)
+      )
+    }
   }
 
   get encryptionPublicKey() {
@@ -287,6 +317,15 @@ class BlindPeer extends ReadyResource {
     return this.trustedPubKeys.has(IdEnc.normalize(key))
   }
 
+  _onadminquerytopk() {
+    return {
+      version: 1,
+      ip: this.topKByIp.topK,
+      referrer: this.topKByReferrer.topK,
+      peerPublicKey: this.topKByPeer.topK
+    }
+  }
+
   async _open() {
     await this.store.ready()
 
@@ -310,6 +349,13 @@ class BlindPeer extends ReadyResource {
     }
     this.swarm.on('connection', this._onconnection.bind(this))
 
+    await Promise.all(
+      this.ipBanLists.map(async (banIpList) => {
+        await banIpList.ready()
+        this.swarm.join(banIpList.discoveryKey, { server: false, client: true })
+      })
+    )
+
     if (this.routerKey) {
       const rpcClient = new ProtomuxRpcClient(this.swarm.dht)
       this.routerPool = new ProtomuxRpcClientPool([this.routerKey], rpcClient, this.routerPoolOpts)
@@ -317,6 +363,8 @@ class BlindPeer extends ReadyResource {
 
     await this.topKByPeer.ready()
     await this.topKByReferrer.ready()
+    await this.topKByIp.ready()
+    if (this.adminRouter) await this.adminRouter.ready()
 
     this._announceCores().catch(safetyCatch) // announcing cores asynchronously
     this.flushInterval = setInterval(this.flush.bind(this), 10_000)
@@ -465,6 +513,7 @@ class BlindPeer extends ReadyResource {
 
     rpc.respond('add-core', AddCoreEncoding, this._onaddcore.bind(this, conn))
     rpc.respond('delete-core', DeleteCoreEncoding, this._ondeletecore.bind(this, conn))
+    if (this.adminRouter) this.adminRouter.handleConnection(conn, ADMIN_CHANNEL_ID)
 
     const self = this
     BlindPeerMuxer.pair(conn, function () {
@@ -484,6 +533,36 @@ class BlindPeer extends ReadyResource {
     })
   }
 
+  _isBlocked(conn) {
+    // current behavior:
+    // as we do simple check ban
+    // bans can run in parallel across lists, but only the original banner can unban
+    for (const ipBanList of this.ipBanLists) {
+      if (ipBanList.isBanned(conn.rawStream.remoteHost)) {
+        return true
+      }
+    }
+    return false
+  }
+
+  async _timeoutThenThrow() {
+    const { promise, resolve } = rrp()
+
+    const done = () => {
+      clearTimeout(timer)
+      this.off('close', done)
+      resolve()
+    }
+
+    const timer = setTimeout(done, this.banTimeout)
+    timer.unref()
+
+    this.on('close', done)
+
+    await promise
+    throw new Error('Timed out')
+  }
+
   async _activateCore(stream, record) {
     this.stats.activations++
 
@@ -584,6 +663,10 @@ class BlindPeer extends ReadyResource {
 
   async _onaddcore(stream, record) {
     if (!this.opened) await this.ready()
+    if (this._isBlocked(stream)) {
+      this.emit('connection-banned', stream)
+      await this._timeoutThenThrow()
+    }
 
     record.priority = Math.min(record.priority, 1) // 2 is reserved for trusted peers
     if (record.announce !== false && !this._isTrustedPeer(stream.remotePublicKey)) {
@@ -624,6 +707,10 @@ class BlindPeer extends ReadyResource {
   }
 
   async _onaddcores(stream, request) {
+    if (this._isBlocked(stream)) {
+      this.emit('connection-banned', stream)
+      throw new Error('Timed out')
+    }
     this.stats.addCoresRx++
 
     const { cores, referrer } = request
@@ -631,6 +718,7 @@ class BlindPeer extends ReadyResource {
       this.topKByReferrer.hit(IdEnc.normalize(referrer))
     }
     this.topKByPeer.hit(IdEnc.normalize(stream.remotePublicKey))
+    this.topKByIp.hit(stream.rawStream.remoteHost)
 
     const priority = Math.min(request.priority, 1) // 2 is reserved for trusted peers
 
@@ -731,6 +819,10 @@ class BlindPeer extends ReadyResource {
   }
 
   async _ondeletecore(stream, { key }) {
+    if (this._isBlocked(stream)) {
+      this.emit('connection-banned', stream)
+      await this._timeoutThenThrow()
+    }
     if (!this._isTrustedPeer(stream.remotePublicKey)) {
       this.emit('delete-blocked', stream, { key })
       throw new Error('Only trusted peers can delete cores')
@@ -780,9 +872,11 @@ class BlindPeer extends ReadyResource {
       await this.routerPool.destroy()
       await this.routerPool.statelessRpc.close()
     }
+    if (this.adminRouter) await this.adminRouter.close()
     clearInterval(this.flushInterval)
     await this.topKByPeer.close()
     await this.topKByReferrer.close()
+    await this.topKByIp.close()
     if (this.ownsWakeup) this.wakeup.destroy()
     if (this.ownsSwarm) await this.swarm.destroy()
     await this.flush()
@@ -905,6 +999,13 @@ class BlindPeer extends ReadyResource {
         this.set(self.topKByReferrer.topKSum())
       }
     })
+    new promClient.Gauge({
+      name: 'blind_peer_add_cores_top5_by_remote_ip',
+      help: 'The total number of requests from the top 5 remote IPs in the last minute',
+      collect() {
+        this.set(self.topKByIp.topKSum())
+      }
+    })
     if (self.rocks.stats) {
       new promClient.Gauge({
         // eslint-disable-line no-new

package/lib/top-k.js

@@ -44,11 +44,11 @@ class TopKWindow extends ReadyResource {
     return sum
   }
 
-  async _open() {
+  _open() {
     this._timer = setInterval(this._rotate.bind(this), this.bucketTime)
   }
 
-  async _close() {
+  _close() {
     if (this._timer !== null) {
       clearInterval(this._timer)
       this._timer = null

package/package.json

@@ -1,28 +1,30 @@
 {
   "name": "blind-peer",
-  "version": "3.5.0",
+  "version": "3.7.0",
   "description": "Blind peers help keep hypercores available",
   "main": "index.js",
   "dependencies": {
     "autobase": "^7.0.18",
     "b4a": "^1.6.7",
-    "blind-peer-encodings": "^3.2.0",
+    "blind-peer-encodings": "^3.3.0",
     "blind-peer-muxer": "^1.0.0",
     "compact-encoding": "^2.16.0",
     "corestore": "^7.4.4",
     "hypercore": "^11.26.0",
     "hypercore-crypto": "^3.5.0",
     "hypercore-id-encoding": "^1.3.0",
-    "hyperdb": "^5.0.0",
+    "hyperdb": "^6.6.3",
     "hyperdht": "^6.29.0",
     "hyperschema": "^1.10.3",
     "hyperswarm": "^4.13.1",
+    "ip-ban-list": "^0.3.0",
     "protomux-rpc": "^1.7.1",
     "protomux-rpc-client": "^2.1.0",
     "protomux-rpc-client-pool": "^2.1.0",
     "protomux-wakeup": "^2.9.0",
     "ready-resource": "^1.1.2",
     "repl-swarm": "^2.3.0",
+    "resolve-reject-promise": "^1.0.3",
     "rocksdb-native": "^3.1.6",
     "safety-catch": "^1.0.2",
     "scope-lock": "^1.2.4"
@@ -32,9 +34,11 @@
     "bare-process": "^4.2.2",
     "bare-prom-client": "^15.1.3",
     "blind-peer-router": "^0.2.2",
-    "blind-peering": "^2.0.1",
+    "blind-peering": "^2.1.0",
     "brittle": "^3.7.0",
     "debounceify": "^1.1.0",
+    "graceful-goodbye": "^1.3.3",
+    "hyperdht-address": "^1.0.1",
     "lunte": "^1.6.0",
     "prettier": "^3.6.2",
     "prettier-config-holepunch": "^2.0.0",
@@ -49,6 +53,7 @@
     "format": "prettier --write .",
     "format:check": "prettier --check .",
     "test": "npm run format:check && lunte &&  brittle test/*.js",
+    "test:netns": "brittle test-netns/*.test.js",
     "test:bare": "bare test/test.js"
   },
   "repository": {