blind-peer 3.1.0 → 3.4.0

package/index.js

@@ -5,6 +5,7 @@ const ReadyResource = require('ready-resource')
 const Hyperswarm = require('hyperswarm')
 const ProtomuxRPC = require('protomux-rpc')
 const c = require('compact-encoding')
+const BlindPeerMuxer = require('blind-peer-muxer')
 const b4a = require('b4a')
 const crypto = require('hypercore-crypto')
 const safetyCatch = require('safety-catch')
@@ -14,8 +15,10 @@ const IdEnc = require('hypercore-id-encoding')
 
 const BlindPeerDB = require('./lib/db.js')
 
-const { AddCoreEncoding } = require('blind-peer-encodings')
-const { DeleteCoreEncoding } = require('blind-peer-encodings')
+// Enable Small wants in Hypercore. Must be before anywhere that uses Hypercore
+Hypercore.enable(Hypercore.SMALL_WANTS)
+
+const { AddCoreEncoding, DeleteCoreEncoding } = require('blind-peer-encodings')
 
 class CoreTracker {
   constructor(blindPeer, core) {
@@ -228,7 +231,10 @@ class BlindPeer extends ReadyResource {
       bytesGcd: 0,
       coresAdded: 0,
       activations: 0,
-      wakeups: 0
+      wakeups: 0,
+      addCoresRx: 0,
+      muxerPaired: 0,
+      muxerErrors: 0
     }
   }
 
@@ -426,6 +432,23 @@ class BlindPeer extends ReadyResource {
 
     rpc.respond('add-core', AddCoreEncoding, this._onaddcore.bind(this, conn))
     rpc.respond('delete-core', DeleteCoreEncoding, this._ondeletecore.bind(this, conn))
+
+    const self = this
+    BlindPeerMuxer.pair(conn, function () {
+      self.emit('muxer-paired', conn)
+      self.stats.muxerPaired++
+      new BlindPeerMuxer(conn, {
+        async oncores(request) {
+          try {
+            await self._onaddcores(conn, request)
+          } catch (e) {
+            self.stats.muxerErrors++
+            self.emit('muxer-error', e, conn)
+            throw e
+          }
+        }
+      })
+    })
   }
 
   async _activateCore(stream, record) {
@@ -497,6 +520,7 @@ class BlindPeer extends ReadyResource {
     const replicationLag = core.length - core.contiguousLength
     if (replicationLag > this.replicationLagThreshold || core.length === 0) {
       activeSession = this.swarm.join(core.discoveryKey, { server: true, client: true })
+      this.emit('core-client-mode-changed', core, true)
     } else {
       activeSession = this.swarm.join(core.discoveryKey, { server: true, client: false })
     }
@@ -549,6 +573,100 @@ class BlindPeer extends ReadyResource {
     return coreRecord
   }
 
+  async _onaddcores(stream, request) {
+    this.stats.addCoresRx++
+    const priority = Math.min(request.priority, 1) // 2 is reserved for trusted peers
+    const { cores, referrer } = request
+
+    if (request.announce !== false && !this._isTrustedPeer(stream.remotePublicKey)) {
+      // Note: we can't use the original downgrade-announce event because that assumes a 'record' object
+      this.emit('add-cores-downgrade-announce', {
+        request,
+        remotePublicKey: stream.remotePublicKey
+      })
+      request.announce = false
+    }
+
+    this.emit('add-cores-received', stream, request)
+
+    const discKeys = []
+    const overview = new Map()
+    for (const c of cores) {
+      const discoveryKey = crypto.discoveryKey(c.key)
+      discKeys.push(discoveryKey)
+      const id = IdEnc.normalize(discoveryKey)
+      overview.set(id, {
+        key: c.key,
+        discoveryKey,
+        remoteLength: c.length,
+        announce: request.announce,
+        priority,
+        referrer,
+        ownLength: 0, // set later
+        ownContigLength: 0, // set later
+        needsActivation: false // changed later if needed
+      })
+    }
+
+    const recordsToAdd = []
+    const infos = await this.store.storage.getInfos(discKeys)
+    for (let i = 0; i < infos.length; i++) {
+      const id = IdEnc.normalize(discKeys[i])
+      const storageInfo = infos[i]
+      const entry = overview.get(id)
+
+      // Note: just the null check does not suffice for storageInfo, because we already try
+      // loading some keys from other contexts, like when the system core of an autobase is
+      // used as a referrer.
+      if (
+        storageInfo === null ||
+        entry.announce ||
+        (storageInfo.head === null && entry.remoteLength > 0)
+      ) {
+        entry.needsActivation = true
+        recordsToAdd.push({
+          key: entry.key,
+          priority: entry.priority,
+          announce: entry.announce,
+          referrer: entry.referrer
+        })
+      } else {
+        entry.ownLength = storageInfo.head?.length || 0
+        entry.ownContigLength = storageInfo.hints?.contiguousLength || 0
+        if (entry.ownLength !== entry.remoteLength) entry.needsActivation = true
+        if (entry.ownLength > entry.ownContigLength) entry.needsActivation = true
+      }
+    }
+
+    // Note: not race condition safe when called with the same cores at a similar time,
+    // but it's no problem if we do add the same core twice
+    for (const record of recordsToAdd) this.db.addCore(record)
+    if (recordsToAdd.length > 0) await this.flush() // flush now as important data
+
+    if (referrer) {
+      const muxer = stream.userData
+      const core = this.store.get({ key: referrer })
+      await core.ready()
+      const discoveryKey = core.discoveryKey
+      await core.close()
+      await this._onwakeup(discoveryKey, muxer)
+    }
+
+    for (const r of recordsToAdd) this.emit('add-new-core', r, true, stream)
+
+    const activateProms = []
+    for (const entry of overview.values()) {
+      if (!entry.needsActivation) continue
+      this.stats.coresAdded++
+      this.emit('add-core', entry, true, stream)
+      activateProms.push(this._activateCore(stream, entry))
+    }
+    await Promise.all(activateProms)
+
+    this.emit('add-cores-done', stream, request)
+    return null
+  }
+
   async _ondeletecore(stream, { key }) {
     if (!this._isTrustedPeer(stream.remotePublicKey)) {
       this.emit('delete-blocked', stream, { key })
@@ -679,6 +797,81 @@ class BlindPeer extends ReadyResource {
         this.set(self.nrAnnouncedCores)
       }
     })
+
+    new promClient.Gauge({
+      // eslint-disable-line no-new
+      name: 'blind_peer_muxer_errors',
+      help: 'The amount of errors on the protomux muxer',
+      collect() {
+        this.set(self.stats.muxerErrors)
+      }
+    })
+    new promClient.Gauge({
+      // eslint-disable-line no-new
+      name: 'blind_peer_muxer_paired',
+      help: 'The amount of blind-peer-muxer sessions paired',
+      collect() {
+        this.set(self.stats.muxerPaired)
+      }
+    })
+    new promClient.Gauge({
+      // eslint-disable-line no-new
+      name: 'blind_peer_add_cores_rx',
+      help: 'The amount of add-cores requests received',
+      collect() {
+        this.set(self.stats.addCoresRx)
+      }
+    })
+    if (self.rocks.stats) {
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_gets',
+        help: 'The amount of get ops from RocksDB',
+        collect() {
+          this.set(self.rocks.stats.gets)
+        }
+      })
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_puts',
+        help: 'The amount of put ops to RocksDB',
+        collect() {
+          this.set(self.rocks.stats.puts)
+        }
+      })
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_deletes',
+        help: 'The amount of delete ops from RocksDB',
+        collect() {
+          this.set(self.rocks.stats.deletes)
+        }
+      })
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_range_deletes',
+        help: 'The amount of range delete ops from RocksDB',
+        collect() {
+          this.set(self.rocks.stats.rangeDeletes)
+        }
+      })
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_read_batches',
+        help: 'The amount of read batches from RocksDB',
+        collect() {
+          this.set(self.rocks.stats.readBatches)
+        }
+      })
+      new promClient.Gauge({
+        // eslint-disable-line no-new
+        name: 'blind_peer_rocks_write_batches',
+        help: 'The amount of write batches to RocksDB',
+        collect() {
+          this.set(self.rocks.stats.writeBatches)
+        }
+      })
+    }
   }
 }
 

package/package.json

@@ -1,15 +1,16 @@
 {
   "name": "blind-peer",
-  "version": "3.1.0",
+  "version": "3.4.0",
   "description": "Blind peers help keep hypercores available",
   "main": "index.js",
   "dependencies": {
     "autobase": "^7.0.18",
     "b4a": "^1.6.7",
-    "blind-peer-encodings": "^3.1.0",
+    "blind-peer-encodings": "^3.1.1",
+    "blind-peer-muxer": "^1.0.0",
     "compact-encoding": "^2.16.0",
     "corestore": "^7.4.4",
-    "hypercore": "^11.0.12",
+    "hypercore": "^11.26.0",
     "hypercore-crypto": "^3.5.0",
     "hypercore-id-encoding": "^1.3.0",
     "hyperdb": "^5.0.0",
@@ -26,13 +27,13 @@
   "devDependencies": {
     "bare-events": "^2.8.2",
     "bare-process": "^4.2.2",
-    "blind-peering": "^1.13.0",
+    "bare-prom-client": "^15.1.3",
+    "blind-peering": "^2.0.0",
     "brittle": "^3.7.0",
     "debounceify": "^1.1.0",
     "hyperdht": "^6.20.1",
     "prettier": "^3.6.2",
     "prettier-config-holepunch": "^2.0.0",
-    "bare-prom-client": "^15.1.3",
     "test-tmp": "^1.3.0",
     "which-runtime": "^1.3.2"
   },