blind-peer 3.0.1 → 3.2.0

package/index.js

@@ -5,6 +5,7 @@ const ReadyResource = require('ready-resource')
 const Hyperswarm = require('hyperswarm')
 const ProtomuxRPC = require('protomux-rpc')
 const c = require('compact-encoding')
+const BlindPeerMuxer = require('blind-peer-muxer')
 const b4a = require('b4a')
 const crypto = require('hypercore-crypto')
 const safetyCatch = require('safety-catch')
@@ -14,8 +15,7 @@ const IdEnc = require('hypercore-id-encoding')
 
 const BlindPeerDB = require('./lib/db.js')
 
-const { AddCoreEncoding } = require('blind-peer-encodings')
-const { DeleteCoreEncoding } = require('blind-peer-encodings')
+const { AddCoreEncoding, DeleteCoreEncoding } = require('blind-peer-encodings')
 
 class CoreTracker {
   constructor(blindPeer, core) {
@@ -196,7 +196,8 @@ class BlindPeer extends ReadyResource {
       trustedPubKeys,
       port,
       announcingInterval = 100,
-      wakeupGcTickTime = null
+      wakeupGcTickTime = null,
+      replicationLagThreshold = 100
     } = {}
   ) {
     super()
@@ -221,12 +222,16 @@ class BlindPeer extends ReadyResource {
     this.enableGc = enableGc
     this.lock = new ScopeLock({ debounce: true })
     this.announcedCores = new Map()
+    this.replicationLagThreshold = replicationLagThreshold
 
     this.stats = {
       bytesGcd: 0,
       coresAdded: 0,
       activations: 0,
-      wakeups: 0
+      wakeups: 0,
+      addCoresRx: 0,
+      muxerPaired: 0,
+      muxerErrors: 0
     }
   }
 
@@ -424,6 +429,23 @@ class BlindPeer extends ReadyResource {
 
     rpc.respond('add-core', AddCoreEncoding, this._onaddcore.bind(this, conn))
     rpc.respond('delete-core', DeleteCoreEncoding, this._ondeletecore.bind(this, conn))
+
+    const self = this
+    BlindPeerMuxer.pair(conn, function () {
+      self.emit('muxer-paired', conn)
+      self.stats.muxerPaired++
+      new BlindPeerMuxer(conn, {
+        async oncores(request) {
+          try {
+            await self._onaddcores(conn, request)
+          } catch (e) {
+            self.stats.muxerErrors++
+            self.emit('muxer-error', e, conn)
+            throw e
+          }
+        }
+      })
+    })
   }
 
   async _activateCore(stream, record) {
@@ -467,17 +489,38 @@ class BlindPeer extends ReadyResource {
     const core = this.store.get({ key })
     this.announcedCores.set(coreId, core)
 
+    let activeSession = null
+
     core.on('append', () => {
+      const replicationLag = core.length - core.contiguousLength
+      if (!activeSession.isClient && replicationLag > this.replicationLagThreshold) {
+        activeSession.refresh({ server: true, client: true })
+        this.emit('core-client-mode-changed', core, true)
+      }
+
       this.emit('core-append', core)
     })
     core.on('download', () => {
-      if (core.length === core.contiguousLength) {
+      const replicationLag = core.length - core.contiguousLength
+      if (replicationLag === 0) {
+        if (activeSession.isClient) {
+          activeSession.refresh({ server: true, client: false })
+          this.emit('core-client-mode-changed', core, false)
+        }
+
         this.emit('core-downloaded', core)
       }
     })
 
     await core.ready()
-    this.swarm.join(core.discoveryKey, { server: true, client: false })
+
+    const replicationLag = core.length - core.contiguousLength
+    if (replicationLag > this.replicationLagThreshold || core.length === 0) {
+      activeSession = this.swarm.join(core.discoveryKey, { server: true, client: true })
+      this.emit('core-client-mode-changed', core, true)
+    } else {
+      activeSession = this.swarm.join(core.discoveryKey, { server: true, client: false })
+    }
 
     // WARNING: we do not yet handle the case where
     // data of an announced core is cleared
@@ -527,6 +570,100 @@ class BlindPeer extends ReadyResource {
     return coreRecord
   }
 
+  async _onaddcores(stream, request) {
+    this.stats.addCoresRx++
+    const priority = Math.min(request.priority, 1) // 2 is reserved for trusted peers
+    const { cores, referrer } = request
+
+    if (request.announce !== false && !this._isTrustedPeer(stream.remotePublicKey)) {
+      // Note: we can't use the original downgrade-announce event because that assumes a 'record' object
+      this.emit('add-cores-downgrade-announce', {
+        request,
+        remotePublicKey: stream.remotePublicKey
+      })
+      request.announce = false
+    }
+
+    this.emit('add-cores-received', stream, request)
+
+    const discKeys = []
+    const overview = new Map()
+    for (const c of cores) {
+      const discoveryKey = crypto.discoveryKey(c.key)
+      discKeys.push(discoveryKey)
+      const id = IdEnc.normalize(discoveryKey)
+      overview.set(id, {
+        key: c.key,
+        discoveryKey,
+        remoteLength: c.length,
+        announce: request.announce,
+        priority,
+        referrer,
+        ownLength: 0, // set later
+        ownContigLength: 0, // set later
+        needsActivation: false // changed later if needed
+      })
+    }
+
+    const recordsToAdd = []
+    const infos = await this.store.storage.getInfos(discKeys)
+    for (let i = 0; i < infos.length; i++) {
+      const id = IdEnc.normalize(discKeys[i])
+      const storageInfo = infos[i]
+      const entry = overview.get(id)
+
+      // Note: just the null check does not suffice for storageInfo, because we already try
+      // loading some keys from other contexts, like when the system core of an autobase is
+      // used as a referrer.
+      if (
+        storageInfo === null ||
+        entry.announce ||
+        (storageInfo.head === null && entry.remoteLength > 0)
+      ) {
+        entry.needsActivation = true
+        recordsToAdd.push({
+          key: entry.key,
+          priority: entry.priority,
+          announce: entry.announce,
+          referrer: entry.referrer
+        })
+      } else {
+        entry.ownLength = storageInfo.head?.length || 0
+        entry.ownContigLength = storageInfo.hints?.contiguousLength || 0
+        if (entry.ownLength !== entry.remoteLength) entry.needsActivation = true
+        if (entry.ownLength > entry.ownContigLength) entry.needsActivation = true
+      }
+    }
+
+    // Note: not race condition safe when called with the same cores at a similar time,
+    // but it's no problem if we do add the same core twice
+    for (const record of recordsToAdd) this.db.addCore(record)
+    if (recordsToAdd.length > 0) await this.flush() // flush now as important data
+
+    if (referrer) {
+      const muxer = stream.userData
+      const core = this.store.get({ key: referrer })
+      await core.ready()
+      const discoveryKey = core.discoveryKey
+      await core.close()
+      await this._onwakeup(discoveryKey, muxer)
+    }
+
+    for (const r of recordsToAdd) this.emit('add-new-core', r, true, stream)
+
+    const activateProms = []
+    for (const entry of overview.values()) {
+      if (!entry.needsActivation) continue
+      this.stats.coresAdded++
+      this.emit('add-core', entry, true, stream)
+      activateProms.push(this._activateCore(stream, entry))
+    }
+    await Promise.all(activateProms)
+
+    this.emit('add-cores-done', stream, request)
+    return null
+  }
+
   async _ondeletecore(stream, { key }) {
     if (!this._isTrustedPeer(stream.remotePublicKey)) {
       this.emit('delete-blocked', stream, { key })
@@ -657,6 +794,31 @@ class BlindPeer extends ReadyResource {
         this.set(self.nrAnnouncedCores)
       }
     })
+
+    new promClient.Gauge({
+      // eslint-disable-line no-new
+      name: 'blind_peer_muxer_errors',
+      help: 'The amount of errors on the protomux muxer',
+      collect() {
+        this.set(self.stats.muxerErrors)
+      }
+    })
+    new promClient.Gauge({
+      // eslint-disable-line no-new
+      name: 'blind_peer_muxer_paired',
+      help: 'The amount of blind-peer-muxer sessions paired',
+      collect() {
+        this.set(self.stats.muxerPaired)
+      }
+    })
+    new promClient.Gauge({
+      // eslint-disable-line no-new
+      name: 'blind_peer_add_cores_rx',
+      help: 'The amount of add-cores requests received',
+      collect() {
+        this.set(self.stats.addCoresRx)
+      }
+    })
   }
 }
 

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "blind-peer",
-  "version": "3.0.1",
+  "version": "3.2.0",
   "description": "Blind peers help keep hypercores available",
   "main": "index.js",
   "dependencies": {
     "autobase": "^7.0.18",
     "b4a": "^1.6.7",
-    "blind-peer-encodings": "^3.1.0",
+    "blind-peer-encodings": "^3.1.1",
+    "blind-peer-muxer": "^1.0.0",
     "compact-encoding": "^2.16.0",
     "corestore": "^7.4.4",
     "hypercore": "^11.0.12",
@@ -26,13 +27,13 @@
   "devDependencies": {
     "bare-events": "^2.8.2",
     "bare-process": "^4.2.2",
-    "blind-peering": "^1.13.0",
+    "bare-prom-client": "^15.1.3",
+    "blind-peering": "^2.0.0",
     "brittle": "^3.7.0",
     "debounceify": "^1.1.0",
     "hyperdht": "^6.20.1",
     "prettier": "^3.6.2",
     "prettier-config-holepunch": "^2.0.0",
-    "bare-prom-client": "^15.1.3",
     "test-tmp": "^1.3.0",
     "which-runtime": "^1.3.2"
   },