blind-peer 0.0.3 → 2.7.4

package/lib/db.js

@@ -0,0 +1,164 @@
+const HyperDB = require('hyperdb')
+const crypto = require('hypercore-crypto')
+const ReadyResource = require('ready-resource')
+const { definition: spec } = require('blind-peer-encodings')
+
+// LOW, NORMAL, HIGH
+const MAX_PRIO = 2
+
+module.exports = class BlindPeerDB extends ReadyResource {
+  constructor (db, auth) {
+    super()
+
+    this.db = HyperDB.rocks(db, spec)
+    this.digest = null
+    this.auth = auth || null
+
+    this.encryptionKeyPair = null
+    this.swarmingKeyPair = null
+
+    this.coresAdding = []
+    this.coresUpdated = new Map()
+
+    this.ready().catch(noop)
+  }
+
+  find (col, q) {
+    return this.db.find(col, q)
+  }
+
+  get (col, q) {
+    return this.db.get(col, q)
+  }
+
+  getCoreRecord (key) {
+    return this.get('@blind-peer/cores', { key })
+  }
+
+  async _open () {
+    await this.db.ready()
+
+    const auth = await this.db.get('@blind-peer/auth')
+
+    this.auth = auth || this.auth || { swarming: null, encryption: null }
+    if (!this.auth.encryption) this.auth.encryption = crypto.randomBytes(32)
+    if (!this.auth.swarming) this.auth.swarming = crypto.randomBytes(32)
+
+    this.encryptionKeyPair = crypto.encryptionKeyPair(this.auth.encryption)
+    this.swarmingKeyPair = crypto.keyPair(this.auth.swarming)
+
+    const digest = await this.db.get('@blind-peer/digest')
+
+    this.digest = digest || { referrers: 0, cores: 0, bytesAllocated: 0, flushed: false }
+    this.digest.flushed = false
+
+    await this.db.insert('@blind-peer/auth', this.auth)
+    await this.db.insert('@blind-peer/digest', this.digest)
+    await this.db.flush()
+  }
+
+  async flush () { // The caller is responsible for ensuring this runs in a lock
+    if (!this.opened) await this.ready()
+
+    const coresAdding = this.coresAdding
+    const coresUpdated = this.coresUpdated
+
+    this.coresAdding = []
+    this.coresUpdated = new Map()
+
+    const tx = this.db.transaction()
+    const time = Date.now()
+
+    let addedCores = 0
+    let addedReferrers = 0
+    let bytesAllocated = 0
+
+    for (const info of coresAdding) {
+      const key = info.key
+      const existing = await tx.get('@blind-peer/cores', { key })
+      // TODO: test this path
+      if (existing) {
+        if (!info.announce && (info.priority || 0) <= existing.priority) continue // would be a downgrade, which we never want
+
+        existing.priority = Math.min(
+          Math.max(info.priority || 0, existing.priority)
+        )
+        existing.announce = info.announce || existing.announce
+        existing.updated = time
+        existing.active = time
+        await tx.insert('@blind-peer/cores', existing)
+      } else {
+        await tx.insert('@blind-peer/cores', {
+          key,
+          length: 0,
+          bytesAllocated: 0,
+          updated: time,
+          active: time,
+          priority: Math.min(MAX_PRIO, info.priority || 0),
+          announce: !!info.announce,
+          referrer: info.referrer || null,
+          blocksCleared: 0,
+          bytesCleared: 0
+        })
+
+        addedCores++
+        if (info.referrer) addedReferrers++
+      }
+    }
+
+    for (const core of coresUpdated.values()) {
+      const c = await tx.get('@blind-peer/cores', { key: core.key })
+      if (!c) continue // state mismatch
+
+      const updated = c.length !== core.length
+
+      bytesAllocated += (core.bytesAllocated - c.bytesAllocated)
+
+      c.length = core.length
+      c.bytesAllocated = core.bytesAllocated
+      c.bytesCleared = core.bytesCleared || 0
+      if (updated) c.updated = time
+      c.active = time
+      c.blocksCleared = core.blocksCleared || 0
+
+      await tx.insert('@blind-peer/cores', c)
+    }
+
+    if (addedCores || addedReferrers || bytesAllocated || this.closing) {
+      this.digest.cores += addedCores
+      this.digest.referrers += addedReferrers
+      this.digest.bytesAllocated += bytesAllocated
+      this.digest.flushed = !!this.closing
+
+      await tx.insert('@blind-peer/digest', this.digest)
+    }
+
+    await tx.flush()
+  }
+
+  addCore (info) {
+    this.coresAdding.push(info)
+  }
+
+  updateCore (core, id) { // TODO: id is technically optional
+    this.coresUpdated.set(id, core)
+  }
+
+  updated () {
+    return this.coresAdding.length > 0 || this.coresUpdated.size > 0
+  }
+
+  createGcCandidateReadStream () {
+    return this.db.find('@blind-peer/cores-by-activity')
+  }
+
+  createAnnouncingCoresStream () {
+    return this.db.find('@blind-peer/cores-by-announce')
+  }
+
+  async _close () {
+    await this.db.close()
+  }
+}
+
+function noop () {}

package/package.json

@@ -1,37 +1,62 @@
 {
   "name": "blind-peer",
-  "version": "0.0.3",
-  "description": "WIP - nothing to see here",
+  "version": "2.7.4",
+  "description": "Blind peers help keep hypercores available",
   "main": "index.js",
   "bin": {
     "blind-peer": "bin.js"
   },
   "dependencies": {
-    "autobase-light-writer": "^1.1.0",
-    "corestore": "^6.18.4",
-    "graceful-goodbye": "^1.3.2",
+    "autobase": "^7.0.18",
+    "autobase-discovery": "^1.0.0",
+    "b4a": "^1.6.7",
+    "blind-peer-encodings": "^3.0.0",
+    "compact-encoding": "^2.16.0",
+    "corestore": "^7.4.4",
+    "graceful-goodbye": "^1.3.3",
+    "hyper-instrument": "^2.0.0",
+    "hypercore": "^11.0.12",
+    "hypercore-crypto": "^3.5.0",
     "hypercore-id-encoding": "^1.3.0",
-    "hyperdb": "^4.1.2",
-    "hyperschema": "^1.0.3",
+    "hyperdb": "^4.9.4",
+    "hyperschema": "^1.10.3",
     "hyperswarm": "^4.8.4",
-    "paparam": "^1.6.1",
-    "protomux-rpc": "^1.6.0"
+    "paparam": "^1.8.0",
+    "pino": "^9.6.0",
+    "protomux-rpc": "^1.7.1",
+    "protomux-wakeup": "^2.2.0",
+    "ready-resource": "^1.1.2",
+    "repl-swarm": "^2.2.0",
+    "rocksdb-native": "^3.1.6",
+    "safety-catch": "^1.0.2",
+    "scope-lock": "^1.2.4",
+    "tiny-byte-size": "^1.1.0"
   },
   "devDependencies": {
+    "blind-peering": "^1.10.1",
+    "brittle": "^3.7.0",
     "debounceify": "^1.1.0",
-    "standard": "^17.1.2"
+    "hyperdht": "^6.20.1",
+    "prom-client": "^15.1.3",
+    "standard": "^17.1.2",
+    "test-tmp": "^1.3.0"
   },
+  "files": [
+    "bin.js",
+    "index.js",
+    "lib/"
+  ],
   "scripts": {
-    "test": "standard"
+    "test": "standard && brittle test/*.js"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/mafintosh/blind-peer.git"
+    "url": "git+https://github.com/holepunchto/blind-peer.git"
   },
   "author": "Holepunch Inc.",
   "license": "Apache-2.0",
   "bugs": {
-    "url": "https://github.com/mafintosh/blind-peer/issues"
+    "url": "https://github.com/holepunchto/blind-peer/issues"
   },
-  "homepage": "https://github.com/mafintosh/blind-peer"
+  "homepage": "https://github.com/holepunchto/blind-peer"
 }

package/.github/workflows/test-node.yml

@@ -1,23 +0,0 @@
-name: Build Status
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-jobs:
-  build:
-    strategy:
-      matrix:
-        node-version: [lts/*]
-        os: [ubuntu-latest, macos-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v3
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
-      with:
-        node-version: ${{ matrix.node-version }}
-    - run: npm install
-    - run: npm test

package/build.js

@@ -1,102 +0,0 @@
-const HyperDB = require('hyperdb/builder')
-const Hyperschema = require('hyperschema')
-
-const SCHEMA_DIR = './spec/hyperschema'
-const DB_DIR = './spec/hyperdb'
-
-const schema = Hyperschema.from(SCHEMA_DIR)
-const blind = schema.namespace('blind-peer')
-
-blind.register({
-  name: 'request-mailbox',
-  fields: [
-    {
-      name: 'autobase',
-      type: 'fixed32',
-      required: true
-    },
-    {
-      name: 'blockEncryptionKey',
-      type: 'fixed32'
-    }
-  ]
-})
-
-blind.register({
-  name: 'response-mailbox',
-  fields: [
-    {
-      name: 'autobase',
-      type: 'fixed32',
-      required: true
-    },
-    {
-      name: 'writer',
-      type: 'fixed32',
-      required: true
-    },
-    {
-      name: 'open',
-      type: 'bool'
-    }
-  ]
-})
-
-blind.register({
-  name: 'request-post',
-  fields: [
-    {
-      name: 'autobase',
-      type: 'fixed32',
-      required: true
-    },
-    {
-      name: 'message',
-      type: 'buffer'
-    }
-  ]
-})
-
-blind.register({
-  name: 'response-post',
-  fields: [
-    {
-      name: 'length',
-      type: 'uint',
-      required: true
-    }
-  ]
-})
-
-blind.register({
-  name: 'mailbox',
-  fields: [
-    {
-      name: 'autobase',
-      type: 'fixed32',
-      required: true
-    },
-    {
-      name: 'writer',
-      type: 'fixed32',
-      required: true
-    },
-    {
-      name: 'blockEncryptionKey',
-      type: 'fixed32'
-    }
-  ]
-})
-
-Hyperschema.toDisk(schema)
-
-const db = HyperDB.from(SCHEMA_DIR, DB_DIR, { offset: 64 })
-const blindDB = db.namespace('blind-peer')
-
-blindDB.collections.register({
-  name: 'mailbox',
-  schema: '@blind-peer/mailbox',
-  key: ['autobase']
-})
-
-HyperDB.toDisk(db)

package/client.js

@@ -1,31 +0,0 @@
-const schema = require('./spec/hyperschema')
-const c = require('compact-encoding')
-const ProtomuxRPC = require('protomux-rpc')
-
-const addMailboxEncoding = {
-  requestEncoding: schema.resolveStruct('@blind-peer/request-mailbox'),
-  responseEncoding: schema.resolveStruct('@blind-peer/response-mailbox')
-}
-
-const postEncoding = {
-  requestEncoding: schema.resolveStruct('@blind-peer/request-post'),
-  responseEncoding: schema.resolveStruct('@blind-peer/response-post')
-}
-
-module.exports = class BlindPeerClient {
-  constructor (stream) {
-    this.stream = stream
-    this.rpc = new ProtomuxRPC(stream, {
-      id: stream.remotePublicKey,
-      valueEncoding: c.none
-    })
-  }
-
-  addMailbox (data) {
-    return this.rpc.request('add-mailbox', data, addMailboxEncoding)
-  }
-
-  post ({ autobase, message }) {
-    return this.rpc.request('post', { autobase, message }, postEncoding)
-  }
-}

package/example/autobase.mjs

@@ -1,66 +0,0 @@
-import BlindPeerClient from '../client.js'
-import Autobase from 'autobase'
-import c from 'compact-encoding'
-import Corestore from 'corestore'
-import Hyperswarm from 'hyperswarm'
-import debounce from 'debounceify'
-import IdEnc from 'hypercore-id-encoding'
-
-const base = new Autobase(new Corestore('/tmp/my-corestore'), {
-  encryptionKey: Buffer.alloc(30).fill('secret'),
-  open (store) {
-    return store.get('view', { valueEncoding: c.json })
-  },
-  async apply (nodes, view, base) {
-    for (const node of nodes) {
-      const jsonValue = JSON.parse(node.value.toString())
-      if (jsonValue.add) await base.addWriter(Buffer.from(jsonValue.key, 'hex'), { indexer: false })
-      view.append(jsonValue)
-    }
-  }
-})
-
-await base.ready()
-console.log('Autobase:', base.key.toString('hex'))
-
-base.view.on('append', debounce(async function () {
-  base.ack() // ack for good messure
-  console.log('someone appended to the autobase!')
-  for (let i = 0; i < base.view.length; i++) {
-    console.log(i, await base.view.get(i))
-  }
-}))
-
-// TODO: record in autobase
-const publicKey = IdEnc.decode(process.argv[2])
-
-const s = new Hyperswarm({ keyPair: await base.store.createKeyPair('tmp') })
-
-s.on('connection', async c => {
-  base.store.replicate(c)
-
-  if (!c.remotePublicKey.equals(publicKey)) return
-
-  const peer = new BlindPeerClient(c)
-
-  const info = await peer.addMailbox({ autobase: base.key })
-
-  if (info.open === false) {
-    const message = Buffer.from(
-      JSON.stringify({ add: true, key: info.writer.toString('hex') })
-    )
-    await base.append(message)
-    await base.update()
-
-    const core = base.store.get({ key: info.writer, active: false })
-    await core.setEncryptionKey(base.encryptionKey)
-    const req = { autobase: base.key, blockEncryptionKey: core.encryption.blockKey }
-    await core.close()
-
-    await peer.addMailbox(req)
-
-    console.log('opened mailbox...')
-  }
-})
-
-s.joinPeer(publicKey)

package/example/blind-peer.mjs

@@ -1,6 +0,0 @@
-import BlindPeer from '../index.js'
-
-const m = new BlindPeer('/tmp/blind-peer')
-await m.listen()
-
-console.log(m.publicKey.toString('hex'))

package/example/post.mjs

@@ -1,25 +0,0 @@
-import BlindPeerClient from '../client.js'
-import Hyperswarm from 'hyperswarm'
-import IdEnc from 'hypercore-id-encoding'
-
-const publicKey = IdEnc.decode(process.argv[2])
-const autobase = IdEnc.decode(process.argv[3])
-const rawMessage = process.argv[4]
-const message = Buffer.from(
-  JSON.stringify({ mailbox: true, message: rawMessage })
-)
-
-const s = new Hyperswarm()
-
-s.on('connection', async c => {
-  if (!c.remotePublicKey.equals(publicKey)) return
-
-  const peer = new BlindPeerClient(c)
-  const reply = await peer.post({ autobase, message })
-
-  console.log(reply)
-
-  s.destroy()
-})
-
-s.joinPeer(publicKey)

package/spec/hyperdb/db.json

@@ -1,19 +0,0 @@
-{
-  "version": 0,
-  "offset": 64,
-  "schema": [
-    {
-      "name": "mailbox",
-      "namespace": "blind-peer",
-      "id": 64,
-      "type": 1,
-      "indexes": [],
-      "schema": "@blind-peer/mailbox",
-      "derived": false,
-      "key": [
-        "autobase"
-      ],
-      "trigger": null
-    }
-  ]
-}

package/spec/hyperdb/index.js

@@ -1,83 +0,0 @@
-// This file is autogenerated by the hyperdb compiler
-/* eslint-disable camelcase */
-
-const { IndexEncoder, c } = require('hyperdb/runtime')
-
-const { version, resolveStruct } = require('./messages.js')
-
-// '@blind-peer/mailbox' collection key
-const collection64_key = new IndexEncoder([
-  IndexEncoder.BUFFER
-], { prefix: 64 })
-
-function collection64_indexify (record) {
-  const a = record.autobase
-  return a === undefined ? [] : [a]
-}
-
-// '@blind-peer/mailbox' reconstruction function
-function collection64_reconstruct (version, keyBuf, valueBuf) {
-  const key = collection64_key.decode(keyBuf)
-  const value = c.decode(resolveStruct('@blind-peer/mailbox/value', version), valueBuf)
-  // TODO: This should be fully code generated
-  return {
-    autobase: key[0],
-    ...value
-  }
-}
-// '@blind-peer/mailbox' key reconstruction function
-function collection64_reconstruct_key (keyBuf) {
-  const key = collection64_key.decode(keyBuf)
-  return {
-    autobase: key[0]
-  }
-}
-
-// '@blind-peer/mailbox'
-const collection64 = {
-  name: '@blind-peer/mailbox',
-  id: 64,
-  encodeKey (record) {
-    const key = [record.autobase]
-    return collection64_key.encode(key)
-  },
-  encodeKeyRange ({ gt, lt, gte, lte } = {}) {
-    return collection64_key.encodeRange({
-      gt: gt ? collection64_indexify(gt) : null,
-      lt: lt ? collection64_indexify(lt) : null,
-      gte: gte ? collection64_indexify(gte) : null,
-      lte: lte ? collection64_indexify(lte) : null
-    })
-  },
-  encodeValue (version, record) {
-    return c.encode(resolveStruct('@blind-peer/mailbox/value', version), record)
-  },
-  trigger: null,
-  reconstruct: collection64_reconstruct,
-  reconstructKey: collection64_reconstruct_key,
-  indexes: []
-}
-
-module.exports = {
-  version,
-  collections: [
-    collection64
-  ],
-  indexes: [
-  ],
-  resolveCollection,
-  resolveIndex
-}
-
-function resolveCollection (name) {
-  switch (name) {
-    case '@blind-peer/mailbox': return collection64
-    default: return null
-  }
-}
-
-function resolveIndex (name) {
-  switch (name) {
-    default: return null
-  }
-}