helia-coord 1.1.0

package/lib/adapters/ipfs-adapter.js

@@ -0,0 +1,263 @@
+/*
+  Adapter library for IPFS, so the rest of the business logic doesn't need to
+  know specifics about the IPFS API.
+*/
+
+// The amount of time to wait to connect to a peer, in milliseconds.
+// Increasing the time makes the network slower but more resilient to latency.
+// Decreasing the time makes the network faster, but more smaller and more fragile.
+// import bootstapNodes from '../../config/bootstrap-circuit-relays.js'
+import { multiaddr } from '@multiformats/multiaddr'
+
+const CONNECTION_TIMEOUT = 10000
+
+class IpfsAdapter {
+  constructor (localConfig = {}) {
+    // Input Validation
+    this.ipfs = localConfig.ipfs
+    if (!this.ipfs) {
+      throw new Error(
+        'An instance of IPFS must be passed when instantiating the IPFS adapter library.'
+      )
+    }
+    this.log = localConfig.log
+    if (!this.log) {
+      throw new Error(
+        'A status log handler must be specified when instantiating IPFS adapter library.'
+      )
+    }
+
+    // 'embedded' node type used as default, will use embedded js-ipfs.
+    // Alternative is 'external' which will use ipfs-http-client to control an
+    // external IPFS node.
+    this.nodeType = localConfig.nodeType
+    if (!this.nodeType) {
+      // console.log('No node type specified. Assuming embedded js-ipfs.')
+      this.nodeType = 'embedded'
+    }
+
+    // Port Settings. Defaults are overwritten if specified in the localConfig.
+    this.tcpPort = 4001
+    if (localConfig.tcpPort) this.tcpPort = localConfig.tcpPort
+    this.wsPort = 4003
+    if (localConfig.wsPort) this.wsPort = localConfig.wsPort
+
+    // Placeholders that will be filled in after the node finishes initializing.
+    this.ipfsPeerId = ''
+    this.ipfsMultiaddrs = ''
+
+    // Encapsulate dependencies
+    this.multiaddr = multiaddr
+  }
+
+  // Start the IPFS node if it hasn't already been started.
+  // Update the state of this adapter with the IPFS node information.
+  async start () {
+    try {
+      // Wait until the IPFS creation Promise has resolved, and the node is
+      // fully instantiated.
+      this.ipfs = await this.ipfs
+
+      // Get ID information about this IPFS node.
+      // const id2 = await this.ipfs.id()
+      // this.state.ipfsPeerId = id2.id
+      // this.ipfsPeerId = id2.id
+      this.ipfsPeerId = this.ipfs.libp2p.peerId.toString()
+
+      // Get multiaddrs that can be used to connect to this node.
+      // const addrs = id2.addresses.map(elem => elem.toString())
+      const addrs = this.ipfs.libp2p.getMultiaddrs().map(elem => elem.toString())
+      // this.state.ipfsMultiaddrs = addrs
+      this.ipfsMultiaddrs = addrs
+
+      // Remove bootstrap nodes, as we have our own bootstrap nodes, and the
+      // the default ones can spam our nodes with a ton of bandwidth.
+      // await this.ipfs.config.set('Bootstrap', [
+      //   bootstapNodes.node[0].multiaddr,
+      //   bootstapNodes.node[1].multiaddr
+      // ])
+
+      // Settings specific to embedded js-ipfs.
+      // if (this.nodeType === 'embedded') {
+      //   // Also remove default Delegates, as they are the same as the default
+      //   // Bootstrap nodes.
+      //   await this.ipfs.config.set('Addresses.Delegates', [])
+      // }
+
+      // Settings specific to external go-ipfs node.
+      // if (this.nodeType === 'external') {
+      //   // Enable RelayClient
+      //   // https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmrelayclient
+      //   // https://github.com/ipfs/go-ipfs/releases/tag/v0.11.0
+      //   await this.ipfs.config.set('Swarm.RelayClient.Enabled', true)
+
+      //   // Enable hole punching for better p2p interaction.
+      //   // https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmenableholepunching
+      //   await this.ipfs.config.set('Swarm.EnableHolePunching', true)
+
+      //   // Enable websocket connections
+      //   await this.ipfs.config.set('Addresses.Swarm', [
+      //     `/ip4/0.0.0.0/tcp/${this.tcpPort}`,
+      //     `/ip6/::/tcp/${this.tcpPort}`,
+      //     // `/ip4/0.0.0.0/udp/${this.tcpPort}/quic`,
+      //     // `/ip6/::/udp/${this.tcpPort}/quic`,
+      //     `/ip4/0.0.0.0/tcp/${this.wsPort}`, // Websockets
+      //     `/ip6/::/tcp/${this.wsPort}`
+      //   ])
+
+      //   // Disable scanning of IP ranges. This is largely driven by Hetzner
+      //   await this.ipfs.config.set('Swarm.AddrFilters', [
+      //     '/ip4/10.0.0.0/ipcidr/8',
+      //     '/ip4/100.0.0.0/ipcidr/8',
+      //     '/ip4/169.254.0.0/ipcidr/16',
+      //     '/ip4/172.16.0.0/ipcidr/12',
+      //     '/ip4/192.0.0.0/ipcidr/24',
+      //     '/ip4/192.0.2.0/ipcidr/24',
+      //     '/ip4/192.168.0.0/ipcidr/16',
+      //     '/ip4/198.18.0.0/ipcidr/15',
+      //     '/ip4/198.51.100.0/ipcidr/24',
+      //     '/ip4/203.0.113.0/ipcidr/24',
+      //     '/ip4/240.0.0.0/ipcidr/4',
+      //     '/ip6/100::/ipcidr/64',
+      //     '/ip6/2001:2::/ipcidr/48',
+      //     '/ip6/2001:db8::/ipcidr/32',
+      //     '/ip6/fc00::/ipcidr/7',
+      //     '/ip6/fe80::/ipcidr/10'
+      //   ])
+
+      //   // go-ipfs v0.10.0
+      //   // await this.ipfs.config.set('Swarm.EnableRelayHop', true)
+      //   // await this.ipfs.config.set('Swarm.EnableAutoRelay', true)
+
+      // // Disable peer discovery
+      // // await this.ipfs.config.set('Routing.Type', 'none')
+      // }
+
+      // Disable preloading
+      // await this.ipfs.config.set('preload.enabled', false)
+
+      // Reduce the default number of peers thisNode connects to at one time.
+      // await this.ipfs.config.set('Swarm.ConnMgr', {
+      //   LowWater: 10,
+      //   HighWater: 30,
+      //   GracePeriod: '2s'
+      // })
+
+      // Reduce the storage size, as this node should not be retaining much data.
+      // await this.ipfs.config.set('Datastore.StorageMax', '2GB')
+    } catch (err) {
+      console.error('Error in ipfs-adapter.js/start()')
+      throw err
+    }
+  }
+
+  // Attempts to connect to an IPFS peer, given its IPFS multiaddr.
+  // Returns true if the connection succeeded. Otherwise returns false.
+  async connectToPeer (ipfsAddr) {
+    try {
+      // TODO: Throw error if ipfs ID is passed, instead of a multiaddr.
+      console.log('ipfsAddr: ', ipfsAddr)
+
+      // await this.ipfs.swarm.connect(ipfsAddr, { timeout: CONNECTION_TIMEOUT })
+      await this.ipfs.libp2p.dial(this.multiaddr(ipfsAddr))
+
+      this.log.statusLog(1, `Successfully connected to peer node ${ipfsAddr}`)
+
+      return true
+    } catch (err) {
+      /* exit quietly */
+      console.warn(
+        `Error trying to connect to peer node ${ipfsAddr}: ${err.message}`
+      )
+
+      // if (this.debugLevel === 1) {
+      //   this.statusLog(
+      //     `status: Error trying to connect to peer node ${ipfsAddr}`
+      //   )
+      // } else if (this.debugLevel === 2) {
+      //   this.statusLog(
+      //     `status: Error trying to connect to peer node ${ipfsAddr}: `,
+      //     err
+      //   )
+      // }
+      this.log.statusLog(2, `Error trying to connect to peer node ${ipfsAddr}`)
+      // console.log(`Error trying to connect to peer node ${ipfsAddr}: `, err)
+
+      // this.log.statusLog(
+      //   3,
+      //   `Error trying to connect to peer node ${ipfsAddr}: `,
+      //   err
+      // )
+
+      return false
+    }
+  }
+
+  // Disconnect from a peer.
+  async disconnectFromPeer (ipfsId) {
+    try {
+      // TODO: If given a multiaddr, extract the IPFS ID.
+
+      // Get the list of peers that we're connected to.
+      const connectedPeers = await this.getPeers()
+      // console.log('connectedPeers: ', connectedPeers)
+
+      // See if we're connected to the given IPFS ID
+      const connectedPeer = connectedPeers.filter(x => x.peer === ipfsId)
+
+      // If we're not connected, exit.
+      if (!connectedPeer.length) {
+        // console.log(`debug: Not connected to ${ipfsId}`)
+        return true
+      }
+
+      // If connected, disconnect from the peer.
+      await this.ipfs.swarm.disconnect(connectedPeer[0].addr, {
+        timeout: CONNECTION_TIMEOUT
+      })
+
+      return true
+    } catch (err) {
+      // exit quietly
+      return false
+    }
+  }
+
+  async disconnectFromMultiaddr (multiaddr) {
+    try {
+      await this.ipfs.swarm.disconnect(multiaddr, {
+        timeout: CONNECTION_TIMEOUT
+      })
+
+      return true
+    } catch (err) {
+      return false
+    }
+  }
+
+  // Get a list of all the IPFS peers This Node is connected to.
+  async getPeers () {
+    try {
+      // console.log('this.ipfs.libp2p: ', this.ipfs.libp2p)
+
+      // Get connected peers
+      // const connectedPeers = await this.ipfs.swarm.peers({
+      //   direction: true,
+      //   streams: true,
+      //   verbose: true,
+      //   latency: true
+      // })
+      let connectedPeers = await this.ipfs.libp2p.getPeers()
+      connectedPeers = connectedPeers.map(x => x.toString())
+      console.log('connectedPeers: ', connectedPeers)
+
+      return connectedPeers
+    } catch (err) {
+      console.error('Error in ipfs-adapter.js/getPeers(): ', err)
+      throw err
+    }
+  }
+}
+
+// module.exports = IpfsAdapter
+export default IpfsAdapter

package/lib/adapters/logs-adapter.js

@@ -0,0 +1,44 @@
+/*
+  Controls the verbosity of the status log
+
+  Default verbosity is zero, which is the least verbose messages.
+  As the value increases, the amount of messages also increases.
+
+  0 - Normal logs
+  1 - More information on connections and error connections.
+  2 - Verbose Error messages
+  3 - Error messages about connections.
+*/
+
+// const util = require('util')
+import util from 'util'
+
+class LogsAdapter {
+  constructor (localConfig = {}) {
+    // Default to debugLevel 0 if not specified.
+    this.debugLevel = localConfig.debugLevel
+    if (!this.debugLevel) this.debugLevel = 0
+
+    this.logHandler = localConfig.statusLog
+    if (!this.logHandler) {
+      throw new Error(
+        'statusLog must be specified when instantiating Logs adapter library.'
+      )
+    }
+  }
+
+  // Print out the data, if the log level is less than or equal to the debug
+  // level set when instantiating the library.
+  statusLog (level, str, object) {
+    if (level <= this.debugLevel) {
+      if (object === undefined) {
+        this.logHandler('status: ' + str)
+      } else {
+        this.logHandler('status: ' + str + ' ' + util.inspect(object))
+      }
+    }
+  }
+}
+
+// module.exports = LogsAdapter
+export default LogsAdapter

package/lib/adapters/pubsub-adapter/README.md

@@ -0,0 +1,86 @@
+# Pubsub Adapter
+
+The [index.js](./index.js) file contains the primary PubSub adapter, which controls the pubsub channel connections between peers. [messaging.js](./messaging.js) controls the message handling for communicating over pubsub channels.
+
+Because IPFS nodes are constantly changing their network connections, it's frequently observed that pubsub messages between peers get 'lost'. Version 6 and older used [orbit-db](https://www.npmjs.com/package/orbit-db) to prevent these lost messages. However, it turned out to not be a very scalable solution. Orbit-db is way too CPU heavy to work as a speedy form of inter-node communication.
+
+Version 7 introduced the [messaging.js](./messaging.js) library. The primary problem this library solves is the 'lost message' issue. The IPFS pubsub channels handle the bulk of the low-level messaging.
+
+## Messaging Protocol
+
+To handle 'lost messages', two peers engage in a message-acknowledge scheme:
+
+Happy Path:
+
+- Node 1 publishes a message to the pubsub channel for Node 2.
+- Node 2 publishes an ACK (acknowledge) message to the pubsub channel for Node 1.
+- If both messages are received, the transaction is complete.
+
+Each message is wrapped in a data object with the following properties:
+
+- timestamp
+- UUID
+- sender (IPFS ID)
+- receiver (IPFS ID)
+- payload
+
+If Node 1 does not receive an ACK message after 5 seconds, it will publish the message to the pubsub channel again. It will do this every 5 seconds, until either an ACK message is received or a retry threshold is met (3 tries).
+
+Node 2 will attempt to send an ACK message any time it receives a message.
+
+It's important to note that two pubsub channels are used. Node 1 sends data on Node 2's pubsub channel. Node 2 responds by publishing data on Node 1's pubsub channel.
+
+## Libraries
+
+To understand the relationships between the messaing (messaging.js) and pubsub (index.js) libraries, and comparison to the [OSI model](https://www.imperva.com/learn/application-security/osi-model/) can be made. In the OSI model, from top to bottom, there is:
+
+- A presentation layer (6).
+- A session layer (5)
+- A transport layer (4)
+
+Analogously:
+- The pubsub library ([index.js](./index.js)) is like the presentation layer (6).
+- The messaging library ([messaging.js](./messaging.js)) is like the session layer (5).
+- The IPFS pubsub API is like the transport layer (4).
+
+The point is that there are two different communication protocols happening at the same time:
+- The underlying messaging protocol described here *does not* care about the content of the messages. It's just trying to ensure messages are passes reliably.
+- The message handling in the pubsub library *does* care about the content of the messages.
+
+
+## Example
+
+Here is an example of an RPC command wrapped inside of a message envelope. The payload contains the encrypted RPC command. Notice the message itself has a UUID (universally unique identifier).
+
+```javascript
+{
+  "timestamp": "2022-03-03T18:03:01.217Z",
+  "uuid": "e0f08e3f-6e23-43ea-8fa9-39b828fd4fdc",
+  "sender": "12D3KooWHS5A6Ey4V8fLWD64jpPn2EKi4r4btGN6FfkNgMTnfqVa",
+  "receiver": "12D3KooWE6tkdArVpCHG9QN61G1cE7eCq2Q7i4bNx6CJFTDprk9f",
+  "payload": "0453769b22a27adbb288bb2a05c19605a1fb033d4aeca92dfbf59f5c61732f89e6668a9a645dea3fe82e1bda20b8b11e713586b2c39d33f00dcf8fddac401263c320f06136a494e965f34193c3c6bb670a146c2ec06cdb5fd564b11a25c8715d574b8e1fd57f90e697c1edf21eb27ec0c431ce83293a4611e9593a53490c220019867208a1e241f23851c91646521b2e47"
+}
+```
+
+Once the payload is decrypted and parsed, it looks like this. Notice that the RPC command has it's own UUID.
+
+```javascript
+{
+  jsonrpc: '2.0',
+  id: 'bb788af4-b7d1-4650-a7ae-bfd03b7f5140',
+  method: 'bch',
+  params: {
+    endpoint: 'utxos',
+    address: 'bitcoincash:qr2u4f2dmva6yvf3npkd5lquryp09qk7gs5vxl423h'
+  }
+}
+```
+
+### Workflow
+
+Using the above RPC command as an example, here is the messaging workflow between Node 1 and Node 2:
+
+- Node 1 sends the RPC command to Node 2
+- Node 2 immediately responds with an ACK message
+- Node 2 processes the RPC command and sends the response to Node 1
+- Node 1 immediately reponds with an ACK message

package/lib/adapters/pubsub-adapter/about-adapter.js

@@ -0,0 +1,117 @@
+/*
+  This adapter is designed to poll the /about JSON RPC endpoint for IPFS Service
+  Providers that leverage ipfs-coord. This allows other consumers of the
+  ipfs-coord library to measure the latency between themselves and potential
+  Circuit Relays.
+
+  Using these metrics, IPFS nodes can prioritize the Circuit Relays with the
+  lowest latency. At scale, this allows the entire IPFS subnet to adapt to
+  changing network conditions.
+*/
+
+class AboutAdapter {
+  constructor (localConfig = {}) {
+    // Time to wait for a reponse from the RPC.
+    this.waitPeriod = 10000
+
+    // Used to pass asynchronous data when pubsub data is received.
+    this.incomingData = false
+  }
+
+  // Query the /about JSON RPC endpoint for a subnet peer.
+  // This function will return true on success or false on failure or timeout
+  // of 10 seconds.
+  // This function is used to measure the time for a response from the peer.
+  async queryAbout (ipfsId, thisNode) {
+    try {
+      // console.log(`Querying Relay ${ipfsId}`)
+      // console.log('thisNode: ', thisNode)
+
+      // Generate the JSON RPC command
+      const idNum = Math.floor(Math.random() * 10000).toString()
+      const id = `metrics${idNum}`
+      const cmdStr = `{"jsonrpc":"2.0","id":"${id}","method":"about"}`
+      // console.log(`cmdStr: ${cmdStr}`)
+
+      // console.log(`Sending JSON RPC /about command to ${ipfsId}`)
+      const result = await this.sendRPC(ipfsId, cmdStr, id, thisNode)
+      // console.log('sendRPC result: ', result)
+
+      return result
+    } catch (err) {
+      console.error('Error in queryAbout()')
+
+      // Do not throw an error.
+      return false
+    }
+  }
+
+  // This function is called by pubsub.captureMetrics() when a response is
+  // recieved to an /about request. The data is used by sendRPC().
+  relayMetricsReceived (inData) {
+    this.incomingData = inData
+  }
+
+  // Send the RPC command to the service, wait a period of time for a response.
+  // Timeout if a response is not recieved.
+  async sendRPC (ipfsId, cmdStr, id, thisNode) {
+    try {
+      let retData = this.incomingData
+
+      // Send the RPC command to the server/service.
+      await thisNode.useCases.peer.sendPrivateMessage(ipfsId, cmdStr, thisNode)
+
+      // Used for calculating the timeout.
+      const start = new Date()
+      let now = start
+      let timeDiff = 0
+
+      // Wait for the response from the server. Exit once the response is
+      // recieved, or a timeout occurs.
+      do {
+        await thisNode.useCases.peer.adapters.bch.bchjs.Util.sleep(250)
+
+        now = new Date()
+
+        timeDiff = now.getTime() - start.getTime()
+        // console.log('timeDiff: ', timeDiff)
+
+        retData = this.incomingData
+
+        // If data came in on the event emitter, analize it.
+        if (retData) {
+          // console.log('retData: ', retData)
+
+          const jsonData = JSON.parse(retData)
+          const respId = jsonData.id
+
+          // If the JSON RPC ID matches, then it's the response thisNode was
+          // waiting for.
+          if (respId === id) {
+            // responseRecieved = true
+            // this.eventEmitter.removeListener('relayMetrics', cb)
+
+            retData = false
+
+            return true
+          }
+        }
+
+      // console.log('retData: ', retData)
+      } while (
+        // Exit once the RPC data comes back, or if a period of time passes.
+        timeDiff < this.waitPeriod
+      )
+
+      // this.eventEmitter.removeListener('relayMetrics', cb)
+      return false
+    } catch (err) {
+      console.error('Error in sendRPC')
+      // this.eventEmitter.removeListener('relayMetrics', cb)
+      throw err
+    }
+  }
+}
+
+// module.exports = AboutAdapter
+export default AboutAdapter