helia-coord 1.5.7 → 1.5.9

package/examples/create-helia-node.js

@@ -81,7 +81,7 @@ class CreateHeliaNode {
       // Configure services
       const services = {
         identify: identify(),
-        pubsub: gossipsub({ allowPublishToZeroPeers: true })
+        pubsub: gossipsub({ allowPublishToZeroTopicPeers: true })
       }
 
       // libp2p is the networking layer that underpins Helia
@@ -143,6 +143,8 @@ class CreateHeliaNode {
 
       !fs.existsSync(`${IPFS_DIR}/datastore`) && fs.mkdirSync(`${IPFS_DIR}/datastore`)
 
+      // !fs.existsSync(`${IPFS_DIR}/datastore/peers`) && fs.mkdirSync(`${IPFS_DIR}/datastore/peers`)
+
       return true
     } catch (err) {
       console.error('Error in ensureBlocksDir(): ', err)

package/examples/memory-usage/README.md

@@ -0,0 +1,12 @@
+# Memory Usage
+
+This is the same example app as the directory above, but it's been modified to take
+periodic memory snapshots and write the memory usage to a JSON file. This is useful
+for tracking potential memory leaks by monitoring memory usage over time.
+
+## Usage
+
+Copy the `memory-log-example.json` to a new file called `memory-log.json`
+
+Run this program with:
+- `node start-node.js`

package/examples/memory-usage/create-helia-node.js

@@ -0,0 +1,154 @@
+/*
+  This library creates a Helia IPFS node. This is done prior to attaching
+  helia-coord to the node.
+  This library is called by start-node.js.
+*/
+
+// Global npm libraries
+import { createHelia } from 'helia'
+import fs from 'fs'
+import { FsBlockstore } from 'blockstore-fs'
+import { FsDatastore } from 'datastore-fs'
+import { createLibp2p } from 'libp2p'
+import { tcp } from '@libp2p/tcp'
+import { noise } from '@chainsafe/libp2p-noise'
+import { yamux } from '@chainsafe/libp2p-yamux'
+// import { bootstrap } from '@libp2p/bootstrap'
+// import { identifyService } from 'libp2p/identify'
+import { identify } from '@libp2p/identify'
+// import { circuitRelayTransport } from 'libp2p/circuit-relay'
+import { circuitRelayTransport } from '@libp2p/circuit-relay-v2'
+import { gossipsub } from '@chainsafe/libp2p-gossipsub'
+import { webSockets } from '@libp2p/websockets'
+import { publicIpv4 } from 'public-ip'
+import { multiaddr } from '@multiformats/multiaddr'
+import { webRTC } from '@libp2p/webrtc'
+
+const ROOT_DIR = './'
+const IPFS_DIR = './.ipfsdata/ipfs'
+
+class CreateHeliaNode {
+  constructor () {
+    this.publicIp = publicIpv4
+  }
+
+  // Start an IPFS node.
+  async start () {
+    try {
+      // Ensure the directory structure exists that is needed by the IPFS node to store data.
+      this.ensureBlocksDir()
+
+      // Create an IPFS node
+      const ipfs = await this.createNode()
+      // console.log('ipfs: ', ipfs)
+
+      this.id = ipfs.libp2p.peerId.toString()
+      console.log('IPFS ID: ', this.id)
+
+      // Attempt to guess our ip4 IP address.
+      const ip4 = await this.publicIp()
+      let detectedMultiaddr = `/ip4/${ip4}/tcp/4001/p2p/${this.id}`
+      detectedMultiaddr = multiaddr(detectedMultiaddr)
+
+      // Get the multiaddrs for the node.
+      const multiaddrs = ipfs.libp2p.getMultiaddrs()
+      multiaddrs.push(detectedMultiaddr)
+      console.log('Multiaddrs: ', multiaddrs)
+
+      this.multiaddrs = multiaddrs
+
+      // Signal that this adapter is ready.
+      this.isReady = true
+
+      this.ipfs = ipfs
+
+      return this.ipfs
+    } catch (err) {
+      console.error('Error in start()')
+
+      throw err
+    }
+  }
+
+  // This function creates an IPFS node using Helia.
+  // It returns the node as an object.
+  async createNode () {
+    try {
+      // Create block and data stores.
+      const blockstore = new FsBlockstore(`${IPFS_DIR}/blockstore`)
+      const datastore = new FsDatastore(`${IPFS_DIR}/datastore`)
+
+      // Configure services
+      const services = {
+        identify: identify(),
+        pubsub: gossipsub({ allowPublishToZeroPeers: true })
+      }
+
+      // libp2p is the networking layer that underpins Helia
+      const libp2p = await createLibp2p({
+        datastore,
+        addresses: {
+          listen: [
+            '/ip4/127.0.0.1/tcp/0',
+            '/ip4/0.0.0.0/tcp/4001',
+            '/ip4/0.0.0.0/tcp/4003/ws',
+            '/webrtc'
+          ]
+        },
+        transports: [
+          tcp(),
+          webSockets(),
+          circuitRelayTransport({ discoverRelays: 3 }),
+          webRTC()
+        ],
+        connectionEncryption: [
+          noise()
+        ],
+        streamMuxers: [
+          yamux()
+        ],
+        services
+      })
+
+      // create a Helia node
+      const helia = await createHelia({
+        blockstore,
+        datastore,
+        libp2p
+      })
+
+      return helia
+    } catch (err) {
+      console.error('Error creating Helia node: ', err)
+
+      throw err
+    }
+  }
+
+  async stop () {
+    await this.ipfs.stop()
+
+    return true
+  }
+
+  // Ensure that the directories exist to store blocks from the IPFS network.
+  // This function is called at startup, before the IPFS node is started.
+  ensureBlocksDir () {
+    try {
+      !fs.existsSync(`${ROOT_DIR}.ipfsdata`) && fs.mkdirSync(`${ROOT_DIR}.ipfsdata`)
+
+      !fs.existsSync(`${IPFS_DIR}`) && fs.mkdirSync(`${IPFS_DIR}`)
+
+      !fs.existsSync(`${IPFS_DIR}/blockstore`) && fs.mkdirSync(`${IPFS_DIR}/blockstore`)
+
+      !fs.existsSync(`${IPFS_DIR}/datastore`) && fs.mkdirSync(`${IPFS_DIR}/datastore`)
+
+      return true
+    } catch (err) {
+      console.error('Error in ensureBlocksDir(): ', err)
+      throw err
+    }
+  }
+}
+
+export default CreateHeliaNode

package/examples/memory-usage/json-to-csv.js

@@ -0,0 +1,28 @@
+/*
+  This app converts the memory-log.json file to a CSV file that can be imported
+  into a spreadsheet for further analysis.
+*/
+
+import fs from 'fs'
+
+async function jsonToCsv () {
+  try {
+    const log = JSON.parse(fs.readFileSync('./memory-log.json'))
+    const memoryLogs = log.memoryLogs
+
+    let outStr = 'timestampIso,timestampJs,heapTotal (MB),heapUsed (MB),external (MB),arrayBuffers (MB),rss (MB)\n'
+
+    for (let i = 0; i < memoryLogs.length; i++) {
+      const thisLog = memoryLogs[i]
+
+      outStr += `${thisLog.timestampIso},${thisLog.timestampJs},${thisLog.heapTotal / 1000000},${thisLog.heapUsed / 1000000},${thisLog.external / 1000000},${thisLog.arrayBuffers / 1000000},${thisLog.rss / 1000000}\n`
+    }
+
+    await fs.writeFileSync('./memory-log.csv', outStr)
+    console.log('JSON memory log converted to CSV file.')
+  } catch (err) {
+    console.error('Error in jsonToCsv(): ', err)
+  }
+}
+
+jsonToCsv()

package/examples/memory-usage/memory-log-example.json

@@ -0,0 +1,3 @@
+{
+  "memoryLogs": []
+}

package/examples/memory-usage/memory-log.json

@@ -0,0 +1,3 @@
+{
+  "memoryLogs": []
+}

package/examples/memory-usage/start-node.js

@@ -0,0 +1,57 @@
+/*
+  This is an example of how to start a Helia IPFS node with node.js and attach
+  the helia-coord library to it.
+*/
+
+// Global npm libraries
+import SlpWallet from 'minimal-slp-wallet'
+import { memoryUsage } from 'node:process'
+import fs from 'fs'
+
+// Local libraries
+import IpfsCoord from '../../index.js'
+import CreateHeliaNode from './create-helia-node.js'
+
+async function start () {
+  // Create an instance of bch-js and IPFS.
+  const wallet = new SlpWallet()
+  await wallet.walletInfoPromise
+
+  const createHeliaNode = new CreateHeliaNode()
+  const ipfs = await createHeliaNode.start()
+
+  // Pass bch-js and IPFS to ipfs-coord when instantiating it.
+  const ipfsCoord = new IpfsCoord({
+    ipfs,
+    wallet,
+    type: 'node.js',
+    // type: 'browser'
+    nodeType: 'external',
+    debugLevel: 2
+  })
+
+  await ipfsCoord.start()
+  console.log('IPFS and the coordination library is ready.')
+
+  setInterval(async function () {
+    try {
+      const memoryLog = JSON.parse(fs.readFileSync('./memory-log.json'))
+      // console.log('memoryLog: ', memoryLog)
+
+      const memory = memoryUsage()
+
+      const now = new Date()
+      memory.timestampIso = now.toISOString()
+      memory.timestampLocal = now.toLocaleString()
+      memory.timestampJs = now.getTime()
+
+      memoryLog.memoryLogs.push(memory)
+
+      await fs.writeFileSync('./memory-log.json', JSON.stringify(memoryLog, null, 2))
+      console.log(`Memory logged at ${now.toLocaleString()}`)
+    } catch (err) {
+      console.error('Error trying to measure memory: ', err)
+    }
+  }, 60000)
+}
+start()

package/index.js

@@ -90,7 +90,7 @@ class IpfsCoord {
     this.useCases.pubsub.updateThisNode(this.thisNode)
 
     // Subscribe to Pubsub Channels
-    // await this.useCases.pubsub.initializePubsub(this.thisNode)
+    // feature commented out
     await this.useCases.pubsub.initializePubsub({ controllers: this.controllers })
 
     // Start timer-based controllers.

package/lib/adapters/ipfs-adapter.js

@@ -76,14 +76,14 @@ class IpfsAdapter {
   // Attempts to connect to an IPFS peer, given its IPFS multiaddr.
   // Returns true if the connection succeeded. Otherwise returns false.
   async connectToPeer (inObj = {}) {
+    const { multiaddr } = inObj
+
     try {
       // console.log('connectToPeer() inObj: ', inObj)
 
       // TODO: Throw error if ipfs ID is passed, instead of a multiaddr.
       // console.log('ipfsAddr: ', ipfsAddr)
 
-      const { multiaddr } = inObj
-
       // await this.ipfs.swarm.connect(ipfsAddr, { timeout: CONNECTION_TIMEOUT })
       await this.ipfs.libp2p.dial(this.multiaddr(multiaddr))
       // console.log('connectToPeer() result: ', result)
@@ -97,7 +97,7 @@ class IpfsAdapter {
     } catch (err) {
       /* exit quietly */
       // console.log('connectToPeer() Error connecting to peer: ', err)
-      this.log.statusLog(2, 'Error trying to connect to peer node : ', err.message)
+      this.log.statusLog(2, `Error trying to connect to peer node ${multiaddr}: `, err.message)
 
       return {
         success: false,

package/lib/controllers/timer-controller.js

@@ -81,9 +81,9 @@ class TimerControllers {
 
     // Periodically try to connect to problematic peers that advertise as
     // potential circuit relays.
-    this.relaySearchHandle = setInterval(async function () {
-      await _this.searchForRelays(thisNode, useCases)
-    }, this.relaySearchInterval)
+    // this.relaySearchHandle = setInterval(async function () {
+    //   await _this.searchForRelays(thisNode, useCases)
+    // }, this.relaySearchInterval)
 
     this.listPubsubChannelsHandle = setInterval(async function () {
       await _this.listPubsubChannels()

package/lib/use-cases/pubsub-use-cases.js

@@ -151,7 +151,7 @@ class PubsubUseCase {
       // Skip any repeated messages
       const shouldProcess = this.checkForDuplicateMsg(msg)
       if (!shouldProcess) {
-        console.log('parseCoordPubsub() duplicate message.')
+        // console.log('parseCoordPubsub() duplicate message.')
         return false
       }
 
@@ -162,7 +162,7 @@ class PubsubUseCase {
       // Ignore this message if it originated from this IPFS node.
       const thisNodeId = this.adapters.ipfs.ipfsPeerId
       if (from === thisNodeId) {
-        console.log('parseCoordPubsub() message came from this node.')
+        // console.log('parseCoordPubsub() message came from this node.')
         return false
       }
 

package/lib/use-cases/this-node-use-cases.js

@@ -97,6 +97,7 @@ class ThisNodeUseCases {
 
     // Subscribe to my own pubsub channel, for receiving info from other peers.
     this.tempHandler = () => {} // This handler will be overwritten by handleNewMessage()
+
     selfData.pubsub = await this.adapters.pubsub.subscribeToPubsubChannel(
       selfData.ipfsId,
       this.tempHandler,

package/mem-test/README.md

@@ -0,0 +1,12 @@
+# Memory Usage
+
+This is the same example app as the directory above, but it's been modified to take
+periodic memory snapshots and write the memory usage to a JSON file. This is useful
+for tracking potential memory leaks by monitoring memory usage over time.
+
+## Usage
+
+Copy the `memory-log-example.json` to a new file called `memory-log.json`
+
+Run this program with:
+- `node start-node.js`

package/mem-test/create-helia-node.js

@@ -0,0 +1,154 @@
+/*
+  This library creates a Helia IPFS node. This is done prior to attaching
+  helia-coord to the node.
+  This library is called by start-node.js.
+*/
+
+// Global npm libraries
+import { createHelia } from 'helia'
+import fs from 'fs'
+import { FsBlockstore } from 'blockstore-fs'
+import { FsDatastore } from 'datastore-fs'
+import { createLibp2p } from 'libp2p'
+import { tcp } from '@libp2p/tcp'
+import { noise } from '@chainsafe/libp2p-noise'
+import { yamux } from '@chainsafe/libp2p-yamux'
+// import { bootstrap } from '@libp2p/bootstrap'
+// import { identifyService } from 'libp2p/identify'
+import { identify } from '@libp2p/identify'
+// import { circuitRelayTransport } from 'libp2p/circuit-relay'
+import { circuitRelayTransport } from '@libp2p/circuit-relay-v2'
+import { gossipsub } from '@chainsafe/libp2p-gossipsub'
+import { webSockets } from '@libp2p/websockets'
+import { publicIpv4 } from 'public-ip'
+import { multiaddr } from '@multiformats/multiaddr'
+import { webRTC } from '@libp2p/webrtc'
+
+const ROOT_DIR = './'
+const IPFS_DIR = './.ipfsdata/ipfs'
+
+class CreateHeliaNode {
+  constructor () {
+    this.publicIp = publicIpv4
+  }
+
+  // Start an IPFS node.
+  async start () {
+    try {
+      // Ensure the directory structure exists that is needed by the IPFS node to store data.
+      this.ensureBlocksDir()
+
+      // Create an IPFS node
+      const ipfs = await this.createNode()
+      // console.log('ipfs: ', ipfs)
+
+      this.id = ipfs.libp2p.peerId.toString()
+      console.log('IPFS ID: ', this.id)
+
+      // Attempt to guess our ip4 IP address.
+      const ip4 = await this.publicIp()
+      let detectedMultiaddr = `/ip4/${ip4}/tcp/4001/p2p/${this.id}`
+      detectedMultiaddr = multiaddr(detectedMultiaddr)
+
+      // Get the multiaddrs for the node.
+      const multiaddrs = ipfs.libp2p.getMultiaddrs()
+      multiaddrs.push(detectedMultiaddr)
+      console.log('Multiaddrs: ', multiaddrs)
+
+      this.multiaddrs = multiaddrs
+
+      // Signal that this adapter is ready.
+      this.isReady = true
+
+      this.ipfs = ipfs
+
+      return this.ipfs
+    } catch (err) {
+      console.error('Error in start()')
+
+      throw err
+    }
+  }
+
+  // This function creates an IPFS node using Helia.
+  // It returns the node as an object.
+  async createNode () {
+    try {
+      // Create block and data stores.
+      const blockstore = new FsBlockstore(`${IPFS_DIR}/blockstore`)
+      const datastore = new FsDatastore(`${IPFS_DIR}/datastore`)
+
+      // Configure services
+      const services = {
+        identify: identify(),
+        pubsub: gossipsub({ allowPublishToZeroPeers: true })
+      }
+
+      // libp2p is the networking layer that underpins Helia
+      const libp2p = await createLibp2p({
+        datastore,
+        addresses: {
+          listen: [
+            '/ip4/127.0.0.1/tcp/0',
+            '/ip4/0.0.0.0/tcp/4001',
+            '/ip4/0.0.0.0/tcp/4003/ws',
+            '/webrtc'
+          ]
+        },
+        transports: [
+          tcp(),
+          webSockets(),
+          circuitRelayTransport({ discoverRelays: 3 }),
+          webRTC()
+        ],
+        connectionEncryption: [
+          noise()
+        ],
+        streamMuxers: [
+          yamux()
+        ],
+        services
+      })
+
+      // create a Helia node
+      const helia = await createHelia({
+        blockstore,
+        datastore,
+        libp2p
+      })
+
+      return helia
+    } catch (err) {
+      console.error('Error creating Helia node: ', err)
+
+      throw err
+    }
+  }
+
+  async stop () {
+    await this.ipfs.stop()
+
+    return true
+  }
+
+  // Ensure that the directories exist to store blocks from the IPFS network.
+  // This function is called at startup, before the IPFS node is started.
+  ensureBlocksDir () {
+    try {
+      !fs.existsSync(`${ROOT_DIR}.ipfsdata`) && fs.mkdirSync(`${ROOT_DIR}.ipfsdata`)
+
+      !fs.existsSync(`${IPFS_DIR}`) && fs.mkdirSync(`${IPFS_DIR}`)
+
+      !fs.existsSync(`${IPFS_DIR}/blockstore`) && fs.mkdirSync(`${IPFS_DIR}/blockstore`)
+
+      !fs.existsSync(`${IPFS_DIR}/datastore`) && fs.mkdirSync(`${IPFS_DIR}/datastore`)
+
+      return true
+    } catch (err) {
+      console.error('Error in ensureBlocksDir(): ', err)
+      throw err
+    }
+  }
+}
+
+export default CreateHeliaNode

package/mem-test/json-to-csv.js

@@ -0,0 +1,28 @@
+/*
+  This app converts the memory-log.json file to a CSV file that can be imported
+  into a spreadsheet for further analysis.
+*/
+
+import fs from 'fs'
+
+async function jsonToCsv () {
+  try {
+    const log = JSON.parse(fs.readFileSync('./memory-log.json'))
+    const memoryLogs = log.memoryLogs
+
+    let outStr = 'timestampIso,timestampJs,heapTotal (MB),heapUsed (MB),external (MB),arrayBuffers (MB),rss (MB)\n'
+
+    for (let i = 0; i < memoryLogs.length; i++) {
+      const thisLog = memoryLogs[i]
+
+      outStr += `${thisLog.timestampIso},${thisLog.timestampJs},${thisLog.heapTotal / 1000000},${thisLog.heapUsed / 1000000},${thisLog.external / 1000000},${thisLog.arrayBuffers / 1000000},${thisLog.rss / 1000000}\n`
+    }
+
+    await fs.writeFileSync('./memory-log.csv', outStr)
+    console.log('JSON memory log converted to CSV file.')
+  } catch (err) {
+    console.error('Error in jsonToCsv(): ', err)
+  }
+}
+
+jsonToCsv()

package/mem-test/memory-log.json

@@ -0,0 +1,3 @@
+{
+  "memoryLogs": []
+}