@helia/interop 3.0.0 → 3.0.1-19bf9ce

package/src/helia-pins.spec.ts

@@ -0,0 +1,64 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import all from 'it-all'
+import drain from 'it-drain'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('helia - pins', () => {
+  let helia: HeliaLibp2p
+  let kubo: Controller
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    kubo = await createKuboNode()
+
+    // connect the two nodes
+    await helia.libp2p.dial(kubo.peer.addresses)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('pinning on kubo should pull from helia', async () => {
+    const input = Uint8Array.from([0, 1, 2, 3, 4])
+    const digest = await sha256.digest(input)
+    const cid = CID.createV1(raw.code, digest)
+
+    expect((await all(kubo.api.refs.local())).map(r => r.ref)).to.not.include(cid.toString())
+
+    await helia.blockstore.put(cid, input)
+
+    const pinned = await kubo.api.pin.add(cid)
+    expect(pinned.toString()).to.equal(cid.toString())
+
+    expect((await all(kubo.api.refs.local())).map(r => r.ref)).to.include(cid.toString())
+  })
+
+  it('pinning on helia should pull from kubo', async () => {
+    const input = Uint8Array.from([0, 1, 2, 3, 4])
+    const { cid } = await kubo.api.add({ content: input }, {
+      cidVersion: 1,
+      rawLeaves: true
+    })
+
+    await expect(helia.blockstore.has(CID.parse(cid.toString()))).to.eventually.be.false()
+
+    await drain(helia.pins.add(CID.parse(cid.toString())))
+
+    await expect(helia.blockstore.has(CID.parse(cid.toString()))).to.eventually.be.true()
+  })
+})

package/src/index.ts

@@ -1 +1,14 @@
+/**
+ * @packageDocumentation
+ *
+ * Runs interop tests between Helia and Kubo.
+ *
+ * @example Testing a new Kubo release
+ *
+ * ```console
+ * $ npm i @helia/interop
+ * $ KUBO_BINARY=/path/to/kubo helia-interop
+ * ```
+ */
+
 export {}

package/src/ipns-pubsub.spec.ts

@@ -0,0 +1,180 @@
+/* eslint-env mocha */
+/* eslint max-nested-callbacks: ["error", 5] */
+
+import { ipns } from '@helia/ipns'
+import { pubsub } from '@helia/ipns/routing'
+import { peerIdFromKeys } from '@libp2p/peer-id'
+import { expect } from 'aegir/chai'
+import last from 'it-last'
+import { base36 } from 'multiformats/bases/base36'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { identity } from 'multiformats/hashes/identity'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { connect } from './fixtures/connect.js'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import { keyTypes } from './fixtures/key-types.js'
+import { waitFor } from './fixtures/wait-for.js'
+import type { IPNS } from '@helia/ipns'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+const LIBP2P_KEY_CODEC = 0x72
+
+// skip RSA tests because we need the DHT enabled to find the public key
+// component of the keypair, but that means we can't test pubsub
+// resolution because Kubo will use the DHT as well
+keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => {
+  describe(`@helia/ipns - pubsub routing with ${keyType} keys`, () => {
+    let helia: HeliaLibp2p
+    let kubo: Controller
+    let name: IPNS
+
+    beforeEach(async () => {
+      helia = await createHeliaNode()
+      kubo = await createKuboNode()
+
+      // connect the two nodes
+      await connect(helia, kubo, '/meshsub/1.1.0')
+
+      name = ipns(helia, {
+        routers: [
+          pubsub(helia)
+        ]
+      })
+    })
+
+    afterEach(async () => {
+      if (helia != null) {
+        await helia.stop()
+      }
+
+      if (kubo != null) {
+        await kubo.stop()
+      }
+    })
+
+    it('should publish on helia and resolve on kubo', async () => {
+      const input = Uint8Array.from([0, 1, 2, 3, 4])
+      const digest = await sha256.digest(input)
+      const cid = CID.createV1(raw.code, digest)
+
+      const keyName = 'my-ipns-key'
+      await helia.libp2p.services.keychain.createKey(keyName, keyType)
+      const peerId = await helia.libp2p.services.keychain.exportPeerId(keyName)
+
+      if (peerId.publicKey == null) {
+        throw new Error('No public key present')
+      }
+
+      // first publish should fail because kubo isn't subscribed to key update channel
+      await expect(name.publish(peerId, cid)).to.eventually.be.rejected()
+        .with.property('message', 'PublishError.InsufficientPeers')
+
+      // should fail to resolve the first time as kubo was not subscribed to the pubsub channel
+      // @ts-expect-error kubo deps are out of date
+      await expect(last(kubo.api.name.resolve(peerId, {
+        timeout: 100
+      }))).to.eventually.be.undefined()
+
+      // magic pubsub subscription name
+      const subscriptionName = `/ipns/${CID.createV1(LIBP2P_KEY_CODEC, identity.digest(peerId.publicKey)).toString(base36)}`
+
+      // wait for kubo to be subscribed to updates
+      await waitFor(async () => {
+        const subs = await kubo.api.name.pubsub.subs()
+
+        return subs.includes(subscriptionName)
+      }, {
+        timeout: 30000
+      })
+
+      // publish should now succeed
+      await name.publish(peerId, cid)
+
+      // kubo should now be able to resolve IPNS name
+      // @ts-expect-error kubo deps are out of date
+      const resolved = await last(kubo.api.name.resolve(peerId, {
+        timeout: 100
+      }))
+
+      expect(resolved).to.equal(`/ipfs/${cid.toString()}`)
+    })
+
+    it('should publish on kubo and resolve on helia', async function () {
+      if (keyType === 'secp256k1') {
+        // Kubo cannot generate secp256k1 keys
+        return this.skip()
+      }
+
+      const keyName = 'my-ipns-key'
+      const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4]))
+      const result = await kubo.api.key.gen(keyName, {
+        // @ts-expect-error kubo needs this in lower case
+        type: keyType.toLowerCase()
+      })
+
+      // the generated id is libp2p-key CID with the public key as an identity multihash
+      const peerCid = CID.parse(result.id, base36)
+      const peerId = await peerIdFromKeys(peerCid.multihash.digest)
+
+      // first call to pubsub resolver should fail but we should now be subscribed for updates
+      await expect(name.resolve(peerId)).to.eventually.be.rejected()
+
+      // actual pubsub subscription name
+      const subscriptionName = `/record/${uint8ArrayToString(uint8ArrayConcat([
+        uint8ArrayFromString('/ipns/'),
+        peerId.toBytes()
+      ]), 'base64url')}`
+
+      // wait for helia to be subscribed to the topic for record updates
+      await waitFor(async () => {
+        return helia.libp2p.services.pubsub.getTopics().includes(subscriptionName)
+      }, {
+        timeout: 30000,
+        message: 'Helia did not register for record updates'
+      })
+
+      // wait for kubo to see that helia is subscribed to the topic for record updates
+      await waitFor(async () => {
+        const peers = await kubo.api.pubsub.peers(subscriptionName)
+
+        return peers.map(p => p.toString()).includes(helia.libp2p.peerId.toString())
+      }, {
+        timeout: 30000,
+        message: 'Kubo did not see that Helia was registered for record updates'
+      })
+
+      // now publish, this should cause a pubsub message on the topic for record updates
+      await kubo.api.name.publish(cid, {
+        key: keyName
+      })
+
+      let resolvedCid: CID | undefined
+
+      // we should get an update eventually
+      await waitFor(async () => {
+        try {
+          resolvedCid = await name.resolve(peerId)
+
+          return true
+        } catch {
+          return false
+        }
+      }, {
+        timeout: 10000,
+        message: 'Helia could not resolve the IPNS record'
+      })
+
+      if (resolvedCid == null) {
+        throw new Error('Failed to resolve CID')
+      }
+
+      expect(resolvedCid.toString()).to.equal(cid.toString())
+    })
+  })
+})

package/src/ipns.spec.ts

@@ -0,0 +1,183 @@
+/* eslint-env mocha */
+
+import { ipns } from '@helia/ipns'
+import { peerIdFromString } from '@libp2p/peer-id'
+import { createEd25519PeerId, createRSAPeerId, createSecp256k1PeerId } from '@libp2p/peer-id-factory'
+import { expect } from 'aegir/chai'
+import last from 'it-last'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { isElectronMain } from 'wherearewe'
+import { connect } from './fixtures/connect.js'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import { sortClosestPeers } from './fixtures/create-peer-ids.js'
+import { keyTypes } from './fixtures/key-types.js'
+import { waitFor } from './fixtures/wait-for.js'
+import type { IPNS } from '@helia/ipns'
+import type { PeerId } from '@libp2p/interface'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+keyTypes.forEach(type => {
+  describe(`@helia/ipns - default routing with ${type} keys`, () => {
+    let helia: HeliaLibp2p
+    let kubo: Controller
+    let name: IPNS
+
+    // the CID we are going to publish
+    let value: CID
+
+    // the public key we will use to publish the value
+    let key: PeerId
+
+    /**
+     * Ensure that for the CID we are going to publish, the resolver has a peer ID that
+     * is KAD-closer to the routing key so we can predict the the resolver will receive
+     * the DHT record containing the IPNS record
+     */
+    async function createNodes (resolver: 'kubo' | 'helia'): Promise<void> {
+      const input = Uint8Array.from([0, 1, 2, 3, 4])
+      const digest = await sha256.digest(input)
+      value = CID.createV1(raw.code, digest)
+
+      helia = await createHeliaNode()
+      kubo = await createKuboNode()
+
+      // find a PeerId that is KAD-closer to the resolver than the publisher when used as an IPNS key
+      while (true) {
+        if (type === 'Ed25519') {
+          key = await createEd25519PeerId()
+        } else if (type === 'secp256k1') {
+          key = await createSecp256k1PeerId()
+        } else {
+          key = await createRSAPeerId()
+        }
+
+        const routingKey = uint8ArrayConcat([
+          uint8ArrayFromString('/ipns/'),
+          key.toBytes()
+        ])
+
+        const [closest] = await sortClosestPeers(routingKey, [
+          helia.libp2p.peerId,
+          peerIdFromString(kubo.peer.id.toString())
+        ])
+
+        if (resolver === 'kubo' && closest.equals(peerIdFromString(kubo.peer.id.toString()))) {
+          break
+        }
+
+        if (resolver === 'helia' && closest.equals(helia.libp2p.peerId)) {
+          break
+        }
+      }
+
+      // connect the two nodes over the KAD-DHT protocol, this should ensure
+      // both nodes have each other in their KAD buckets
+      await connect(helia, kubo, '/ipfs/lan/kad/1.0.0')
+
+      await waitFor(async () => {
+        let found = false
+
+        for await (const event of helia.libp2p.services.dht.findPeer(peerIdFromString(kubo.peer.id.toString()))) {
+          if (event.name === 'FINAL_PEER') {
+            found = true
+          }
+        }
+
+        return found
+      }, {
+        timeout: 30000,
+        delay: 1000,
+        message: 'Helia could not find Kubo on the DHT'
+      })
+
+      await waitFor(async () => {
+        let found = false
+
+        // @ts-expect-error kubo deps are out of date
+        for await (const event of kubo.api.dht.findPeer(helia.libp2p.peerId)) {
+          if (event.name === 'FINAL_PEER') {
+            found = true
+          }
+        }
+
+        return found
+      }, {
+        timeout: 30000,
+        delay: 1000,
+        message: 'Kubo could not find Helia on the DHT'
+      })
+
+      name = ipns(helia)
+    }
+
+    afterEach(async () => {
+      if (helia != null) {
+        await helia.stop()
+      }
+
+      if (kubo != null) {
+        await kubo.stop()
+      }
+    })
+
+    it(`should publish on helia and resolve on kubo using a ${type} key`, async () => {
+      await createNodes('kubo')
+
+      const keyName = 'my-ipns-key'
+      await helia.libp2p.services.keychain.importPeer(keyName, key)
+
+      await name.publish(key, value)
+
+      const resolved = await last(kubo.api.name.resolve(key.toString()))
+
+      if (resolved == null) {
+        throw new Error('kubo failed to resolve name')
+      }
+
+      expect(resolved).to.equal(`/ipfs/${value.toString()}`)
+    })
+
+    it('should publish on kubo and resolve on helia', async function () {
+      if (isElectronMain) {
+        // electron main does not have fetch, FormData or Blob APIs
+        // can revisit when kubo-rpc-client supports the key.import API
+        return this.skip()
+      }
+
+      if (type === 'secp256k1') {
+        // Kubo cannot import secp256k1 keys
+        return this.skip()
+      }
+
+      await createNodes('helia')
+
+      const keyName = 'my-ipns-key'
+      const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4]))
+
+      // ensure the key is in the kubo keychain so we can use it to publish the IPNS record
+      const body = new FormData()
+      body.append('key', new Blob([key.privateKey ?? new Uint8Array(0)]))
+
+      // can't use the kubo-rpc-api for this call yet
+      const response = await fetch(`http://${kubo.api.apiHost}:${kubo.api.apiPort}/api/v0/key/import?arg=${keyName}`, {
+        method: 'POST',
+        body
+      })
+
+      expect(response).to.have.property('status', 200)
+
+      await kubo.api.name.publish(cid, {
+        key: keyName
+      })
+
+      const resolvedCid = await name.resolve(key)
+      expect(resolvedCid.toString()).to.equal(cid.toString())
+    })
+  })
+})

package/src/json.spec.ts

@@ -0,0 +1,65 @@
+/* eslint-env mocha */
+
+import { json, type JSON, type AddOptions } from '@helia/json'
+import { expect } from 'aegir/chai'
+import { CID } from 'multiformats/cid'
+import * as jsonCodec from 'multiformats/codecs/json'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('@helia/json', () => {
+  let helia: HeliaLibp2p
+  let j: JSON
+  let kubo: Controller
+
+  async function expectSameCid (data: () => any, heliaOpts: Partial<AddOptions> = {}, kuboOpts: KuboAddOptions = { format: 'json' }): Promise<void> {
+    const heliaCid = await j.add(data(), heliaOpts)
+    const kuboCid = await kubo.api.block.put(jsonCodec.encode(data()), kuboOpts)
+
+    expect(heliaCid.toString()).to.equal(kuboCid.toString())
+  }
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    j = json(helia)
+    kubo = await createKuboNode()
+
+    await helia.libp2p.dial((await (kubo.api.id())).addresses)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should create the same CID for a string', async () => {
+    const candidate = (): any => ({ hello: 'world' })
+
+    await expectSameCid(candidate)
+  })
+
+  it('should add to helia and fetch from kubo', async () => {
+    const input = { hello: 'world' }
+    const cid = await j.add(input)
+    const block = await kubo.api.block.get(cid)
+    const output = jsonCodec.decode(block)
+
+    expect(output).to.deep.equal(input)
+  })
+
+  it('should add to kubo and fetch from helia', async () => {
+    const input = { hello: 'world' }
+    const cid = await kubo.api.block.put(jsonCodec.encode(input))
+    const output = await j.get(CID.parse(cid.toString()))
+
+    expect(output).to.deep.equal(input)
+  })
+})

package/src/mfs.spec.ts

@@ -0,0 +1,105 @@
+/* eslint-env mocha */
+
+import { type MFS, mfs } from '@helia/mfs'
+import { expect } from 'aegir/chai'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('@helia/mfs', () => {
+  let helia: HeliaLibp2p
+  let fs: MFS
+  let kubo: Controller
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    fs = mfs(helia)
+    kubo = await createKuboNode()
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should have the same CID initially', async () => {
+    const heliaStat = await fs.stat('/')
+    const kuboStat = await kubo.api.files.stat('/')
+
+    expect(heliaStat.cid.toV1().toString()).to.equal(kuboStat.cid.toV1().toString())
+  })
+
+  it('should have the same CID after creating a directory', async () => {
+    const dirPath = '/foo'
+    await fs.mkdir(dirPath)
+    await kubo.api.files.mkdir(dirPath, {
+      cidVersion: 1
+    })
+
+    const heliaStat = await fs.stat('/')
+    const kuboStat = await kubo.api.files.stat('/')
+
+    expect(heliaStat.cid.toV1().toString()).to.equal(kuboStat.cid.toV1().toString())
+  })
+
+  it('should have the same CID after removing a directory', async () => {
+    const dirPath = '/foo'
+    await fs.mkdir(dirPath)
+    await fs.rm(dirPath)
+    await kubo.api.files.mkdir(dirPath, {
+      cidVersion: 1
+    })
+    await kubo.api.files.rm(dirPath, {
+      recursive: true
+    })
+
+    const heliaStat = await fs.stat('/')
+    const kuboStat = await kubo.api.files.stat('/')
+
+    expect(heliaStat.cid.toV1().toString()).to.equal(kuboStat.cid.toV1().toString())
+  })
+
+  it('should have the same CID after creating a file', async () => {
+    const filePath = '/foo.txt'
+    const fileData = Uint8Array.from([0, 1, 2, 3, 4])
+    await fs.writeBytes(fileData, filePath, {
+      rawLeaves: true,
+      reduceSingleLeafToSelf: false
+    })
+    await kubo.api.files.write(filePath, fileData, {
+      cidVersion: 1,
+      create: true
+    })
+
+    const heliaStat = await fs.stat('/')
+    const kuboStat = await kubo.api.files.stat('/')
+
+    expect(heliaStat.cid.toV1().toString()).to.equal(kuboStat.cid.toV1().toString())
+  })
+
+  it('should have the same CID after removing a file', async () => {
+    const filePath = '/foo.txt'
+    const fileData = Uint8Array.from([0, 1, 2, 3, 4])
+    await fs.writeBytes(fileData, filePath, {
+      rawLeaves: true,
+      reduceSingleLeafToSelf: false
+    })
+    await fs.rm(filePath)
+    await kubo.api.files.write(filePath, fileData, {
+      cidVersion: 1,
+      create: true
+    })
+    await kubo.api.files.rm(filePath)
+
+    const heliaStat = await fs.stat('/')
+    const kuboStat = await kubo.api.files.stat('/')
+
+    expect(heliaStat.cid.toV1().toString()).to.equal(kuboStat.cid.toV1().toString())
+  })
+})

package/src/strings.spec.ts

@@ -0,0 +1,67 @@
+/* eslint-env mocha */
+
+import { strings, type Strings, type AddOptions } from '@helia/strings'
+import { expect } from 'aegir/chai'
+import { CID } from 'multiformats/cid'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('@helia/strings', () => {
+  let helia: HeliaLibp2p
+  let str: Strings
+  let kubo: Controller
+
+  async function expectSameCid (data: () => string, heliaOpts: Partial<AddOptions> = {}, kuboOpts: KuboAddOptions = {}): Promise<void> {
+    const heliaCid = await str.add(data(), heliaOpts)
+    const kuboCid = await kubo.api.block.put(uint8ArrayFromString(data()), kuboOpts)
+
+    expect(heliaCid.toString()).to.equal(kuboCid.toString())
+  }
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    str = strings(helia)
+    kubo = await createKuboNode()
+
+    const id = await kubo.api.id()
+    await helia.libp2p.dial(id.addresses)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should create the same CID for a string', async () => {
+    const candidate = (): string => 'hello world'
+
+    await expectSameCid(candidate)
+  })
+
+  it('should add to helia and fetch from kubo', async () => {
+    const input = 'hello world'
+    const cid = await str.add(input)
+    const block = await kubo.api.block.get(cid)
+    const output = uint8ArrayToString(block)
+
+    expect(output).to.equal(input)
+  })
+
+  it('should add to kubo and fetch from helia', async () => {
+    const input = 'hello world'
+    const cid = await kubo.api.block.put(uint8ArrayFromString(input))
+    const output = await str.get(CID.parse(cid.toString()))
+
+    expect(output).to.equal(input)
+  })
+})