@helia/interop 3.0.1 → 4.0.0-e554493

package/src/fixtures/wait-for.ts

@@ -0,0 +1,26 @@
+export interface WaitForOptions {
+  timeout: number
+  delay?: number
+  message?: string
+}
+
+export async function waitFor (fn: () => Promise<boolean>, options: WaitForOptions): Promise<void> {
+  const delay = options.delay ?? 100
+  const timeoutAt = Date.now() + options.timeout
+
+  while (true) {
+    const result = await fn()
+
+    if (result) {
+      return
+    }
+
+    await new Promise<void>((resolve) => {
+      setTimeout(() => { resolve() }, delay)
+    })
+
+    if (Date.now() > timeoutAt) {
+      throw new Error(options.message ?? 'WaitFor timed out')
+    }
+  }
+}

package/src/helia-blockstore.spec.ts

@@ -0,0 +1,59 @@
+/* eslint-env mocha */
+
+import { peerIdFromString } from '@libp2p/peer-id'
+import { expect } from 'aegir/chai'
+import toBuffer from 'it-to-buffer'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('helia - blockstore', () => {
+  let helia: HeliaLibp2p
+  let kubo: Controller
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    kubo = await createKuboNode()
+
+    // connect the two nodes
+    await helia.libp2p.peerStore.merge(peerIdFromString(kubo.peer.id.toString()), {
+      multiaddrs: kubo.peer.addresses
+    })
+    await helia.libp2p.dial(peerIdFromString(kubo.peer.id.toString()))
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should be able to send a block', async () => {
+    const input = Uint8Array.from([0, 1, 2, 3, 4])
+    const digest = await sha256.digest(input)
+    const cid = CID.createV1(raw.code, digest)
+    await helia.blockstore.put(cid, input)
+    const output = await toBuffer(kubo.api.cat(cid))
+
+    expect(output).to.equalBytes(input)
+  })
+
+  it('should be able to receive a block', async () => {
+    const input = Uint8Array.from([0, 1, 2, 3, 4])
+    const { cid } = await kubo.api.add({ content: input }, {
+      cidVersion: 1,
+      rawLeaves: true
+    })
+    const output = await helia.blockstore.get(CID.parse(cid.toString()))
+
+    expect(output).to.equalBytes(input)
+  })
+})

package/src/helia-hashes.spec.ts

@@ -0,0 +1,61 @@
+/* eslint-env mocha */
+
+import { peerIdFromString } from '@libp2p/peer-id'
+import { sha3512 } from '@multiformats/sha3'
+import { expect } from 'aegir/chai'
+import toBuffer from 'it-to-buffer'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('helia - hashes', () => {
+  let helia: HeliaLibp2p
+  let kubo: Controller
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    kubo = await createKuboNode()
+
+    // connect the two nodes
+    await helia.libp2p.peerStore.merge(peerIdFromString(kubo.peer.id.toString()), {
+      multiaddrs: kubo.peer.addresses
+    })
+    await helia.libp2p.dial(peerIdFromString(kubo.peer.id.toString()))
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should be able to send a block with a non-default hash', async () => {
+    const input = Uint8Array.from([0, 1, 2, 3, 4])
+    const digest = await sha3512.digest(input)
+    const cid = CID.createV1(raw.code, digest)
+    await helia.blockstore.put(cid, input)
+    const output = await toBuffer(kubo.api.cat(cid))
+
+    expect(output).to.equalBytes(input)
+  })
+
+  it('should be able to receive a block with a non-default hash', async () => {
+    const input = Uint8Array.from([0, 1, 2, 3, 4])
+    const { cid } = await kubo.api.add({ content: input }, {
+      cidVersion: 1,
+      rawLeaves: true,
+      hashAlg: 'sha3-512'
+    })
+    expect(cid.multihash.code).to.equal(sha3512.code)
+    const output = await helia.blockstore.get(CID.parse(cid.toString()))
+
+    expect(output).to.equalBytes(input)
+  })
+})

package/src/helia-pins.spec.ts

@@ -0,0 +1,64 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import all from 'it-all'
+import drain from 'it-drain'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('helia - pins', () => {
+  let helia: HeliaLibp2p
+  let kubo: Controller
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    kubo = await createKuboNode()
+
+    // connect the two nodes
+    await helia.libp2p.dial(kubo.peer.addresses)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('pinning on kubo should pull from helia', async () => {
+    const input = Uint8Array.from([0, 1, 2, 3, 4])
+    const digest = await sha256.digest(input)
+    const cid = CID.createV1(raw.code, digest)
+
+    expect((await all(kubo.api.refs.local())).map(r => r.ref)).to.not.include(cid.toString())
+
+    await helia.blockstore.put(cid, input)
+
+    const pinned = await kubo.api.pin.add(cid)
+    expect(pinned.toString()).to.equal(cid.toString())
+
+    expect((await all(kubo.api.refs.local())).map(r => r.ref)).to.include(cid.toString())
+  })
+
+  it('pinning on helia should pull from kubo', async () => {
+    const input = Uint8Array.from([0, 1, 2, 3, 4])
+    const { cid } = await kubo.api.add({ content: input }, {
+      cidVersion: 1,
+      rawLeaves: true
+    })
+
+    await expect(helia.blockstore.has(CID.parse(cid.toString()))).to.eventually.be.false()
+
+    await drain(helia.pins.add(CID.parse(cid.toString())))
+
+    await expect(helia.blockstore.has(CID.parse(cid.toString()))).to.eventually.be.true()
+  })
+})

package/src/index.ts

@@ -1 +1,14 @@
+/**
+ * @packageDocumentation
+ *
+ * Runs interop tests between Helia and Kubo.
+ *
+ * @example Testing a new Kubo release
+ *
+ * ```console
+ * $ npm i @helia/interop
+ * $ KUBO_BINARY=/path/to/kubo helia-interop
+ * ```
+ */
+
 export {}

package/src/ipns-http.spec.ts

@@ -0,0 +1,68 @@
+/* eslint-env mocha */
+
+import { ipns } from '@helia/ipns'
+import { delegatedHTTPRouting } from '@helia/routers'
+import { peerIdFromString } from '@libp2p/peer-id'
+import { expect } from 'aegir/chai'
+import { isNode } from 'wherearewe'
+import { createHeliaHTTP } from './fixtures/create-helia-http.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { Helia } from '@helia/interface'
+import type { IPNS } from '@helia/ipns'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('@helia/ipns - http', () => {
+  let helia: Helia
+  let kubo: Controller
+  let name: IPNS
+
+  /**
+   * Ensure that for the CID we are going to publish, the resolver has a peer ID that
+   * is KAD-closer to the routing key so we can predict the the resolver will receive
+   * the DHT record containing the IPNS record
+   */
+  beforeEach(async () => {
+    kubo = await createKuboNode()
+    helia = await createHeliaHTTP({
+      routers: [
+        delegatedHTTPRouting('http://127.0.0.1:8180')
+      ]
+    })
+    name = ipns(helia)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should publish on kubo and resolve on helia', async function () {
+    if (!isNode) {
+      // https://github.com/protocol/bifrost-community/issues/4#issuecomment-1898417008
+      return this.skip()
+    }
+
+    const keyName = 'my-ipns-key'
+    const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4]))
+
+    await kubo.api.key.gen(keyName, {
+      // @ts-expect-error the types say upper-case E, Kubo errors unless it's a
+      // lower case e
+      type: 'ed25519'
+    })
+
+    const res = await kubo.api.name.publish(cid, {
+      key: keyName
+    })
+
+    const key = peerIdFromString(res.name)
+
+    const resolvedCid = await name.resolve(key)
+    expect(resolvedCid.toString()).to.equal(cid.toString())
+  })
+})

package/src/ipns-pubsub.spec.ts

@@ -0,0 +1,187 @@
+/* eslint-env mocha */
+/* eslint max-nested-callbacks: ["error", 5] */
+
+import { gossipsub } from '@chainsafe/libp2p-gossipsub'
+import { ipns } from '@helia/ipns'
+import { pubsub } from '@helia/ipns/routing'
+import { peerIdFromKeys } from '@libp2p/peer-id'
+import { expect } from 'aegir/chai'
+import last from 'it-last'
+import { base36 } from 'multiformats/bases/base36'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { identity } from 'multiformats/hashes/identity'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { connect } from './fixtures/connect.js'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import { keyTypes } from './fixtures/key-types.js'
+import { waitFor } from './fixtures/wait-for.js'
+import type { IPNS } from '@helia/ipns'
+import type { Libp2p, PubSub } from '@libp2p/interface'
+import type { Keychain } from '@libp2p/keychain'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+const LIBP2P_KEY_CODEC = 0x72
+
+// skip RSA tests because we need the DHT enabled to find the public key
+// component of the keypair, but that means we can't test pubsub
+// resolution because Kubo will use the DHT as well
+keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => {
+  describe(`@helia/ipns - pubsub routing with ${keyType} keys`, () => {
+    let helia: HeliaLibp2p<Libp2p<{ pubsub: PubSub, keychain: Keychain }>>
+    let kubo: Controller
+    let name: IPNS
+
+    beforeEach(async () => {
+      helia = await createHeliaNode({
+        services: {
+          pubsub: gossipsub()
+        }
+      })
+      kubo = await createKuboNode()
+
+      // connect the two nodes
+      await connect(helia, kubo, '/meshsub/1.1.0')
+
+      name = ipns(helia, {
+        routers: [
+          pubsub(helia)
+        ]
+      })
+    })
+
+    afterEach(async () => {
+      if (helia != null) {
+        await helia.stop()
+      }
+
+      if (kubo != null) {
+        await kubo.stop()
+      }
+    })
+
+    it('should publish on helia and resolve on kubo', async () => {
+      const input = Uint8Array.from([0, 1, 2, 3, 4])
+      const digest = await sha256.digest(input)
+      const cid = CID.createV1(raw.code, digest)
+
+      const keyName = 'my-ipns-key'
+      await helia.libp2p.services.keychain.createKey(keyName, keyType)
+      const peerId = await helia.libp2p.services.keychain.exportPeerId(keyName)
+
+      if (peerId.publicKey == null) {
+        throw new Error('No public key present')
+      }
+
+      // first publish should fail because kubo isn't subscribed to key update channel
+      await expect(name.publish(peerId, cid)).to.eventually.be.rejected()
+        .with.property('message', 'PublishError.InsufficientPeers')
+
+      // should fail to resolve the first time as kubo was not subscribed to the pubsub channel
+      // @ts-expect-error kubo deps are out of date
+      await expect(last(kubo.api.name.resolve(peerId, {
+        timeout: 100
+      }))).to.eventually.be.undefined()
+
+      // magic pubsub subscription name
+      const subscriptionName = `/ipns/${CID.createV1(LIBP2P_KEY_CODEC, identity.digest(peerId.publicKey)).toString(base36)}`
+
+      // wait for kubo to be subscribed to updates
+      await waitFor(async () => {
+        const subs = await kubo.api.name.pubsub.subs()
+
+        return subs.includes(subscriptionName)
+      }, {
+        timeout: 30000
+      })
+
+      // publish should now succeed
+      await name.publish(peerId, cid)
+
+      // kubo should now be able to resolve IPNS name
+      // @ts-expect-error kubo deps are out of date
+      const resolved = await last(kubo.api.name.resolve(peerId, {
+        timeout: 100
+      }))
+
+      expect(resolved).to.equal(`/ipfs/${cid.toString()}`)
+    })
+
+    it('should publish on kubo and resolve on helia', async function () {
+      if (keyType === 'secp256k1') {
+        // Kubo cannot generate secp256k1 keys
+        return this.skip()
+      }
+
+      const keyName = 'my-ipns-key'
+      const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4]))
+      const result = await kubo.api.key.gen(keyName, {
+        // @ts-expect-error kubo needs this in lower case
+        type: keyType.toLowerCase()
+      })
+
+      // the generated id is libp2p-key CID with the public key as an identity multihash
+      const peerCid = CID.parse(result.id, base36)
+      const peerId = await peerIdFromKeys(peerCid.multihash.digest)
+
+      // first call to pubsub resolver should fail but we should now be subscribed for updates
+      await expect(name.resolve(peerId)).to.eventually.be.rejected()
+
+      // actual pubsub subscription name
+      const subscriptionName = `/record/${uint8ArrayToString(uint8ArrayConcat([
+        uint8ArrayFromString('/ipns/'),
+        peerId.toBytes()
+      ]), 'base64url')}`
+
+      // wait for helia to be subscribed to the topic for record updates
+      await waitFor(async () => {
+        return helia.libp2p.services.pubsub.getTopics().includes(subscriptionName)
+      }, {
+        timeout: 30000,
+        message: 'Helia did not register for record updates'
+      })
+
+      // wait for kubo to see that helia is subscribed to the topic for record updates
+      await waitFor(async () => {
+        const peers = await kubo.api.pubsub.peers(subscriptionName)
+
+        return peers.map(p => p.toString()).includes(helia.libp2p.peerId.toString())
+      }, {
+        timeout: 30000,
+        message: 'Kubo did not see that Helia was registered for record updates'
+      })
+
+      // now publish, this should cause a pubsub message on the topic for record updates
+      await kubo.api.name.publish(cid, {
+        key: keyName
+      })
+
+      let resolvedCid: CID | undefined
+
+      // we should get an update eventually
+      await waitFor(async () => {
+        try {
+          resolvedCid = await name.resolve(peerId)
+
+          return true
+        } catch {
+          return false
+        }
+      }, {
+        timeout: 10000,
+        message: 'Helia could not resolve the IPNS record'
+      })
+
+      if (resolvedCid == null) {
+        throw new Error('Failed to resolve CID')
+      }
+
+      expect(resolvedCid.toString()).to.equal(cid.toString())
+    })
+  })
+})

package/src/ipns.spec.ts

@@ -0,0 +1,183 @@
+/* eslint-env mocha */
+
+import { ipns } from '@helia/ipns'
+import { peerIdFromString } from '@libp2p/peer-id'
+import { createEd25519PeerId, createRSAPeerId, createSecp256k1PeerId } from '@libp2p/peer-id-factory'
+import { expect } from 'aegir/chai'
+import last from 'it-last'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { isElectronMain } from 'wherearewe'
+import { connect } from './fixtures/connect.js'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import { sortClosestPeers } from './fixtures/create-peer-ids.js'
+import { keyTypes } from './fixtures/key-types.js'
+import { waitFor } from './fixtures/wait-for.js'
+import type { IPNS } from '@helia/ipns'
+import type { PeerId } from '@libp2p/interface'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+keyTypes.forEach(type => {
+  describe(`@helia/ipns - default routing with ${type} keys`, () => {
+    let helia: HeliaLibp2p
+    let kubo: Controller
+    let name: IPNS
+
+    // the CID we are going to publish
+    let value: CID
+
+    // the public key we will use to publish the value
+    let key: PeerId
+
+    /**
+     * Ensure that for the CID we are going to publish, the resolver has a peer ID that
+     * is KAD-closer to the routing key so we can predict the the resolver will receive
+     * the DHT record containing the IPNS record
+     */
+    async function createNodes (resolver: 'kubo' | 'helia'): Promise<void> {
+      const input = Uint8Array.from([0, 1, 2, 3, 4])
+      const digest = await sha256.digest(input)
+      value = CID.createV1(raw.code, digest)
+
+      helia = await createHeliaNode()
+      kubo = await createKuboNode()
+
+      // find a PeerId that is KAD-closer to the resolver than the publisher when used as an IPNS key
+      while (true) {
+        if (type === 'Ed25519') {
+          key = await createEd25519PeerId()
+        } else if (type === 'secp256k1') {
+          key = await createSecp256k1PeerId()
+        } else {
+          key = await createRSAPeerId()
+        }
+
+        const routingKey = uint8ArrayConcat([
+          uint8ArrayFromString('/ipns/'),
+          key.toBytes()
+        ])
+
+        const [closest] = await sortClosestPeers(routingKey, [
+          helia.libp2p.peerId,
+          peerIdFromString(kubo.peer.id.toString())
+        ])
+
+        if (resolver === 'kubo' && closest.equals(peerIdFromString(kubo.peer.id.toString()))) {
+          break
+        }
+
+        if (resolver === 'helia' && closest.equals(helia.libp2p.peerId)) {
+          break
+        }
+      }
+
+      // connect the two nodes over the KAD-DHT protocol, this should ensure
+      // both nodes have each other in their KAD buckets
+      await connect(helia, kubo, '/ipfs/lan/kad/1.0.0')
+
+      await waitFor(async () => {
+        let found = false
+
+        for await (const event of helia.libp2p.services.dht.findPeer(peerIdFromString(kubo.peer.id.toString()))) {
+          if (event.name === 'FINAL_PEER') {
+            found = true
+          }
+        }
+
+        return found
+      }, {
+        timeout: 30000,
+        delay: 1000,
+        message: 'Helia could not find Kubo on the DHT'
+      })
+
+      await waitFor(async () => {
+        let found = false
+
+        // @ts-expect-error kubo deps are out of date
+        for await (const event of kubo.api.dht.findPeer(helia.libp2p.peerId)) {
+          if (event.name === 'FINAL_PEER') {
+            found = true
+          }
+        }
+
+        return found
+      }, {
+        timeout: 30000,
+        delay: 1000,
+        message: 'Kubo could not find Helia on the DHT'
+      })
+
+      name = ipns(helia)
+    }
+
+    afterEach(async () => {
+      if (helia != null) {
+        await helia.stop()
+      }
+
+      if (kubo != null) {
+        await kubo.stop()
+      }
+    })
+
+    it(`should publish on helia and resolve on kubo using a ${type} key`, async () => {
+      await createNodes('kubo')
+
+      const keyName = 'my-ipns-key'
+      await helia.libp2p.services.keychain.importPeer(keyName, key)
+
+      await name.publish(key, value)
+
+      const resolved = await last(kubo.api.name.resolve(key.toString()))
+
+      if (resolved == null) {
+        throw new Error('kubo failed to resolve name')
+      }
+
+      expect(resolved).to.equal(`/ipfs/${value.toString()}`)
+    })
+
+    it('should publish on kubo and resolve on helia', async function () {
+      if (isElectronMain) {
+        // electron main does not have fetch, FormData or Blob APIs
+        // can revisit when kubo-rpc-client supports the key.import API
+        return this.skip()
+      }
+
+      if (type === 'secp256k1') {
+        // Kubo cannot import secp256k1 keys
+        return this.skip()
+      }
+
+      await createNodes('helia')
+
+      const keyName = 'my-ipns-key'
+      const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4]))
+
+      // ensure the key is in the kubo keychain so we can use it to publish the IPNS record
+      const body = new FormData()
+      body.append('key', new Blob([key.privateKey ?? new Uint8Array(0)]))
+
+      // can't use the kubo-rpc-api for this call yet
+      const response = await fetch(`http://${kubo.api.apiHost}:${kubo.api.apiPort}/api/v0/key/import?arg=${keyName}`, {
+        method: 'POST',
+        body
+      })
+
+      expect(response).to.have.property('status', 200)
+
+      await kubo.api.name.publish(cid, {
+        key: keyName
+      })
+
+      const resolvedCid = await name.resolve(key)
+      expect(resolvedCid.toString()).to.equal(cid.toString())
+    })
+  })
+})