@helia/interop 3.0.1 → 4.0.0-e554493

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/interop",
-  "version": "3.0.1",
+  "version": "4.0.0-e554493",
   "description": "Interop tests for Helia",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia/tree/main/packages/interop#readme",
@@ -11,16 +11,24 @@
   "bugs": {
     "url": "https://github.com/ipfs/helia/issues"
   },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
   "keywords": [
     "IPFS"
   ],
+  "bin": {
+    "helia-interop": "./dist/src/bin.js"
+  },
   "type": "module",
   "types": "./dist/src/index.d.ts",
   "files": [
     "src",
     "dist",
     "!dist/test",
-    "!**/*.tsbuildinfo"
+    "!**/*.tsbuildinfo",
+    ".aegir.js"
   ],
   "exports": {
     ".": {
@@ -48,28 +56,31 @@
     "test:node": "aegir test -t node --cov",
     "test:electron-main": "aegir test -t electron-main"
   },
-  "devDependencies": {
-    "@helia/block-brokers": "~1.0.0",
-    "@helia/car": "^2.0.1",
-    "@helia/dag-cbor": "^2.0.1",
-    "@helia/dag-json": "^2.0.1",
-    "@helia/interface": "^3.0.1",
-    "@helia/ipns": "^4.0.0",
-    "@helia/json": "^2.0.1",
-    "@helia/mfs": "^2.0.1",
-    "@helia/strings": "^2.0.1",
-    "@helia/unixfs": "^2.0.1",
+  "dependencies": {
+    "@chainsafe/libp2p-gossipsub": "^11.1.0",
+    "@helia/block-brokers": "2.0.0-e554493",
+    "@helia/car": "3.0.0-e554493",
+    "@helia/dag-cbor": "3.0.0-e554493",
+    "@helia/dag-json": "3.0.0-e554493",
+    "@helia/http": "1.0.0-e554493",
+    "@helia/interface": "4.0.0-e554493",
+    "@helia/ipns": "5.0.0-e554493",
+    "@helia/json": "3.0.0-e554493",
+    "@helia/mfs": "3.0.0-e554493",
+    "@helia/routers": "1.0.0-e554493",
+    "@helia/strings": "3.0.0-e554493",
+    "@helia/unixfs": "3.0.0-e554493",
     "@ipld/car": "^5.2.5",
     "@ipld/dag-cbor": "^9.0.7",
     "@libp2p/interface": "^1.1.1",
     "@libp2p/kad-dht": "^12.0.2",
-    "@libp2p/keychain": "^4.0.5",
-    "@libp2p/peer-id": "^4.0.3",
+    "@libp2p/keychain": "^4.0.7",
+    "@libp2p/peer-id": "^4.0.5",
     "@libp2p/peer-id-factory": "^4.0.3",
-    "@libp2p/websockets": "^8.0.1",
+    "@libp2p/websockets": "^8.0.10",
     "@multiformats/sha3": "^3.0.0",
     "aegir": "^42.1.0",
-    "helia": "^3.0.1",
+    "helia": "4.0.0-e554493",
     "ipfs-core-types": "^0.14.1",
     "ipfs-unixfs-importer": "^15.2.3",
     "ipfsd-ctl": "^13.0.0",
@@ -78,16 +89,18 @@
     "it-last": "^3.0.4",
     "it-map": "^3.0.5",
     "it-to-buffer": "^4.0.2",
-    "kubo": "^0.25.0",
+    "kubo": "^0.26.0",
     "kubo-rpc-client": "^3.0.1",
+    "libp2p": "^1.2.0",
     "multiformats": "^13.0.0",
     "p-defer": "^4.0.0",
     "uint8arrays": "^5.0.1",
     "wherearewe": "^2.0.1"
   },
   "browser": {
-    "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js",
-    "./dist/test/fixtures/create-kubo.js": "./dist/test/fixtures/create-kubo.browser.js",
+    "./dist/src/fixtures/create-helia.js": "./dist/src/fixtures/create-helia.browser.js",
+    "./dist/src/fixtures/create-kubo.js": "./dist/src/fixtures/create-kubo.browser.js",
+    "./dist/src/bin.js": "./dist/src/index.js",
     "kubo": false
   }
 }

package/src/bin.ts

@@ -0,0 +1,25 @@
+#! /usr/bin/env node
+/* eslint-disable no-console */
+
+import { spawn } from 'node:child_process'
+import { dirname, resolve } from 'path'
+import { fileURLToPath } from 'url'
+
+// aegir should be run from `node_modules/@helia/interop`
+const cwd = resolve(dirname(fileURLToPath(import.meta.url)), '../../')
+
+const test = spawn('npx', ['aegir', 'test'], {
+  cwd
+})
+
+test.stdout.on('data', (data) => {
+  process.stdout.write(data)
+})
+
+test.stderr.on('data', (data) => {
+  process.stderr.write(data)
+})
+
+test.on('close', (code) => {
+  process.exit(code ?? 0)
+})

package/src/car.spec.ts

@@ -0,0 +1,102 @@
+/* eslint-env mocha */
+
+import { car } from '@helia/car'
+import { type UnixFS, unixfs } from '@helia/unixfs'
+import { CarReader } from '@ipld/car'
+import { expect } from 'aegir/chai'
+import drain from 'it-drain'
+import toBuffer from 'it-to-buffer'
+import { CID } from 'multiformats/cid'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import { memoryCarWriter } from './fixtures/memory-car.js'
+import type { Car } from '@helia/car'
+import type { HeliaLibp2p } from 'helia'
+import type { FileCandidate } from 'ipfs-unixfs-importer'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('@helia/car', () => {
+  let helia: HeliaLibp2p
+  let c: Car
+  let u: UnixFS
+  let kubo: Controller
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    c = car(helia)
+    u = unixfs(helia)
+    kubo = await createKuboNode()
+
+    // connect helia to kubo
+    await helia.libp2p.dial(kubo.peer.addresses)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should export a car from Helia, import and read the contents from kubo', async () => {
+    const chunkSize = 1024 * 1024
+    const size = chunkSize * 10
+    const input: Uint8Array[] = []
+
+    const candidate: FileCandidate = {
+      content: (async function * () {
+        for (let i = 0; i < size; i += chunkSize) {
+          const buf = new Uint8Array(chunkSize)
+          input.push(buf)
+
+          yield buf
+        }
+      }())
+    }
+
+    const cid = await u.addFile(candidate)
+    const writer = memoryCarWriter(cid)
+    await c.export(cid, writer)
+
+    const buf = await writer.bytes()
+
+    await drain(kubo.api.dag.import([buf]))
+
+    const output: Uint8Array[] = []
+
+    for await (const b of kubo.api.cat(cid)) {
+      output.push(b)
+    }
+
+    expect(toBuffer(output)).to.equalBytes(toBuffer(input))
+  })
+
+  it('should export a car from kubo, import and read the contents from Helia', async () => {
+    const chunkSize = 1024 * 1024
+    const size = chunkSize * 10
+    const input: Uint8Array[] = []
+
+    const candidate: FileCandidate = {
+      content: (async function * () {
+        for (let i = 0; i < size; i += chunkSize) {
+          const buf = new Uint8Array(chunkSize)
+          input.push(buf)
+
+          yield buf
+        }
+      }())
+    }
+
+    const { cid } = await kubo.api.add(candidate.content)
+    const bytes = await toBuffer(kubo.api.dag.export(cid))
+
+    const reader = await CarReader.fromBytes(bytes)
+
+    await c.import(reader)
+
+    expect(await toBuffer(u.cat(CID.parse(cid.toString())))).to.equalBytes(toBuffer(input))
+  })
+})

package/src/dag-cbor.spec.ts

@@ -0,0 +1,65 @@
+/* eslint-env mocha */
+
+import { dagCbor, type DAGCBOR, type AddOptions } from '@helia/dag-cbor'
+import * as codec from '@ipld/dag-cbor'
+import { expect } from 'aegir/chai'
+import { CID } from 'multiformats/cid'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('@helia/dag-cbor', () => {
+  let helia: HeliaLibp2p
+  let d: DAGCBOR
+  let kubo: Controller
+
+  async function expectSameCid (data: () => any, heliaOpts: Partial<AddOptions> = {}, kuboOpts: KuboAddOptions = {}): Promise<void> {
+    const heliaCid = await d.add(data(), heliaOpts)
+    const kuboCid = await kubo.api.dag.put(data(), kuboOpts)
+
+    expect(heliaCid.toString()).to.equal(kuboCid.toString())
+  }
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    d = dagCbor(helia)
+    kubo = await createKuboNode()
+
+    await helia.libp2p.dial((await (kubo.api.id())).addresses)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should create the same CID for a string', async () => {
+    const candidate = (): any => ({ hello: 'world' })
+
+    await expectSameCid(candidate)
+  })
+
+  it('should add to helia and fetch from kubo', async () => {
+    const input = { hello: 'world' }
+    const cid = await d.add(input)
+    const block = await kubo.api.block.get(cid)
+    const output = codec.decode(block)
+
+    expect(output).to.deep.equal(input)
+  })
+
+  it('should add to kubo and fetch from helia', async () => {
+    const input = { hello: 'world' }
+    const cid = await kubo.api.block.put(codec.encode(input))
+    const output = await d.get(CID.parse(cid.toString()))
+
+    expect(output).to.deep.equal(input)
+  })
+})

package/src/dag-json.spec.ts

@@ -0,0 +1,65 @@
+/* eslint-env mocha */
+
+import { dagJson, type DAGJSON, type AddOptions } from '@helia/dag-json'
+import { expect } from 'aegir/chai'
+import { CID } from 'multiformats/cid'
+import * as codec from 'multiformats/codecs/json'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('@helia/dag-json', () => {
+  let helia: HeliaLibp2p
+  let d: DAGJSON
+  let kubo: Controller
+
+  async function expectSameCid (data: () => any, heliaOpts: Partial<AddOptions> = {}, kuboOpts: KuboAddOptions = { format: 'dag-json' }): Promise<void> {
+    const heliaCid = await d.add(data(), heliaOpts)
+    const kuboCid = await kubo.api.block.put(codec.encode(data()), kuboOpts)
+
+    expect(heliaCid.toString()).to.equal(kuboCid.toString())
+  }
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    d = dagJson(helia)
+    kubo = await createKuboNode()
+
+    await helia.libp2p.dial((await (kubo.api.id())).addresses)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should create the same CID for a string', async () => {
+    const candidate = (): any => ({ hello: 'world' })
+
+    await expectSameCid(candidate)
+  })
+
+  it('should add to helia and fetch from kubo', async () => {
+    const input = { hello: 'world' }
+    const cid = await d.add(input)
+    const block = await kubo.api.block.get(cid)
+    const output = codec.decode(block)
+
+    expect(output).to.deep.equal(input)
+  })
+
+  it('should add to kubo and fetch from helia', async () => {
+    const input = { hello: 'world' }
+    const cid = await kubo.api.block.put(codec.encode(input))
+    const output = await d.get(CID.parse(cid.toString()))
+
+    expect(output).to.deep.equal(input)
+  })
+})

package/src/fixtures/connect.ts

@@ -0,0 +1,19 @@
+import { expect } from 'aegir/chai'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+/**
+ * Connect the two nodes by dialing a protocol stream
+ */
+export async function connect (helia: HeliaLibp2p<any>, kubo: Controller, protocol: string): Promise<void> {
+  let connected = false
+  for (const addr of kubo.peer.addresses) {
+    try {
+      await helia.libp2p.dialProtocol(addr, protocol)
+      connected = true
+      break
+    } catch { }
+  }
+
+  expect(connected).to.be.true('could not connect Helia to Kubo')
+}

package/src/fixtures/create-helia-http.ts

@@ -0,0 +1,8 @@
+import { createHeliaHTTP as createHelia, type HeliaHTTPInit } from '@helia/http'
+import type { Helia } from '@helia/interface'
+
+export async function createHeliaHTTP (init: Partial<HeliaHTTPInit> = {}): Promise<Helia> {
+  return createHelia({
+    ...init
+  })
+}

package/src/fixtures/create-helia.browser.ts

@@ -0,0 +1,68 @@
+import { bitswap } from '@helia/block-brokers'
+import { ipnsValidator, ipnsSelector } from '@helia/ipns'
+import { kadDHT, removePublicAddressesMapper } from '@libp2p/kad-dht'
+import { webSockets } from '@libp2p/websockets'
+import { all } from '@libp2p/websockets/filters'
+import { sha3512 } from '@multiformats/sha3'
+import { createHelia, libp2pDefaults } from 'helia'
+import type { Libp2p } from '@libp2p/interface'
+import type { DefaultLibp2pServices, HeliaLibp2p } from 'helia'
+import type { Libp2pOptions } from 'libp2p'
+
+export async function createHeliaNode (): Promise<HeliaLibp2p<Libp2p<DefaultLibp2pServices>>>
+export async function createHeliaNode <Services extends Record<string, unknown>> (libp2pOptions: Libp2pOptions<Services>): Promise<HeliaLibp2p<Libp2p<Services & DefaultLibp2pServices>>>
+export async function createHeliaNode (libp2pOptions?: Libp2pOptions): Promise<HeliaLibp2p<Libp2p<DefaultLibp2pServices>>> {
+  const defaults = libp2pDefaults()
+
+  // allow dialing insecure WebSockets
+  defaults.transports?.pop()
+  defaults.transports = [
+    ...(defaults.transports ?? []),
+    webSockets({
+      filter: all
+    })
+  ]
+
+  // allow dialing loopback
+  defaults.connectionGater = {
+    denyDialMultiaddr: () => false
+  }
+
+  // use LAN DHT
+  defaults.services = {
+    ...(defaults.services ?? {}),
+    ...(libp2pOptions?.services ?? {}),
+    dht: kadDHT({
+      validators: {
+        ipns: ipnsValidator
+      },
+      selectors: {
+        ipns: ipnsSelector
+      },
+      // skips waiting for the initial self-query to find peers
+      allowQueryWithZeroPeers: true,
+
+      protocol: '/ipfs/lan/kad/1.0.0',
+      peerInfoMapper: removePublicAddressesMapper,
+      clientMode: false
+    })
+  }
+
+  // remove bootstrappers
+  defaults.peerDiscovery = []
+
+  // remove services that are not used in tests
+  delete defaults.services.autoNAT
+  delete defaults.services.dcutr
+  delete defaults.services.delegatedRouting
+
+  return createHelia<Libp2p<DefaultLibp2pServices>>({
+    blockBrokers: [
+      bitswap()
+    ],
+    libp2p: defaults,
+    hashers: [
+      sha3512
+    ]
+  })
+}

package/src/fixtures/create-helia.ts

@@ -0,0 +1,55 @@
+import { bitswap } from '@helia/block-brokers'
+import { ipnsValidator, ipnsSelector } from '@helia/ipns'
+import { kadDHT, removePublicAddressesMapper } from '@libp2p/kad-dht'
+import { sha3512 } from '@multiformats/sha3'
+import { createHelia, libp2pDefaults } from 'helia'
+import type { Libp2p } from '@libp2p/interface'
+import type { DefaultLibp2pServices, HeliaLibp2p } from 'helia'
+import type { Libp2pOptions } from 'libp2p'
+
+export async function createHeliaNode (): Promise<HeliaLibp2p<Libp2p<DefaultLibp2pServices>>>
+export async function createHeliaNode <Services extends Record<string, unknown>> (libp2pOptions: Libp2pOptions<Services>): Promise<HeliaLibp2p<Libp2p<Services & DefaultLibp2pServices>>>
+export async function createHeliaNode (libp2pOptions?: Libp2pOptions): Promise<HeliaLibp2p<Libp2p<DefaultLibp2pServices>>> {
+  const defaults = libp2pDefaults()
+  defaults.addresses = {
+    listen: [
+      '/ip4/0.0.0.0/tcp/0'
+    ]
+  }
+  defaults.services = {
+    ...(defaults.services ?? {}),
+    ...(libp2pOptions?.services ?? {}),
+    dht: kadDHT({
+      validators: {
+        ipns: ipnsValidator
+      },
+      selectors: {
+        ipns: ipnsSelector
+      },
+      // skips waiting for the initial self-query to find peers
+      allowQueryWithZeroPeers: true,
+
+      protocol: '/ipfs/lan/kad/1.0.0',
+      peerInfoMapper: removePublicAddressesMapper,
+      clientMode: false
+    })
+  }
+
+  // remove bootstrappers, mdns, etc
+  defaults.peerDiscovery = []
+
+  // remove services that are not used in tests
+  delete defaults.services.autoNAT
+  delete defaults.services.dcutr
+  delete defaults.services.delegatedRouting
+
+  return createHelia<Libp2p<DefaultLibp2pServices>>({
+    blockBrokers: [
+      bitswap()
+    ],
+    libp2p: defaults,
+    hashers: [
+      sha3512
+    ]
+  })
+}

package/src/fixtures/create-kubo.browser.ts

@@ -0,0 +1,30 @@
+import { type Controller, createController } from 'ipfsd-ctl'
+import * as kuboRpcClient from 'kubo-rpc-client'
+
+export async function createKuboNode (): Promise<Controller> {
+  return createController({
+    kuboRpcModule: kuboRpcClient,
+    test: true,
+    endpoint: process.env.IPFSD_SERVER,
+    ipfsOptions: {
+      config: {
+        Addresses: {
+          Swarm: [
+            '/ip4/0.0.0.0/tcp/0',
+            '/ip4/0.0.0.0/tcp/0/ws'
+          ],
+          Gateway: '/ip4/127.0.0.1/tcp/8180'
+        },
+        Gateway: {
+          NoFetch: true,
+          ExposeRoutingAPI: true,
+          HTTPHeaders: {
+            'Access-Control-Allow-Origin': ['*'],
+            'Access-Control-Allow-Methods': ['GET', 'POST', 'PUT', 'OPTIONS']
+          }
+        }
+      }
+    },
+    args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub']
+  })
+}

package/src/fixtures/create-kubo.ts

@@ -0,0 +1,32 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */
+import { createController, type Controller } from 'ipfsd-ctl'
+import { path as kuboPath } from 'kubo'
+import * as kuboRpcClient from 'kubo-rpc-client'
+
+export async function createKuboNode (): Promise<Controller> {
+  return createController({
+    kuboRpcModule: kuboRpcClient,
+    ipfsBin: kuboPath(),
+    test: true,
+    ipfsOptions: {
+      config: {
+        Addresses: {
+          Swarm: [
+            '/ip4/0.0.0.0/tcp/4001',
+            '/ip4/0.0.0.0/tcp/4002/ws'
+          ],
+          Gateway: '/ip4/127.0.0.1/tcp/8180'
+        },
+        Gateway: {
+          NoFetch: true,
+          ExposeRoutingAPI: true,
+          HTTPHeaders: {
+            'Access-Control-Allow-Origin': ['*'],
+            'Access-Control-Allow-Methods': ['GET', 'POST', 'PUT', 'OPTIONS']
+          }
+        }
+      }
+    },
+    args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub']
+  })
+}

package/src/fixtures/create-peer-ids.ts

@@ -0,0 +1,46 @@
+import all from 'it-all'
+import map from 'it-map'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { compare as uint8ArrayCompare } from 'uint8arrays/compare'
+import { xor as uint8ArrayXor } from 'uint8arrays/xor'
+import type { PeerId } from '@libp2p/interface'
+
+/**
+ * Sort peers by distance to the KadID of the passed buffer
+ */
+export async function sortClosestPeers (buf: Uint8Array, peers: PeerId[]): Promise<PeerId[]> {
+  const kadId = await convertBuffer(buf)
+
+  const distances = await all(
+    map(peers, async (peer) => {
+      const id = await convertPeerId(peer)
+
+      return {
+        peer,
+        distance: uint8ArrayXor(id, kadId)
+      }
+    })
+  )
+
+  return distances
+    .sort((a, b) => {
+      return uint8ArrayCompare(a.distance, b.distance)
+    })
+    .map((d) => d.peer)
+}
+
+/**
+ * Creates a DHT ID by hashing a Peer ID
+ */
+export async function convertPeerId (peerId: PeerId): Promise<Uint8Array> {
+  return convertBuffer(peerId.toBytes())
+}
+
+/**
+ * Creates a DHT ID by hashing a given Uint8Array
+ */
+export async function convertBuffer (buf: Uint8Array): Promise<Uint8Array> {
+  const multihash = await sha256.digest(buf)
+
+  return multihash.digest
+}

package/src/fixtures/key-types.ts

@@ -0,0 +1,7 @@
+import type { PeerIdType } from '@libp2p/interface'
+
+export const keyTypes: PeerIdType[] = [
+  'Ed25519',
+  'secp256k1',
+  'RSA'
+]

package/src/fixtures/memory-car.ts

@@ -0,0 +1,33 @@
+import { CarWriter } from '@ipld/car'
+import toBuffer from 'it-to-buffer'
+import defer from 'p-defer'
+import type { CID } from 'multiformats/cid'
+
+export interface MemoryCar extends Pick<CarWriter, 'put' | 'close'> {
+  bytes(): Promise<Uint8Array>
+}
+
+export function memoryCarWriter (root: CID | CID[]): MemoryCar {
+  const deferred = defer<Uint8Array>()
+  const { writer, out } = CarWriter.create(Array.isArray(root) ? root : [root])
+
+  Promise.resolve()
+    .then(async () => {
+      deferred.resolve(await toBuffer(out))
+    })
+    .catch(err => {
+      deferred.reject(err)
+    })
+
+  return {
+    async put (block: { cid: CID, bytes: Uint8Array }): Promise<void> {
+      await writer.put(block)
+    },
+    async close (): Promise<void> {
+      await writer.close()
+    },
+    async bytes (): Promise<Uint8Array> {
+      return deferred.promise
+    }
+  }
+}