@helia/interop 3.0.0 → 3.0.1-031519c

package/src/bin.ts

@@ -0,0 +1,25 @@
+#! /usr/bin/env node
+/* eslint-disable no-console */
+
+import { spawn } from 'node:child_process'
+import { dirname, resolve } from 'path'
+import { fileURLToPath } from 'url'
+
+// aegir should be run from `node_modules/@helia/interop`
+const cwd = resolve(dirname(fileURLToPath(import.meta.url)), '../../')
+
+const test = spawn('npx', ['aegir', 'test'], {
+  cwd
+})
+
+test.stdout.on('data', (data) => {
+  process.stdout.write(data)
+})
+
+test.stderr.on('data', (data) => {
+  process.stderr.write(data)
+})
+
+test.on('close', (code) => {
+  process.exit(code ?? 0)
+})

package/src/car.spec.ts

@@ -0,0 +1,102 @@
+/* eslint-env mocha */
+
+import { car } from '@helia/car'
+import { type UnixFS, unixfs } from '@helia/unixfs'
+import { CarReader } from '@ipld/car'
+import { expect } from 'aegir/chai'
+import drain from 'it-drain'
+import toBuffer from 'it-to-buffer'
+import { CID } from 'multiformats/cid'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import { memoryCarWriter } from './fixtures/memory-car.js'
+import type { Car } from '@helia/car'
+import type { HeliaLibp2p } from 'helia'
+import type { FileCandidate } from 'ipfs-unixfs-importer'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('@helia/car', () => {
+  let helia: HeliaLibp2p
+  let c: Car
+  let u: UnixFS
+  let kubo: Controller
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    c = car(helia)
+    u = unixfs(helia)
+    kubo = await createKuboNode()
+
+    // connect helia to kubo
+    await helia.libp2p.dial(kubo.peer.addresses)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should export a car from Helia, import and read the contents from kubo', async () => {
+    const chunkSize = 1024 * 1024
+    const size = chunkSize * 10
+    const input: Uint8Array[] = []
+
+    const candidate: FileCandidate = {
+      content: (async function * () {
+        for (let i = 0; i < size; i += chunkSize) {
+          const buf = new Uint8Array(chunkSize)
+          input.push(buf)
+
+          yield buf
+        }
+      }())
+    }
+
+    const cid = await u.addFile(candidate)
+    const writer = memoryCarWriter(cid)
+    await c.export(cid, writer)
+
+    const buf = await writer.bytes()
+
+    await drain(kubo.api.dag.import([buf]))
+
+    const output: Uint8Array[] = []
+
+    for await (const b of kubo.api.cat(cid)) {
+      output.push(b)
+    }
+
+    expect(toBuffer(output)).to.equalBytes(toBuffer(input))
+  })
+
+  it('should export a car from kubo, import and read the contents from Helia', async () => {
+    const chunkSize = 1024 * 1024
+    const size = chunkSize * 10
+    const input: Uint8Array[] = []
+
+    const candidate: FileCandidate = {
+      content: (async function * () {
+        for (let i = 0; i < size; i += chunkSize) {
+          const buf = new Uint8Array(chunkSize)
+          input.push(buf)
+
+          yield buf
+        }
+      }())
+    }
+
+    const { cid } = await kubo.api.add(candidate.content)
+    const bytes = await toBuffer(kubo.api.dag.export(cid))
+
+    const reader = await CarReader.fromBytes(bytes)
+
+    await c.import(reader)
+
+    expect(await toBuffer(u.cat(CID.parse(cid.toString())))).to.equalBytes(toBuffer(input))
+  })
+})

package/src/dag-cbor.spec.ts

@@ -0,0 +1,65 @@
+/* eslint-env mocha */
+
+import { dagCbor, type DAGCBOR, type AddOptions } from '@helia/dag-cbor'
+import * as codec from '@ipld/dag-cbor'
+import { expect } from 'aegir/chai'
+import { CID } from 'multiformats/cid'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('@helia/dag-cbor', () => {
+  let helia: HeliaLibp2p
+  let d: DAGCBOR
+  let kubo: Controller
+
+  async function expectSameCid (data: () => any, heliaOpts: Partial<AddOptions> = {}, kuboOpts: KuboAddOptions = {}): Promise<void> {
+    const heliaCid = await d.add(data(), heliaOpts)
+    const kuboCid = await kubo.api.dag.put(data(), kuboOpts)
+
+    expect(heliaCid.toString()).to.equal(kuboCid.toString())
+  }
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    d = dagCbor(helia)
+    kubo = await createKuboNode()
+
+    await helia.libp2p.dial((await (kubo.api.id())).addresses)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should create the same CID for a string', async () => {
+    const candidate = (): any => ({ hello: 'world' })
+
+    await expectSameCid(candidate)
+  })
+
+  it('should add to helia and fetch from kubo', async () => {
+    const input = { hello: 'world' }
+    const cid = await d.add(input)
+    const block = await kubo.api.block.get(cid)
+    const output = codec.decode(block)
+
+    expect(output).to.deep.equal(input)
+  })
+
+  it('should add to kubo and fetch from helia', async () => {
+    const input = { hello: 'world' }
+    const cid = await kubo.api.block.put(codec.encode(input))
+    const output = await d.get(CID.parse(cid.toString()))
+
+    expect(output).to.deep.equal(input)
+  })
+})

package/src/dag-json.spec.ts

@@ -0,0 +1,65 @@
+/* eslint-env mocha */
+
+import { dagJson, type DAGJSON, type AddOptions } from '@helia/dag-json'
+import { expect } from 'aegir/chai'
+import { CID } from 'multiformats/cid'
+import * as codec from 'multiformats/codecs/json'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('@helia/dag-json', () => {
+  let helia: HeliaLibp2p
+  let d: DAGJSON
+  let kubo: Controller
+
+  async function expectSameCid (data: () => any, heliaOpts: Partial<AddOptions> = {}, kuboOpts: KuboAddOptions = { format: 'dag-json' }): Promise<void> {
+    const heliaCid = await d.add(data(), heliaOpts)
+    const kuboCid = await kubo.api.block.put(codec.encode(data()), kuboOpts)
+
+    expect(heliaCid.toString()).to.equal(kuboCid.toString())
+  }
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    d = dagJson(helia)
+    kubo = await createKuboNode()
+
+    await helia.libp2p.dial((await (kubo.api.id())).addresses)
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should create the same CID for a string', async () => {
+    const candidate = (): any => ({ hello: 'world' })
+
+    await expectSameCid(candidate)
+  })
+
+  it('should add to helia and fetch from kubo', async () => {
+    const input = { hello: 'world' }
+    const cid = await d.add(input)
+    const block = await kubo.api.block.get(cid)
+    const output = codec.decode(block)
+
+    expect(output).to.deep.equal(input)
+  })
+
+  it('should add to kubo and fetch from helia', async () => {
+    const input = { hello: 'world' }
+    const cid = await kubo.api.block.put(codec.encode(input))
+    const output = await d.get(CID.parse(cid.toString()))
+
+    expect(output).to.deep.equal(input)
+  })
+})

package/src/fixtures/connect.ts

@@ -0,0 +1,19 @@
+import { expect } from 'aegir/chai'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+/**
+ * Connect the two nodes by dialing a protocol stream
+ */
+export async function connect (helia: HeliaLibp2p, kubo: Controller, protocol: string): Promise<void> {
+  let connected = false
+  for (const addr of kubo.peer.addresses) {
+    try {
+      await helia.libp2p.dialProtocol(addr, protocol)
+      connected = true
+      break
+    } catch { }
+  }
+
+  expect(connected).to.be.true('could not connect Helia to Kubo')
+}

package/src/fixtures/create-helia-http.ts

@@ -0,0 +1,8 @@
+import { createHeliaHTTP as createHelia, type HeliaHTTPInit } from '@helia/http'
+import type { Helia } from '@helia/interface'
+
+export async function createHeliaHTTP (init: Partial<HeliaHTTPInit> = {}): Promise<Helia> {
+  return createHelia({
+    ...init
+  })
+}

package/src/fixtures/create-helia.browser.ts

@@ -0,0 +1,64 @@
+import { bitswap } from '@helia/block-brokers'
+import { ipnsValidator, ipnsSelector } from '@helia/ipns'
+import { kadDHT, removePublicAddressesMapper } from '@libp2p/kad-dht'
+import { webSockets } from '@libp2p/websockets'
+import { all } from '@libp2p/websockets/filters'
+import { sha3512 } from '@multiformats/sha3'
+import { createHelia, libp2pDefaults } from 'helia'
+import type { Libp2p } from '@libp2p/interface'
+import type { DefaultLibp2pServices, HeliaLibp2p } from 'helia'
+
+export async function createHeliaNode (): Promise<HeliaLibp2p<Libp2p<DefaultLibp2pServices>>> {
+  const defaults = libp2pDefaults()
+
+  // allow dialing insecure WebSockets
+  defaults.transports?.pop()
+  defaults.transports = [
+    ...(defaults.transports ?? []),
+    webSockets({
+      filter: all
+    })
+  ]
+
+  // allow dialing loopback
+  defaults.connectionGater = {
+    denyDialMultiaddr: () => false
+  }
+
+  // use LAN DHT
+  defaults.services = {
+    ...(defaults.services ?? {}),
+    dht: kadDHT({
+      validators: {
+        ipns: ipnsValidator
+      },
+      selectors: {
+        ipns: ipnsSelector
+      },
+      // skips waiting for the initial self-query to find peers
+      allowQueryWithZeroPeers: true,
+
+      protocol: '/ipfs/lan/kad/1.0.0',
+      peerInfoMapper: removePublicAddressesMapper,
+      clientMode: false
+    })
+  }
+
+  // remove bootstrappers
+  defaults.peerDiscovery = []
+
+  // remove services that are not used in tests
+  delete defaults.services.autoNAT
+  delete defaults.services.dcutr
+  delete defaults.services.delegatedRouting
+
+  return createHelia<Libp2p<DefaultLibp2pServices>>({
+    blockBrokers: [
+      bitswap()
+    ],
+    libp2p: defaults,
+    hashers: [
+      sha3512
+    ]
+  })
+}

package/src/fixtures/create-helia.ts

@@ -0,0 +1,51 @@
+import { bitswap } from '@helia/block-brokers'
+import { ipnsValidator, ipnsSelector } from '@helia/ipns'
+import { kadDHT, removePublicAddressesMapper } from '@libp2p/kad-dht'
+import { sha3512 } from '@multiformats/sha3'
+import { createHelia, libp2pDefaults } from 'helia'
+import type { Libp2p } from '@libp2p/interface'
+import type { DefaultLibp2pServices, HeliaLibp2p } from 'helia'
+
+export async function createHeliaNode (): Promise<HeliaLibp2p<Libp2p<DefaultLibp2pServices>>> {
+  const defaults = libp2pDefaults()
+  defaults.addresses = {
+    listen: [
+      '/ip4/0.0.0.0/tcp/0'
+    ]
+  }
+  defaults.services = {
+    ...(defaults.services ?? {}),
+    dht: kadDHT({
+      validators: {
+        ipns: ipnsValidator
+      },
+      selectors: {
+        ipns: ipnsSelector
+      },
+      // skips waiting for the initial self-query to find peers
+      allowQueryWithZeroPeers: true,
+
+      protocol: '/ipfs/lan/kad/1.0.0',
+      peerInfoMapper: removePublicAddressesMapper,
+      clientMode: false
+    })
+  }
+
+  // remove bootstrappers, mdns, etc
+  defaults.peerDiscovery = []
+
+  // remove services that are not used in tests
+  delete defaults.services.autoNAT
+  delete defaults.services.dcutr
+  delete defaults.services.delegatedRouting
+
+  return createHelia<Libp2p<DefaultLibp2pServices>>({
+    blockBrokers: [
+      bitswap()
+    ],
+    libp2p: defaults,
+    hashers: [
+      sha3512
+    ]
+  })
+}

package/src/fixtures/create-kubo.browser.ts

@@ -0,0 +1,30 @@
+import { type Controller, createController } from 'ipfsd-ctl'
+import * as kuboRpcClient from 'kubo-rpc-client'
+
+export async function createKuboNode (): Promise<Controller> {
+  return createController({
+    kuboRpcModule: kuboRpcClient,
+    test: true,
+    endpoint: process.env.IPFSD_SERVER,
+    ipfsOptions: {
+      config: {
+        Addresses: {
+          Swarm: [
+            '/ip4/0.0.0.0/tcp/0',
+            '/ip4/0.0.0.0/tcp/0/ws'
+          ],
+          Gateway: '/ip4/127.0.0.1/tcp/8180'
+        },
+        Gateway: {
+          NoFetch: true,
+          ExposeRoutingAPI: true,
+          HTTPHeaders: {
+            'Access-Control-Allow-Origin': ['*'],
+            'Access-Control-Allow-Methods': ['GET', 'POST', 'PUT', 'OPTIONS']
+          }
+        }
+      }
+    },
+    args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub']
+  })
+}

package/src/fixtures/create-kubo.ts

@@ -0,0 +1,32 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */
+import { createController, type Controller } from 'ipfsd-ctl'
+import { path as kuboPath } from 'kubo'
+import * as kuboRpcClient from 'kubo-rpc-client'
+
+export async function createKuboNode (): Promise<Controller> {
+  return createController({
+    kuboRpcModule: kuboRpcClient,
+    ipfsBin: kuboPath(),
+    test: true,
+    ipfsOptions: {
+      config: {
+        Addresses: {
+          Swarm: [
+            '/ip4/0.0.0.0/tcp/4001',
+            '/ip4/0.0.0.0/tcp/4002/ws'
+          ],
+          Gateway: '/ip4/127.0.0.1/tcp/8180'
+        },
+        Gateway: {
+          NoFetch: true,
+          ExposeRoutingAPI: true,
+          HTTPHeaders: {
+            'Access-Control-Allow-Origin': ['*'],
+            'Access-Control-Allow-Methods': ['GET', 'POST', 'PUT', 'OPTIONS']
+          }
+        }
+      }
+    },
+    args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub']
+  })
+}

package/src/fixtures/create-peer-ids.ts

@@ -0,0 +1,46 @@
+import all from 'it-all'
+import map from 'it-map'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { compare as uint8ArrayCompare } from 'uint8arrays/compare'
+import { xor as uint8ArrayXor } from 'uint8arrays/xor'
+import type { PeerId } from '@libp2p/interface'
+
+/**
+ * Sort peers by distance to the KadID of the passed buffer
+ */
+export async function sortClosestPeers (buf: Uint8Array, peers: PeerId[]): Promise<PeerId[]> {
+  const kadId = await convertBuffer(buf)
+
+  const distances = await all(
+    map(peers, async (peer) => {
+      const id = await convertPeerId(peer)
+
+      return {
+        peer,
+        distance: uint8ArrayXor(id, kadId)
+      }
+    })
+  )
+
+  return distances
+    .sort((a, b) => {
+      return uint8ArrayCompare(a.distance, b.distance)
+    })
+    .map((d) => d.peer)
+}
+
+/**
+ * Creates a DHT ID by hashing a Peer ID
+ */
+export async function convertPeerId (peerId: PeerId): Promise<Uint8Array> {
+  return convertBuffer(peerId.toBytes())
+}
+
+/**
+ * Creates a DHT ID by hashing a given Uint8Array
+ */
+export async function convertBuffer (buf: Uint8Array): Promise<Uint8Array> {
+  const multihash = await sha256.digest(buf)
+
+  return multihash.digest
+}

package/src/fixtures/key-types.ts

@@ -0,0 +1,7 @@
+import type { PeerIdType } from '@libp2p/interface'
+
+export const keyTypes: PeerIdType[] = [
+  'Ed25519',
+  'secp256k1',
+  'RSA'
+]

package/src/fixtures/memory-car.ts

@@ -0,0 +1,33 @@
+import { CarWriter } from '@ipld/car'
+import toBuffer from 'it-to-buffer'
+import defer from 'p-defer'
+import type { CID } from 'multiformats/cid'
+
+export interface MemoryCar extends Pick<CarWriter, 'put' | 'close'> {
+  bytes(): Promise<Uint8Array>
+}
+
+export function memoryCarWriter (root: CID | CID[]): MemoryCar {
+  const deferred = defer<Uint8Array>()
+  const { writer, out } = CarWriter.create(Array.isArray(root) ? root : [root])
+
+  Promise.resolve()
+    .then(async () => {
+      deferred.resolve(await toBuffer(out))
+    })
+    .catch(err => {
+      deferred.reject(err)
+    })
+
+  return {
+    async put (block: { cid: CID, bytes: Uint8Array }): Promise<void> {
+      await writer.put(block)
+    },
+    async close (): Promise<void> {
+      await writer.close()
+    },
+    async bytes (): Promise<Uint8Array> {
+      return deferred.promise
+    }
+  }
+}

package/src/fixtures/wait-for.ts

@@ -0,0 +1,26 @@
+export interface WaitForOptions {
+  timeout: number
+  delay?: number
+  message?: string
+}
+
+export async function waitFor (fn: () => Promise<boolean>, options: WaitForOptions): Promise<void> {
+  const delay = options.delay ?? 100
+  const timeoutAt = Date.now() + options.timeout
+
+  while (true) {
+    const result = await fn()
+
+    if (result) {
+      return
+    }
+
+    await new Promise<void>((resolve) => {
+      setTimeout(() => { resolve() }, delay)
+    })
+
+    if (Date.now() > timeoutAt) {
+      throw new Error(options.message ?? 'WaitFor timed out')
+    }
+  }
+}

package/src/helia-blockstore.spec.ts

@@ -0,0 +1,59 @@
+/* eslint-env mocha */
+
+import { peerIdFromString } from '@libp2p/peer-id'
+import { expect } from 'aegir/chai'
+import toBuffer from 'it-to-buffer'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { createHeliaNode } from './fixtures/create-helia.js'
+import { createKuboNode } from './fixtures/create-kubo.js'
+import type { HeliaLibp2p } from 'helia'
+import type { Controller } from 'ipfsd-ctl'
+
+describe('helia - blockstore', () => {
+  let helia: HeliaLibp2p
+  let kubo: Controller
+
+  beforeEach(async () => {
+    helia = await createHeliaNode()
+    kubo = await createKuboNode()
+
+    // connect the two nodes
+    await helia.libp2p.peerStore.merge(peerIdFromString(kubo.peer.id.toString()), {
+      multiaddrs: kubo.peer.addresses
+    })
+    await helia.libp2p.dial(peerIdFromString(kubo.peer.id.toString()))
+  })
+
+  afterEach(async () => {
+    if (helia != null) {
+      await helia.stop()
+    }
+
+    if (kubo != null) {
+      await kubo.stop()
+    }
+  })
+
+  it('should be able to send a block', async () => {
+    const input = Uint8Array.from([0, 1, 2, 3, 4])
+    const digest = await sha256.digest(input)
+    const cid = CID.createV1(raw.code, digest)
+    await helia.blockstore.put(cid, input)
+    const output = await toBuffer(kubo.api.cat(cid))
+
+    expect(output).to.equalBytes(input)
+  })
+
+  it('should be able to receive a block', async () => {
+    const input = Uint8Array.from([0, 1, 2, 3, 4])
+    const { cid } = await kubo.api.add({ content: input }, {
+      cidVersion: 1,
+      rawLeaves: true
+    })
+    const output = await helia.blockstore.get(CID.parse(cid.toString()))
+
+    expect(output).to.equalBytes(input)
+  })
+})