helia 2.0.2 → 2.0.3-0749cbf

package/src/block-brokers/trustless-gateway-block-broker.ts

@@ -0,0 +1,78 @@
+import { logger } from '@libp2p/logger'
+import type { BlockRetriever } from '@helia/interface/blocks'
+import type { AbortOptions } from 'interface-store'
+import type { CID } from 'multiformats/cid'
+import type { ProgressEvent, ProgressOptions } from 'progress-events'
+
+const log = logger('helia:trustless-gateway-block-provider')
+
+export type TrustlessGatewayGetBlockProgressEvents =
+  ProgressEvent<'trustless-gateway:get-block:fetch', URL>
+
+/**
+ * A class that accepts a list of trustless gateways that are queried
+ * for blocks.
+ */
+export class TrustedGatewayBlockBroker implements BlockRetriever<
+ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
+> {
+  private readonly gateways: URL[]
+
+  constructor (urls: Array<string | URL>) {
+    this.gateways = urls.map(url => new URL(url.toString()))
+  }
+
+  async retrieve (cid: CID, options: AbortOptions & ProgressOptions<TrustlessGatewayGetBlockProgressEvents> = {}): Promise<Uint8Array> {
+    // choose a gateway
+    const url = this.gateways[Math.floor(Math.random() * this.gateways.length)]
+
+    log('getting block for %c from %s', cid, url)
+
+    try {
+      const block = await getRawBlockFromGateway(url, cid, options.signal)
+      log('got block for %c from %s', cid, url)
+
+      return block
+    } catch (err: any) {
+      log.error('failed to get block for %c from %s', cid, url, err)
+
+      throw err
+    }
+  }
+}
+
+async function getRawBlockFromGateway (url: URL, cid: CID, signal?: AbortSignal): Promise<Uint8Array> {
+  const gwUrl = new URL(url)
+  gwUrl.pathname = `/ipfs/${cid.toString()}`
+
+  // necessary as not every gateway supports dag-cbor, but every should support
+  // sending raw block as-is
+  gwUrl.search = '?format=raw'
+
+  if (signal?.aborted === true) {
+    throw new Error(`Signal to fetch raw block for CID ${cid} from gateway ${gwUrl.toString()} was aborted prior to fetch`)
+  }
+
+  try {
+    const res = await fetch(gwUrl.toString(), {
+      signal,
+      headers: {
+        // also set header, just in case ?format= is filtered out by some
+        // reverse proxy
+        Accept: 'application/vnd.ipld.raw'
+      },
+      cache: 'force-cache'
+    })
+    if (!res.ok) {
+      throw new Error(`unable to fetch raw block for CID ${cid} from gateway ${gwUrl.toString()}`)
+    }
+    return new Uint8Array(await res.arrayBuffer())
+  } catch (cause) {
+    // @ts-expect-error - TS thinks signal?.aborted can only be false now
+    // because it was checked for true above.
+    if (signal?.aborted === true) {
+      throw new Error(`fetching raw block for CID ${cid} from gateway ${gwUrl.toString()} was aborted`)
+    }
+    throw new Error(`unable to fetch raw block for CID ${cid}`)
+  }
+}

package/src/helia.ts

@@ -1,8 +1,6 @@
+import { start, stop } from '@libp2p/interface/startable'
 import { logger } from '@libp2p/logger'
-import { type Bitswap, createBitswap } from 'ipfs-bitswap'
 import drain from 'it-drain'
-import { identity } from 'multiformats/hashes/identity'
-import { sha256, sha512 } from 'multiformats/hashes/sha2'
 import { CustomProgressEvent } from 'progress-events'
 import { PinsImpl } from './pins.js'
 import { BlockStorage } from './storage.js'
@@ -15,7 +13,6 @@ import type { Libp2p } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
 import type { Datastore } from 'interface-datastore'
 import type { CID } from 'multiformats/cid'
-import type { MultihashHasher } from 'multiformats/hashes/interface'
 
 const log = logger('helia')
 
@@ -31,34 +28,10 @@ export class HeliaImpl implements Helia {
   public datastore: Datastore
   public pins: Pins
 
-  #bitswap?: Bitswap
-
   constructor (init: HeliaImplInit) {
-    const hashers: MultihashHasher[] = [
-      sha256,
-      sha512,
-      identity,
-      ...(init.hashers ?? [])
-    ]
-
-    this.#bitswap = createBitswap(init.libp2p, init.blockstore, {
-      hashLoader: {
-        getHasher: async (codecOrName: string | number): Promise<MultihashHasher<number>> => {
-          const hasher = hashers.find(hasher => {
-            return hasher.code === codecOrName || hasher.name === codecOrName
-          })
-
-          if (hasher != null) {
-            return hasher
-          }
-
-          throw new Error(`Could not load hasher for code/name "${codecOrName}"`)
-        }
-      }
-    })
-
     const networkedStorage = new NetworkedStorage(init.blockstore, {
-      bitswap: this.#bitswap
+      blockBrokers: init.blockBrokers,
+      hashers: init.hashers
     })
 
     this.pins = new PinsImpl(init.datastore, networkedStorage, init.dagWalkers ?? [])
@@ -72,14 +45,13 @@ export class HeliaImpl implements Helia {
 
   async start (): Promise<void> {
     await assertDatastoreVersionIsCurrent(this.datastore)
-
-    await this.#bitswap?.start()
+    await start(this.blockstore)
     await this.libp2p.start()
   }
 
   async stop (): Promise<void> {
     await this.libp2p.stop()
-    await this.#bitswap?.stop()
+    await stop(this.blockstore)
   }
 
   async gc (options: GCOptions = {}): Promise<void> {

package/src/index.ts

@@ -24,13 +24,15 @@
 import { logger } from '@libp2p/logger'
 import { MemoryBlockstore } from 'blockstore-core'
 import { MemoryDatastore } from 'datastore-core'
+import { BitswapBlockBroker, TrustedGatewayBlockBroker } from './block-brokers/index.js'
 import { HeliaImpl } from './helia.js'
+import { defaultHashers } from './utils/default-hashers.js'
 import { createLibp2p } from './utils/libp2p.js'
 import { name, version } from './version.js'
+import type { DefaultLibp2pServices } from './utils/libp2p-defaults.js'
 import type { Helia } from '@helia/interface'
+import type { BlockBroker } from '@helia/interface/blocks'
 import type { Libp2p } from '@libp2p/interface'
-import type { PubSub } from '@libp2p/interface/pubsub'
-import type { DualKadDHT } from '@libp2p/kad-dht'
 import type { Blockstore } from 'interface-blockstore'
 import type { Datastore } from 'interface-datastore'
 import type { Libp2pOptions } from 'libp2p'
@@ -50,7 +52,7 @@ const log = logger('helia')
  */
 export interface DAGWalker {
   codec: number
-  walk: (block: Uint8Array) => AsyncGenerator<CID, void, undefined>
+  walk(block: Uint8Array): AsyncGenerator<CID, void, undefined>
 }
 
 /**
@@ -92,6 +94,12 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
    */
   dagWalkers?: DAGWalker[]
 
+  /**
+   * A list of strategies used to fetch blocks when they are not present in
+   * the local blockstore
+   */
+  blockBrokers?: BlockBroker[]
+
   /**
    * Pass `false` to not start the Helia node
    */
@@ -115,11 +123,28 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
   holdGcLock?: boolean
 }
 
+const DEFAULT_TRUSTLESS_GATEWAYS = [
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://dweb.link',
+
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://cf-ipfs.com',
+
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://4everland.io',
+
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://w3s.link',
+
+  // 2023-10-03: IPNS, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://cloudflare-ipfs.com'
+]
+
 /**
  * Create and return a Helia node
  */
 export async function createHelia <T extends Libp2p> (init: HeliaInit<T>): Promise<Helia<T>>
-export async function createHelia (init?: HeliaInit<Libp2p<{ dht: DualKadDHT, pubsub: PubSub }>>): Promise<Helia<Libp2p<{ dht: DualKadDHT, pubsub: PubSub }>>>
+export async function createHelia (init?: HeliaInit<Libp2p<DefaultLibp2pServices>>): Promise<Helia<Libp2p<DefaultLibp2pServices>>>
 export async function createHelia (init: HeliaInit = {}): Promise<Helia<unknown>> {
   const datastore = init.datastore ?? new MemoryDatastore()
   const blockstore = init.blockstore ?? new MemoryBlockstore()
@@ -132,11 +157,20 @@ export async function createHelia (init: HeliaInit = {}): Promise<Helia<unknown>
     libp2p = await createLibp2p(datastore, init.libp2p)
   }
 
+  const hashers = defaultHashers(init.hashers)
+
+  const blockBrokers = init.blockBrokers ?? [
+    new BitswapBlockBroker(libp2p, blockstore, hashers),
+    new TrustedGatewayBlockBroker(DEFAULT_TRUSTLESS_GATEWAYS)
+  ]
+
   const helia = new HeliaImpl({
     ...init,
     datastore,
     blockstore,
-    libp2p
+    libp2p,
+    blockBrokers,
+    hashers
   })
 
   if (init.start !== false) {
@@ -144,16 +178,7 @@ export async function createHelia (init: HeliaInit = {}): Promise<Helia<unknown>
   }
 
   // add helia to agent version
-  if (helia.libp2p.isStarted()) {
-    await addHeliaToAgentVersion(helia)
-  } else {
-    helia.libp2p.addEventListener('start', () => {
-      addHeliaToAgentVersion(helia)
-        .catch(err => {
-          log.error('could not add Helia to agent version', err)
-        })
-    })
-  }
+  addHeliaToAgentVersion(helia)
 
   return helia
 }
@@ -170,34 +195,18 @@ function isLibp2p (obj: any): obj is Libp2p {
   return funcs.every(m => typeof obj[m] === 'function')
 }
 
-async function addHeliaToAgentVersion (helia: Helia): Promise<void> {
+function addHeliaToAgentVersion (helia: Helia<any>): void {
   // add helia to agent version
-  const peer = await helia.libp2p.peerStore.get(helia.libp2p.peerId)
-  const versionBuf = peer.metadata.get('AgentVersion')
-
-  if (versionBuf == null) {
-    // identify was not configured
-    return
-  }
+  try {
+    const existingAgentVersion = helia.libp2p.services.identify.host.agentVersion
 
-  let versionStr = new TextDecoder().decode(versionBuf)
-
-  if (versionStr.match(/js-libp2p\/\d+\.\d+\.\d+\sUserAgent=/) == null) {
-    // the user changed the agent version
-    return
-  }
+    if (existingAgentVersion.match(/js-libp2p\/\d+\.\d+\.\d+\sUserAgent=/) == null) {
+      // the user changed the agent version
+      return
+    }
 
-  if (versionStr.includes(name)) {
-    // update version name
-    versionStr = `${name}/${version} ${versionStr.split(' ').slice(1).join(' ')}`
-  } else {
-    // just prepend version name
-    versionStr = `${name}/${version} ${versionStr}`
+    helia.libp2p.services.identify.host.agentVersion = `${name}/${version} ${helia.libp2p.services.identify.host.agentVersion}`
+  } catch (err) {
+    log.error('could not add Helia to agent version', err)
   }
-
-  await helia.libp2p.peerStore.merge(helia.libp2p.peerId, {
-    metadata: {
-      AgentVersion: new TextEncoder().encode(versionStr)
-    }
-  })
 }

package/src/storage.ts

@@ -1,3 +1,4 @@
+import { start, stop, type Startable } from '@libp2p/interface/startable'
 import createMortice from 'mortice'
 import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions } from '@helia/interface/blocks'
 import type { Pins } from '@helia/interface/pins'
@@ -13,7 +14,7 @@ export interface BlockStorageInit {
 }
 
 export interface GetOptions extends AbortOptions {
-  progress?: (evt: Event) => void
+  progress?(evt: Event): void
 }
 
 /**
@@ -21,10 +22,11 @@ export interface GetOptions extends AbortOptions {
  * blockstore (that may be on disk, s3, or something else). If the blocks are
  * not present Bitswap will be used to fetch them from network peers.
  */
-export class BlockStorage implements Blocks {
+export class BlockStorage implements Blocks, Startable {
   public lock: Mortice
   private readonly child: Blockstore
   private readonly pins: Pins
+  private started: boolean
 
   /**
    * Create a new BlockStorage
@@ -35,6 +37,21 @@ export class BlockStorage implements Blocks {
     this.lock = createMortice({
       singleProcess: options.holdGcLock
     })
+    this.started = false
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  async start (): Promise<void> {
+    await start(this.child)
+    this.started = true
+  }
+
+  async stop (): Promise<void> {
+    await stop(this.child)
+    this.started = false
   }
 
   unwrap (): Blockstore {

package/src/utils/default-hashers.ts

@@ -0,0 +1,12 @@
+import { identity } from 'multiformats/hashes/identity'
+import { sha256, sha512 } from 'multiformats/hashes/sha2'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+
+export function defaultHashers (hashers: MultihashHasher[] = []): MultihashHasher[] {
+  return [
+    sha256,
+    sha512,
+    identity,
+    ...hashers
+  ]
+}

package/src/utils/libp2p-defaults.browser.ts

@@ -13,12 +13,22 @@ import { ipnsValidator } from 'ipns/validator'
 import { autoNATService } from 'libp2p/autonat'
 import { circuitRelayTransport } from 'libp2p/circuit-relay'
 import { dcutrService } from 'libp2p/dcutr'
-import { identifyService } from 'libp2p/identify'
+import { type IdentifyService, identifyService } from 'libp2p/identify'
+import { pingService, type PingService } from 'libp2p/ping'
 import { bootstrapConfig } from './bootstrappers.js'
 import type { PubSub } from '@libp2p/interface/pubsub'
 import type { Libp2pOptions } from 'libp2p'
 
-export function libp2pDefaults (): Libp2pOptions<{ dht: DualKadDHT, pubsub: PubSub, identify: unknown, autoNAT: unknown, dcutr: unknown }> {
+export interface DefaultLibp2pServices extends Record<string, unknown> {
+  dht: DualKadDHT
+  pubsub: PubSub
+  identify: IdentifyService
+  autoNAT: unknown
+  dcutr: unknown
+  ping: PingService
+}
+
+export function libp2pDefaults (): Libp2pOptions<DefaultLibp2pServices> {
   return {
     addresses: {
       listen: [
@@ -26,13 +36,13 @@ export function libp2pDefaults (): Libp2pOptions<{ dht: DualKadDHT, pubsub: PubS
       ]
     },
     transports: [
+      circuitRelayTransport({
+        discoverRelays: 1
+      }),
       webRTC(),
       webRTCDirect(),
       webTransport(),
-      webSockets(),
-      circuitRelayTransport({
-        discoverRelays: 1
-      })
+      webSockets()
     ],
     connectionEncryption: [
       noise()
@@ -60,7 +70,8 @@ export function libp2pDefaults (): Libp2pOptions<{ dht: DualKadDHT, pubsub: PubS
         selectors: {
           ipns: ipnsSelector
         }
-      })
+      }),
+      ping: pingService()
     }
   }
 }

package/src/utils/libp2p-defaults.ts

@@ -7,31 +7,47 @@ import { type DualKadDHT, kadDHT } from '@libp2p/kad-dht'
 import { mdns } from '@libp2p/mdns'
 import { mplex } from '@libp2p/mplex'
 import { tcp } from '@libp2p/tcp'
+import { webRTC, webRTCDirect } from '@libp2p/webrtc'
 import { webSockets } from '@libp2p/websockets'
 import { ipnsSelector } from 'ipns/selector'
 import { ipnsValidator } from 'ipns/validator'
 import { autoNATService } from 'libp2p/autonat'
 import { circuitRelayTransport, circuitRelayServer, type CircuitRelayService } from 'libp2p/circuit-relay'
 import { dcutrService } from 'libp2p/dcutr'
-import { identifyService } from 'libp2p/identify'
+import { type IdentifyService, identifyService } from 'libp2p/identify'
+import { pingService, type PingService } from 'libp2p/ping'
 import { uPnPNATService } from 'libp2p/upnp-nat'
 import { bootstrapConfig } from './bootstrappers.js'
 import type { PubSub } from '@libp2p/interface/pubsub'
 import type { Libp2pOptions } from 'libp2p'
 
-export function libp2pDefaults (): Libp2pOptions<{ dht: DualKadDHT, pubsub: PubSub, relay: CircuitRelayService, identify: unknown, autoNAT: unknown, upnp: unknown, dcutr: unknown }> {
+export interface DefaultLibp2pServices extends Record<string, unknown> {
+  dht: DualKadDHT
+  pubsub: PubSub
+  relay: CircuitRelayService
+  identify: IdentifyService
+  autoNAT: unknown
+  upnp: unknown
+  dcutr: unknown
+  ping: PingService
+}
+
+export function libp2pDefaults (): Libp2pOptions<DefaultLibp2pServices> {
   return {
     addresses: {
       listen: [
-        '/ip4/0.0.0.0/tcp/0'
+        '/ip4/0.0.0.0/tcp/0',
+        '/webrtc'
       ]
     },
     transports: [
-      tcp(),
-      webSockets(),
       circuitRelayTransport({
         discoverRelays: 1
-      })
+      }),
+      tcp(),
+      webRTC(),
+      webRTCDirect(),
+      webSockets()
     ],
     connectionEncryption: [
       noise()
@@ -63,7 +79,8 @@ export function libp2pDefaults (): Libp2pOptions<{ dht: DualKadDHT, pubsub: PubS
       }),
       relay: circuitRelayServer({
         advertise: true
-      })
+      }),
+      ping: pingService()
     }
   }
 }

package/src/utils/libp2p.ts

@@ -1,17 +1,14 @@
 import { createLibp2p as create, type Libp2pOptions } from 'libp2p'
-import { libp2pDefaults } from './libp2p-defaults.js'
+import { type DefaultLibp2pServices, libp2pDefaults } from './libp2p-defaults.js'
 import type { Libp2p } from '@libp2p/interface'
-import type { PubSub } from '@libp2p/interface/pubsub'
-import type { DualKadDHT } from '@libp2p/kad-dht'
 import type { Datastore } from 'interface-datastore'
-import type { CircuitRelayService } from 'libp2p/circuit-relay'
 
 export interface CreateLibp2pOptions {
   datastore: Datastore
   start?: boolean
 }
 
-export async function createLibp2p (datastore: Datastore, options?: Libp2pOptions<any>): Promise<Libp2p<{ dht: DualKadDHT, pubsub: PubSub, relay: CircuitRelayService }>> {
+export async function createLibp2p (datastore: Datastore, options?: Libp2pOptions<any>): Promise<Libp2p<DefaultLibp2pServices>> {
   const defaults = libp2pDefaults()
   options = options ?? {}