helia 2.0.3 → 2.1.0

package/src/block-brokers/bitswap.ts

@@ -0,0 +1,78 @@
+import { createBitswap } from 'ipfs-bitswap'
+import type { BlockAnnouncer, BlockBroker, BlockRetrievalOptions, BlockRetriever } from '@helia/interface/blocks'
+import type { Libp2p } from '@libp2p/interface'
+import type { Startable } from '@libp2p/interface/startable'
+import type { Blockstore } from 'interface-blockstore'
+import type { Bitswap, BitswapNotifyProgressEvents, BitswapOptions, BitswapWantBlockProgressEvents } from 'ipfs-bitswap'
+import type { CID } from 'multiformats/cid'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+import type { ProgressOptions } from 'progress-events'
+
+interface BitswapComponents {
+  libp2p: Libp2p
+  blockstore: Blockstore
+  hashers: MultihashHasher[]
+}
+
+export interface BitswapInit extends BitswapOptions {
+
+}
+
+class BitswapBlockBroker implements BlockAnnouncer<ProgressOptions<BitswapNotifyProgressEvents>>, BlockRetriever<
+ProgressOptions<BitswapWantBlockProgressEvents>
+>, Startable {
+  private readonly bitswap: Bitswap
+  private started: boolean
+
+  constructor (components: BitswapComponents, init: BitswapInit = {}) {
+    const { libp2p, blockstore, hashers } = components
+
+    this.bitswap = createBitswap(libp2p, blockstore, {
+      hashLoader: {
+        getHasher: async (codecOrName: string | number): Promise<MultihashHasher<number>> => {
+          const hasher = hashers.find(hasher => {
+            return hasher.code === codecOrName || hasher.name === codecOrName
+          })
+
+          if (hasher != null) {
+            return hasher
+          }
+
+          throw new Error(`Could not load hasher for code/name "${codecOrName}"`)
+        }
+      },
+      ...init
+    })
+    this.started = false
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  async start (): Promise<void> {
+    await this.bitswap.start()
+    this.started = true
+  }
+
+  async stop (): Promise<void> {
+    await this.bitswap.stop()
+    this.started = false
+  }
+
+  announce (cid: CID, block: Uint8Array, options?: ProgressOptions<BitswapNotifyProgressEvents>): void {
+    this.bitswap.notify(cid, block, options)
+  }
+
+  async retrieve (cid: CID, { validateFn, ...options }: BlockRetrievalOptions<ProgressOptions<BitswapWantBlockProgressEvents>> = {}): Promise<Uint8Array> {
+    return this.bitswap.want(cid, options)
+  }
+}
+
+/**
+ * A helper factory for users who want to override Helia `blockBrokers` but
+ * still want to use the default `BitswapBlockBroker`.
+ */
+export function bitswap (init: BitswapInit = {}): (components: BitswapComponents) => BlockBroker {
+  return (components) => new BitswapBlockBroker(components, init)
+}

package/src/block-brokers/index.ts

@@ -0,0 +1,2 @@
+export { bitswap } from './bitswap.js'
+export { trustlessGateway } from './trustless-gateway/index.js'

package/src/block-brokers/trustless-gateway/broker.ts

@@ -0,0 +1,65 @@
+import { logger } from '@libp2p/logger'
+import { TrustlessGateway } from './trustless-gateway.js'
+import { DEFAULT_TRUSTLESS_GATEWAYS } from './index.js'
+import type { TrustlessGatewayBlockBrokerInit, TrustlessGatewayGetBlockProgressEvents } from './index.js'
+import type { BlockRetrievalOptions, BlockRetriever } from '@helia/interface/blocks'
+import type { CID } from 'multiformats/cid'
+import type { ProgressOptions } from 'progress-events'
+
+const log = logger('helia:trustless-gateway-block-broker')
+
+/**
+ * A class that accepts a list of trustless gateways that are queried
+ * for blocks.
+ */
+export class TrustlessGatewayBlockBroker implements BlockRetriever<
+ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
+> {
+  private readonly gateways: TrustlessGateway[]
+
+  constructor (init: TrustlessGatewayBlockBrokerInit = {}) {
+    this.gateways = (init.gateways ?? DEFAULT_TRUSTLESS_GATEWAYS)
+      .map((gatewayOrUrl) => {
+        return new TrustlessGateway(gatewayOrUrl)
+      })
+  }
+
+  async retrieve (cid: CID, options: BlockRetrievalOptions<ProgressOptions<TrustlessGatewayGetBlockProgressEvents>> = {}): Promise<Uint8Array> {
+    // Loop through the gateways until we get a block or run out of gateways
+    // TODO: switch to toSorted when support is better
+    const sortedGateways = this.gateways.sort((a, b) => b.reliability() - a.reliability())
+    const aggregateErrors: Error[] = []
+
+    for (const gateway of sortedGateways) {
+      log('getting block for %c from %s', cid, gateway.url)
+      try {
+        const block = await gateway.getRawBlock(cid, options.signal)
+        log.trace('got block for %c from %s', cid, gateway.url)
+        try {
+          await options.validateFn?.(block)
+        } catch (err) {
+          log.error('failed to validate block for %c from %s', cid, gateway.url, err)
+          gateway.incrementInvalidBlocks()
+
+          throw new Error(`unable to validate block for CID ${cid} from gateway ${gateway.url}`)
+        }
+
+        return block
+      } catch (err: unknown) {
+        log.error('failed to get block for %c from %s', cid, gateway.url, err)
+        if (err instanceof Error) {
+          aggregateErrors.push(err)
+        } else {
+          aggregateErrors.push(new Error(`unable to fetch raw block for CID ${cid} from gateway ${gateway.url}`))
+        }
+        // if signal was aborted, exit the loop
+        if (options.signal?.aborted === true) {
+          log.trace('request aborted while fetching raw block for CID %c from gateway %s', cid, gateway.url)
+          break
+        }
+      }
+    }
+
+    throw new AggregateError(aggregateErrors, `unable to fetch raw block for CID ${cid} from any gateway`)
+  }
+}

package/src/block-brokers/trustless-gateway/index.ts

@@ -0,0 +1,28 @@
+import { TrustlessGatewayBlockBroker } from './broker.js'
+import type { BlockRetriever } from '@helia/interface/src/blocks.js'
+import type { ProgressEvent } from 'progress-events'
+
+export const DEFAULT_TRUSTLESS_GATEWAYS = [
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://dweb.link',
+
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://cf-ipfs.com',
+
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://4everland.io',
+
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://w3s.link'
+]
+
+export type TrustlessGatewayGetBlockProgressEvents =
+  ProgressEvent<'trustless-gateway:get-block:fetch', URL>
+
+export interface TrustlessGatewayBlockBrokerInit {
+  gateways?: Array<string | URL>
+}
+
+export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): () => BlockRetriever {
+  return () => new TrustlessGatewayBlockBroker(init)
+}

package/src/block-brokers/trustless-gateway/trustless-gateway.ts

@@ -0,0 +1,126 @@
+import type { CID } from 'multiformats/cid'
+
+/**
+ * A `TrustlessGateway` keeps track of the number of attempts, errors, and
+ * successes for a given gateway url so that we can prioritize gateways that
+ * have been more reliable in the past, and ensure that requests are distributed
+ * across all gateways within a given `TrustlessGatewayBlockBroker` instance.
+ */
+export class TrustlessGateway {
+  public readonly url: URL
+  /**
+   * The number of times this gateway has been attempted to be used to fetch a
+   * block. This includes successful, errored, and aborted attempts. By counting
+   * even aborted attempts, slow gateways that are out-raced by others will be
+   * considered less reliable.
+   */
+  #attempts = 0
+
+  /**
+   * The number of times this gateway has errored while attempting to fetch a
+   * block. This includes `response.ok === false` and any other errors that
+   * throw while attempting to fetch a block. This does not include aborted
+   * attempts.
+   */
+  #errors = 0
+
+  /**
+   * The number of times this gateway has returned an invalid block. A gateway
+   * that returns the wrong blocks for a CID should be considered for removal
+   * from the list of gateways to fetch blocks from.
+   */
+  #invalidBlocks = 0
+
+  /**
+   * The number of times this gateway has successfully fetched a block.
+   */
+  #successes = 0
+
+  constructor (url: URL | string) {
+    this.url = url instanceof URL ? url : new URL(url)
+  }
+
+  /**
+   * Fetch a raw block from `this.url` following the specification defined at
+   * https://specs.ipfs.tech/http-gateways/trustless-gateway/
+   */
+  async getRawBlock (cid: CID, signal?: AbortSignal): Promise<Uint8Array> {
+    const gwUrl = this.url
+    gwUrl.pathname = `/ipfs/${cid.toString()}`
+
+    // necessary as not every gateway supports dag-cbor, but every should support
+    // sending raw block as-is
+    gwUrl.search = '?format=raw'
+
+    if (signal?.aborted === true) {
+      throw new Error(`Signal to fetch raw block for CID ${cid} from gateway ${this.url} was aborted prior to fetch`)
+    }
+
+    try {
+      this.#attempts++
+      const res = await fetch(gwUrl.toString(), {
+        signal,
+        headers: {
+        // also set header, just in case ?format= is filtered out by some
+        // reverse proxy
+          Accept: 'application/vnd.ipld.raw'
+        },
+        cache: 'force-cache'
+      })
+      if (!res.ok) {
+        this.#errors++
+        throw new Error(`unable to fetch raw block for CID ${cid} from gateway ${this.url}`)
+      }
+      this.#successes++
+      return new Uint8Array(await res.arrayBuffer())
+    } catch (cause) {
+      // @ts-expect-error - TS thinks signal?.aborted can only be false now
+      // because it was checked for true above.
+      if (signal?.aborted === true) {
+        throw new Error(`fetching raw block for CID ${cid} from gateway ${this.url} was aborted`)
+      }
+      this.#errors++
+      throw new Error(`unable to fetch raw block for CID ${cid}`)
+    }
+  }
+
+  /**
+   * Encapsulate the logic for determining whether a gateway is considered
+   * reliable, for prioritization. This is based on the number of successful attempts made
+   * and the number of errors encountered.
+   *
+   * Unused gateways have 100% reliability; They will be prioritized over
+   * gateways with a 100% success rate to ensure that we attempt all gateways.
+   */
+  reliability (): number {
+    /**
+     * if we have never tried to use this gateway, it is considered the most
+     * reliable until we determine otherwise (prioritize unused gateways)
+     */
+    if (this.#attempts === 0) {
+      return 1
+    }
+
+    if (this.#invalidBlocks > 0) {
+      // this gateway may not be trustworthy..
+      return -Infinity
+    }
+
+    /**
+     * We have attempted the gateway, so we need to calculate the reliability
+     * based on the number of attempts, errors, and successes. Gateways that
+     * return a single error should drop their reliability score more than a
+     * single success increases it.
+     *
+     * Play around with the below reliability function at https://www.desmos.com/calculator/d6hfhf5ukm
+     */
+    return this.#successes / (this.#attempts + (this.#errors * 3))
+  }
+
+  /**
+   * Increment the number of invalid blocks returned by this gateway.
+   */
+  incrementInvalidBlocks (): void {
+    this.#invalidBlocks++
+  }
+}

package/src/helia.ts

@@ -1,12 +1,12 @@
+import { start, stop } from '@libp2p/interface/startable'
 import { logger } from '@libp2p/logger'
-import { type Bitswap, createBitswap } from 'ipfs-bitswap'
 import drain from 'it-drain'
-import { identity } from 'multiformats/hashes/identity'
-import { sha256, sha512 } from 'multiformats/hashes/sha2'
 import { CustomProgressEvent } from 'progress-events'
+import { bitswap, trustlessGateway } from './block-brokers/index.js'
 import { PinsImpl } from './pins.js'
 import { BlockStorage } from './storage.js'
 import { assertDatastoreVersionIsCurrent } from './utils/datastore-version.js'
+import { defaultHashers } from './utils/default-hashers.js'
 import { NetworkedStorage } from './utils/networked-storage.js'
 import type { HeliaInit } from '.'
 import type { GCOptions, Helia } from '@helia/interface'
@@ -15,7 +15,6 @@ import type { Libp2p } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
 import type { Datastore } from 'interface-datastore'
 import type { CID } from 'multiformats/cid'
-import type { MultihashHasher } from 'multiformats/hashes/interface'
 
 const log = logger('helia')
 
@@ -31,34 +30,26 @@ export class HeliaImpl implements Helia {
   public datastore: Datastore
   public pins: Pins
 
-  #bitswap?: Bitswap
-
   constructor (init: HeliaImplInit) {
-    const hashers: MultihashHasher[] = [
-      sha256,
-      sha512,
-      identity,
-      ...(init.hashers ?? [])
-    ]
-
-    this.#bitswap = createBitswap(init.libp2p, init.blockstore, {
-      hashLoader: {
-        getHasher: async (codecOrName: string | number): Promise<MultihashHasher<number>> => {
-          const hasher = hashers.find(hasher => {
-            return hasher.code === codecOrName || hasher.name === codecOrName
-          })
+    const hashers = defaultHashers(init.hashers)
 
-          if (hasher != null) {
-            return hasher
-          }
+    const components = {
+      blockstore: init.blockstore,
+      datastore: init.datastore,
+      libp2p: init.libp2p,
+      hashers
+    }
 
-          throw new Error(`Could not load hasher for code/name "${codecOrName}"`)
-        }
-      }
-    })
+    const blockBrokers = init.blockBrokers?.map((fn) => {
+      return fn(components)
+    }) ?? [
+      bitswap()(components),
+      trustlessGateway()()
+    ]
 
     const networkedStorage = new NetworkedStorage(init.blockstore, {
-      bitswap: this.#bitswap
+      blockBrokers,
+      hashers
     })
 
     this.pins = new PinsImpl(init.datastore, networkedStorage, init.dagWalkers ?? [])
@@ -72,14 +63,13 @@ export class HeliaImpl implements Helia {
 
   async start (): Promise<void> {
     await assertDatastoreVersionIsCurrent(this.datastore)
-
-    await this.#bitswap?.start()
+    await start(this.blockstore)
     await this.libp2p.start()
   }
 
   async stop (): Promise<void> {
     await this.libp2p.stop()
-    await this.#bitswap?.stop()
+    await stop(this.blockstore)
   }
 
   async gc (options: GCOptions = {}): Promise<void> {

package/src/index.ts

@@ -1,22 +1,20 @@
 /**
  * @packageDocumentation
  *
- * Create a Helia node.
+ * Exports a `createHelia` function that returns an object that implements the {@link Helia} API.
+ *
+ * Pass it to other modules like {@link https://www.npmjs.com/package/@helia/unixfs | @helia/unixfs} to make files available on the distributed web.
  *
  * @example
  *
  * ```typescript
- * import { MemoryDatastore } from 'datastore-core'
- * import { MemoryBlockstore } from 'blockstore-core'
  * import { createHelia } from 'helia'
  * import { unixfs } from '@helia/unixfs'
  * import { CID } from 'multiformats/cid'
  *
- * const node = await createHelia({
- *   blockstore: new MemoryBlockstore(),
- *   datastore: new MemoryDatastore()
- * })
- * const fs = unixfs(node)
+ * const helia = await createHelia()
+ *
+ * const fs = unixfs(helia)
  * fs.cat(CID.parse('bafyFoo'))
  * ```
  */
@@ -29,6 +27,7 @@ import { createLibp2p } from './utils/libp2p.js'
 import { name, version } from './version.js'
 import type { DefaultLibp2pServices } from './utils/libp2p-defaults.js'
 import type { Helia } from '@helia/interface'
+import type { BlockBroker } from '@helia/interface/blocks'
 import type { Libp2p } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
 import type { Datastore } from 'interface-datastore'
@@ -49,7 +48,7 @@ const log = logger('helia')
  */
 export interface DAGWalker {
   codec: number
-  walk: (block: Uint8Array) => AsyncGenerator<CID, void, undefined>
+  walk(block: Uint8Array): AsyncGenerator<CID, void, undefined>
 }
 
 /**
@@ -91,6 +90,12 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
    */
   dagWalkers?: DAGWalker[]
 
+  /**
+   * A list of strategies used to fetch blocks when they are not present in
+   * the local blockstore
+   */
+  blockBrokers?: Array<(components: any) => BlockBroker>
+
   /**
    * Pass `false` to not start the Helia node
    */

package/src/storage.ts

@@ -1,3 +1,4 @@
+import { start, stop, type Startable } from '@libp2p/interface/startable'
 import createMortice from 'mortice'
 import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions } from '@helia/interface/blocks'
 import type { Pins } from '@helia/interface/pins'
@@ -13,7 +14,7 @@ export interface BlockStorageInit {
 }
 
 export interface GetOptions extends AbortOptions {
-  progress?: (evt: Event) => void
+  progress?(evt: Event): void
 }
 
 /**
@@ -21,10 +22,11 @@ export interface GetOptions extends AbortOptions {
  * blockstore (that may be on disk, s3, or something else). If the blocks are
  * not present Bitswap will be used to fetch them from network peers.
  */
-export class BlockStorage implements Blocks {
+export class BlockStorage implements Blocks, Startable {
   public lock: Mortice
   private readonly child: Blockstore
   private readonly pins: Pins
+  private started: boolean
 
   /**
    * Create a new BlockStorage
@@ -35,6 +37,21 @@ export class BlockStorage implements Blocks {
     this.lock = createMortice({
       singleProcess: options.holdGcLock
     })
+    this.started = false
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  async start (): Promise<void> {
+    await start(this.child)
+    this.started = true
+  }
+
+  async stop (): Promise<void> {
+    await stop(this.child)
+    this.started = false
   }
 
   unwrap (): Blockstore {

package/src/utils/default-hashers.ts

@@ -0,0 +1,12 @@
+import { identity } from 'multiformats/hashes/identity'
+import { sha256, sha512 } from 'multiformats/hashes/sha2'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+
+export function defaultHashers (hashers: MultihashHasher[] = []): MultihashHasher[] {
+  return [
+    sha256,
+    sha512,
+    identity,
+    ...hashers
+  ]
+}

package/src/utils/libp2p-defaults.browser.ts

@@ -1,8 +1,8 @@
 import { gossipsub } from '@chainsafe/libp2p-gossipsub'
 import { noise } from '@chainsafe/libp2p-noise'
 import { yamux } from '@chainsafe/libp2p-yamux'
+import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client'
 import { bootstrap } from '@libp2p/bootstrap'
-import { ipniContentRouting } from '@libp2p/ipni-content-routing'
 import { type DualKadDHT, kadDHT } from '@libp2p/kad-dht'
 import { mplex } from '@libp2p/mplex'
 import { webRTC, webRTCDirect } from '@libp2p/webrtc'
@@ -21,6 +21,7 @@ import type { Libp2pOptions } from 'libp2p'
 
 export interface DefaultLibp2pServices extends Record<string, unknown> {
   dht: DualKadDHT
+  delegatedRouting: unknown
   pubsub: PubSub
   identify: IdentifyService
   autoNAT: unknown
@@ -54,14 +55,12 @@ export function libp2pDefaults (): Libp2pOptions<DefaultLibp2pServices> {
     peerDiscovery: [
       bootstrap(bootstrapConfig)
     ],
-    contentRouters: [
-      ipniContentRouting('https://cid.contact')
-    ],
     services: {
       identify: identifyService(),
       autoNAT: autoNATService(),
       pubsub: gossipsub(),
       dcutr: dcutrService(),
+      delegatedRouting: () => createDelegatedRoutingV1HttpApiClient('https://delegated-ipfs.dev'),
       dht: kadDHT({
         clientMode: true,
         validators: {

package/src/utils/libp2p-defaults.ts

@@ -1,8 +1,8 @@
 import { gossipsub } from '@chainsafe/libp2p-gossipsub'
 import { noise } from '@chainsafe/libp2p-noise'
 import { yamux } from '@chainsafe/libp2p-yamux'
+import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client'
 import { bootstrap } from '@libp2p/bootstrap'
-import { ipniContentRouting } from '@libp2p/ipni-content-routing'
 import { type DualKadDHT, kadDHT } from '@libp2p/kad-dht'
 import { mdns } from '@libp2p/mdns'
 import { mplex } from '@libp2p/mplex'
@@ -23,6 +23,7 @@ import type { Libp2pOptions } from 'libp2p'
 
 export interface DefaultLibp2pServices extends Record<string, unknown> {
   dht: DualKadDHT
+  delegatedRouting: unknown
   pubsub: PubSub
   relay: CircuitRelayService
   identify: IdentifyService
@@ -37,6 +38,7 @@ export function libp2pDefaults (): Libp2pOptions<DefaultLibp2pServices> {
     addresses: {
       listen: [
         '/ip4/0.0.0.0/tcp/0',
+        '/ip6/::/tcp/0',
         '/webrtc'
       ]
     },
@@ -60,15 +62,13 @@ export function libp2pDefaults (): Libp2pOptions<DefaultLibp2pServices> {
       mdns(),
       bootstrap(bootstrapConfig)
     ],
-    contentRouters: [
-      ipniContentRouting('https://cid.contact')
-    ],
     services: {
       identify: identifyService(),
       autoNAT: autoNATService(),
       upnp: uPnPNATService(),
       pubsub: gossipsub(),
       dcutr: dcutrService(),
+      delegatedRouting: () => createDelegatedRoutingV1HttpApiClient('https://delegated-ipfs.dev'),
       dht: kadDHT({
         validators: {
           ipns: ipnsValidator