@helia/block-brokers 2.0.2 → 2.0.3-329652a

package/src/trustless-gateway/broker.ts

@@ -1,30 +1,67 @@
+import { DEFAULT_SESSION_MIN_PROVIDERS, DEFAULT_SESSION_MAX_PROVIDERS, DEFAULT_SESSION_PROVIDER_QUERY_CONCURRENCY, DEFAULT_SESSION_PROVIDER_QUERY_TIMEOUT } from '@helia/interface'
+import { PeerQueue } from '@libp2p/utils/peer-queue'
+import { isPrivateIp } from '@libp2p/utils/private-ip'
+import { DNS, HTTP, HTTPS } from '@multiformats/multiaddr-matcher'
+import { multiaddrToUri } from '@multiformats/multiaddr-to-uri'
+import pDefer from 'p-defer'
 import { TrustlessGateway } from './trustless-gateway.js'
 import { DEFAULT_TRUSTLESS_GATEWAYS } from './index.js'
 import type { TrustlessGatewayBlockBrokerInit, TrustlessGatewayComponents, TrustlessGatewayGetBlockProgressEvents } from './index.js'
-import type { BlockRetrievalOptions, BlockRetriever } from '@helia/interface/blocks'
+import type { Routing, BlockRetrievalOptions, BlockBroker, CreateSessionOptions } from '@helia/interface'
 import type { Logger } from '@libp2p/interface'
 import type { CID } from 'multiformats/cid'
-import type { ProgressOptions } from 'progress-events'
+
+export interface CreateTrustlessGatewaySessionOptions extends CreateSessionOptions<TrustlessGatewayGetBlockProgressEvents> {
+  /**
+   * Specify the cache control header to send to the remote. 'only-if-cached'
+   * will prevent the gateway from fetching the content if they don't have it.
+   *
+   * @default only-if-cached
+   */
+  cacheControl?: string
+
+  /**
+   * By default we will only connect to peers with HTTPS addresses, pass true
+   * to also connect to HTTP addresses.
+   *
+   * @default false
+   */
+  allowInsecure?: boolean
+
+  /**
+   * By default we will only connect to peers with public or DNS addresses, pass
+   * true to also connect to private addresses.
+   *
+   * @default false
+   */
+  allowLocal?: boolean
+}
 
 /**
  * A class that accepts a list of trustless gateways that are queried
  * for blocks.
  */
-export class TrustlessGatewayBlockBroker implements BlockRetriever<
-ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
-> {
+export class TrustlessGatewayBlockBroker implements BlockBroker<TrustlessGatewayGetBlockProgressEvents> {
+  private readonly components: TrustlessGatewayComponents
   private readonly gateways: TrustlessGateway[]
+  private readonly routing: Routing
   private readonly log: Logger
 
   constructor (components: TrustlessGatewayComponents, init: TrustlessGatewayBlockBrokerInit = {}) {
+    this.components = components
     this.log = components.logger.forComponent('helia:trustless-gateway-block-broker')
+    this.routing = components.routing
     this.gateways = (init.gateways ?? DEFAULT_TRUSTLESS_GATEWAYS)
       .map((gatewayOrUrl) => {
-        return new TrustlessGateway(gatewayOrUrl)
+        return new TrustlessGateway(gatewayOrUrl, components.logger)
       })
   }
 
-  async retrieve (cid: CID, options: BlockRetrievalOptions<ProgressOptions<TrustlessGatewayGetBlockProgressEvents>> = {}): Promise<Uint8Array> {
+  addGateway (gatewayOrUrl: string): void {
+    this.gateways.push(new TrustlessGateway(gatewayOrUrl, this.components.logger))
+  }
+
+  async retrieve (cid: CID, options: BlockRetrievalOptions<TrustlessGatewayGetBlockProgressEvents> = {}): Promise<Uint8Array> {
     // Loop through the gateways until we get a block or run out of gateways
     // TODO: switch to toSorted when support is better
     const sortedGateways = this.gateways.sort((a, b) => b.reliability() - a.reliability())
@@ -41,7 +78,7 @@ ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
           this.log.error('failed to validate block for %c from %s', cid, gateway.url, err)
           gateway.incrementInvalidBlocks()
 
-          throw new Error(`unable to validate block for CID ${cid} from gateway ${gateway.url}`)
+          throw new Error(`Block for CID ${cid} from gateway ${gateway.url} failed validation`)
         }
 
         return block
@@ -50,7 +87,7 @@ ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
         if (err instanceof Error) {
           aggregateErrors.push(err)
         } else {
-          aggregateErrors.push(new Error(`unable to fetch raw block for CID ${cid} from gateway ${gateway.url}`))
+          aggregateErrors.push(new Error(`Unable to fetch raw block for CID ${cid} from gateway ${gateway.url}`))
         }
         // if signal was aborted, exit the loop
         if (options.signal?.aborted === true) {
@@ -60,6 +97,106 @@ ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
       }
     }
 
-    throw new AggregateError(aggregateErrors, `unable to fetch raw block for CID ${cid} from any gateway`)
+    if (aggregateErrors.length > 0) {
+      throw new AggregateError(aggregateErrors, `Unable to fetch raw block for CID ${cid} from any gateway`)
+    } else {
+      throw new Error(`Unable to fetch raw block for CID ${cid} from any gateway`)
+    }
+  }
+
+  async createSession (root: CID, options: CreateTrustlessGatewaySessionOptions = {}): Promise<BlockBroker<TrustlessGatewayGetBlockProgressEvents>> {
+    const gateways: string[] = []
+    const minProviders = options.minProviders ?? DEFAULT_SESSION_MIN_PROVIDERS
+    const maxProviders = options.minProviders ?? DEFAULT_SESSION_MAX_PROVIDERS
+    const deferred = pDefer<BlockBroker<TrustlessGatewayGetBlockProgressEvents>>()
+    const broker = new TrustlessGatewayBlockBroker(this.components, {
+      gateways
+    })
+
+    this.log('finding transport-ipfs-gateway-http providers for cid %c', root)
+
+    const queue = new PeerQueue({
+      concurrency: options.providerQueryConcurrency ?? DEFAULT_SESSION_PROVIDER_QUERY_CONCURRENCY
+    })
+
+    Promise.resolve().then(async () => {
+      for await (const provider of this.routing.findProviders(root, options)) {
+        const httpAddresses = provider.multiaddrs.filter(ma => {
+          if (HTTPS.matches(ma) || (options.allowInsecure === true && HTTP.matches(ma))) {
+            if (options.allowLocal === true) {
+              return true
+            }
+
+            if (DNS.matches(ma)) {
+              return true
+            }
+
+            return isPrivateIp(ma.toOptions().host) === false
+          }
+
+          return false
+        })
+
+        if (httpAddresses.length === 0) {
+          continue
+        }
+
+        this.log('found transport-ipfs-gateway-http provider %p for cid %c', provider.id, root)
+
+        void queue.add(async () => {
+          for (const ma of httpAddresses) {
+            let uri: string | undefined
+
+            try {
+              // /ip4/x.x.x.x/tcp/31337/http
+              // /ip4/x.x.x.x/tcp/31337/https
+              // etc
+              uri = multiaddrToUri(ma)
+
+              const resource = `${uri}/ipfs/${root.toString()}?format=raw`
+
+              // make sure the peer is available - HEAD support doesn't seem to
+              // be very widely implemented so as long as the remote responds
+              // we are happy they are valid
+              // https://specs.ipfs.tech/http-gateways/trustless-gateway/#head-ipfs-cid-path-params
+
+              // in the future we should be able to request `${uri}/.well-known/libp2p-http
+              // and discover an IPFS gateway from $.protocols['/ipfs/gateway'].path
+              // in the response
+              // https://github.com/libp2p/specs/pull/508/files
+              const response = await fetch(resource, {
+                method: 'HEAD',
+                headers: {
+                  Accept: 'application/vnd.ipld.raw',
+                  'Cache-Control': options.cacheControl ?? 'only-if-cached'
+                },
+                signal: AbortSignal.timeout(options.providerQueryTimeout ?? DEFAULT_SESSION_PROVIDER_QUERY_TIMEOUT)
+              })
+
+              this.log('HEAD %s %d', resource, response.status)
+              gateways.push(uri)
+              broker.addGateway(uri)
+
+              this.log('found %d transport-ipfs-gateway-http providers for cid %c', gateways.length, root)
+
+              if (gateways.length === minProviders) {
+                deferred.resolve(broker)
+              }
+
+              if (gateways.length === maxProviders) {
+                queue.clear()
+              }
+            } catch (err: any) {
+              this.log.error('could not fetch %c from %a', root, uri ?? ma, err)
+            }
+          }
+        })
+      }
+    })
+      .catch(err => {
+        this.log.error('error creating session for %c', root, err)
+      })
+
+    return deferred.promise
   }
 }

package/src/trustless-gateway/index.ts

@@ -1,5 +1,5 @@
 import { TrustlessGatewayBlockBroker } from './broker.js'
-import type { BlockRetriever } from '@helia/interface/src/blocks.js'
+import type { Routing, BlockBroker } from '@helia/interface'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressEvent } from 'progress-events'
 
@@ -22,9 +22,10 @@ export interface TrustlessGatewayBlockBrokerInit {
 }
 
 export interface TrustlessGatewayComponents {
+  routing: Routing
   logger: ComponentLogger
 }
 
-export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): (components: TrustlessGatewayComponents) => BlockRetriever {
+export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): (components: TrustlessGatewayComponents) => BlockBroker<TrustlessGatewayGetBlockProgressEvents> {
   return (components) => new TrustlessGatewayBlockBroker(components, init)
 }

package/src/trustless-gateway/trustless-gateway.ts

@@ -1,3 +1,4 @@
+import type { ComponentLogger, Logger } from '@libp2p/interface'
 import type { CID } from 'multiformats/cid'
 
 /**
@@ -36,8 +37,11 @@ export class TrustlessGateway {
    */
   #successes = 0
 
-  constructor (url: URL | string) {
+  private readonly log: Logger
+
+  constructor (url: URL | string, logger: ComponentLogger) {
     this.url = url instanceof URL ? url : new URL(url)
+    this.log = logger.forComponent(`helia:trustless-gateway-block-broker:${this.url.hostname}`)
   }
 
   /**
@@ -67,6 +71,9 @@ export class TrustlessGateway {
         },
         cache: 'force-cache'
       })
+
+      this.log('GET %s %d', gwUrl, res.status)
+
       if (!res.ok) {
         this.#errors++
         throw new Error(`unable to fetch raw block for CID ${cid} from gateway ${this.url}`)

package/dist/typedoc-urls.json

@@ -1,4 +0,0 @@
-{
-  "bitswap": "https://ipfs.github.io/helia/functions/_helia_block_brokers.bitswap.html",
-  "trustlessGateway": "https://ipfs.github.io/helia/functions/_helia_block_brokers.trustlessGateway.html"
-}