@helia/block-brokers 2.0.3 → 2.1.0-59de059

package/src/trustless-gateway/broker.ts

@@ -1,30 +1,56 @@
+import { createTrustlessGatewaySession } from './session.js'
 import { TrustlessGateway } from './trustless-gateway.js'
 import { DEFAULT_TRUSTLESS_GATEWAYS } from './index.js'
 import type { TrustlessGatewayBlockBrokerInit, TrustlessGatewayComponents, TrustlessGatewayGetBlockProgressEvents } from './index.js'
-import type { BlockRetrievalOptions, BlockRetriever } from '@helia/interface/blocks'
-import type { Logger } from '@libp2p/interface'
+import type { Routing, BlockRetrievalOptions, BlockBroker, CreateSessionOptions } from '@helia/interface'
+import type { ComponentLogger, Logger } from '@libp2p/interface'
 import type { CID } from 'multiformats/cid'
-import type { ProgressOptions } from 'progress-events'
+
+export interface CreateTrustlessGatewaySessionOptions extends CreateSessionOptions<TrustlessGatewayGetBlockProgressEvents> {
+  /**
+   * By default we will only connect to peers with HTTPS addresses, pass true
+   * to also connect to HTTP addresses.
+   *
+   * @default false
+   */
+  allowInsecure?: boolean
+
+  /**
+   * By default we will only connect to peers with public or DNS addresses, pass
+   * true to also connect to private addresses.
+   *
+   * @default false
+   */
+  allowLocal?: boolean
+}
 
 /**
  * A class that accepts a list of trustless gateways that are queried
  * for blocks.
  */
-export class TrustlessGatewayBlockBroker implements BlockRetriever<
-ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
-> {
+export class TrustlessGatewayBlockBroker implements BlockBroker<TrustlessGatewayGetBlockProgressEvents> {
+  private readonly components: TrustlessGatewayComponents
   private readonly gateways: TrustlessGateway[]
+  private readonly routing: Routing
   private readonly log: Logger
+  private readonly logger: ComponentLogger
 
   constructor (components: TrustlessGatewayComponents, init: TrustlessGatewayBlockBrokerInit = {}) {
+    this.components = components
     this.log = components.logger.forComponent('helia:trustless-gateway-block-broker')
+    this.logger = components.logger
+    this.routing = components.routing
     this.gateways = (init.gateways ?? DEFAULT_TRUSTLESS_GATEWAYS)
       .map((gatewayOrUrl) => {
-        return new TrustlessGateway(gatewayOrUrl)
+        return new TrustlessGateway(gatewayOrUrl, components.logger)
       })
   }
 
-  async retrieve (cid: CID, options: BlockRetrievalOptions<ProgressOptions<TrustlessGatewayGetBlockProgressEvents>> = {}): Promise<Uint8Array> {
+  addGateway (gatewayOrUrl: string): void {
+    this.gateways.push(new TrustlessGateway(gatewayOrUrl, this.components.logger))
+  }
+
+  async retrieve (cid: CID, options: BlockRetrievalOptions<TrustlessGatewayGetBlockProgressEvents> = {}): Promise<Uint8Array> {
     // Loop through the gateways until we get a block or run out of gateways
     // TODO: switch to toSorted when support is better
     const sortedGateways = this.gateways.sort((a, b) => b.reliability() - a.reliability())
@@ -41,7 +67,7 @@ ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
           this.log.error('failed to validate block for %c from %s', cid, gateway.url, err)
           gateway.incrementInvalidBlocks()
 
-          throw new Error(`unable to validate block for CID ${cid} from gateway ${gateway.url}`)
+          throw new Error(`Block for CID ${cid} from gateway ${gateway.url} failed validation`)
         }
 
         return block
@@ -50,7 +76,7 @@ ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
         if (err instanceof Error) {
           aggregateErrors.push(err)
         } else {
-          aggregateErrors.push(new Error(`unable to fetch raw block for CID ${cid} from gateway ${gateway.url}`))
+          aggregateErrors.push(new Error(`Unable to fetch raw block for CID ${cid} from gateway ${gateway.url}`))
         }
         // if signal was aborted, exit the loop
         if (options.signal?.aborted === true) {
@@ -60,6 +86,17 @@ ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
       }
     }
 
-    throw new AggregateError(aggregateErrors, `unable to fetch raw block for CID ${cid} from any gateway`)
+    if (aggregateErrors.length > 0) {
+      throw new AggregateError(aggregateErrors, `Unable to fetch raw block for CID ${cid} from any gateway`)
+    } else {
+      throw new Error(`Unable to fetch raw block for CID ${cid} from any gateway`)
+    }
+  }
+
+  createSession (options: CreateTrustlessGatewaySessionOptions = {}): BlockBroker<TrustlessGatewayGetBlockProgressEvents> {
+    return createTrustlessGatewaySession({
+      logger: this.logger,
+      routing: this.routing
+    }, options)
   }
 }

package/src/trustless-gateway/index.ts

@@ -1,5 +1,5 @@
 import { TrustlessGatewayBlockBroker } from './broker.js'
-import type { BlockRetriever } from '@helia/interface/src/blocks.js'
+import type { Routing, BlockBroker } from '@helia/interface'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressEvent } from 'progress-events'
 
@@ -22,9 +22,10 @@ export interface TrustlessGatewayBlockBrokerInit {
 }
 
 export interface TrustlessGatewayComponents {
+  routing: Routing
   logger: ComponentLogger
 }
 
-export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): (components: TrustlessGatewayComponents) => BlockRetriever {
+export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): (components: TrustlessGatewayComponents) => BlockBroker<TrustlessGatewayGetBlockProgressEvents> {
   return (components) => new TrustlessGatewayBlockBroker(components, init)
 }

package/src/trustless-gateway/session.ts

@@ -0,0 +1,98 @@
+import { AbstractSession } from '@helia/utils'
+import { isPrivateIp } from '@libp2p/utils/private-ip'
+import { DNS, HTTP, HTTPS } from '@multiformats/multiaddr-matcher'
+import { multiaddrToUri } from '@multiformats/multiaddr-to-uri'
+import { TrustlessGateway } from './trustless-gateway.js'
+import type { CreateTrustlessGatewaySessionOptions } from './broker.js'
+import type { TrustlessGatewayGetBlockProgressEvents } from './index.js'
+import type { BlockRetrievalOptions, Routing } from '@helia/interface'
+import type { ComponentLogger } from '@libp2p/interface'
+import type { Multiaddr } from '@multiformats/multiaddr'
+import type { AbortOptions } from 'interface-store'
+import type { CID } from 'multiformats/cid'
+
+const DEFAULT_ALLOW_INSECURE = false
+const DEFAULT_ALLOW_LOCAL = false
+
+export interface TrustlessGatewaySessionComponents {
+  logger: ComponentLogger
+  routing: Routing
+}
+
+class TrustlessGatewaySession extends AbstractSession<TrustlessGateway, TrustlessGatewayGetBlockProgressEvents> {
+  private readonly routing: Routing
+  private readonly allowInsecure: boolean
+  private readonly allowLocal: boolean
+
+  constructor (components: TrustlessGatewaySessionComponents, init: CreateTrustlessGatewaySessionOptions) {
+    super(components, {
+      ...init,
+      name: 'helia:trustless-gateway:session'
+    })
+
+    this.routing = components.routing
+    this.allowInsecure = init.allowInsecure ?? DEFAULT_ALLOW_INSECURE
+    this.allowLocal = init.allowLocal ?? DEFAULT_ALLOW_LOCAL
+  }
+
+  async queryProvider (cid: CID, provider: TrustlessGateway, options: BlockRetrievalOptions): Promise<Uint8Array> {
+    this.log('fetching BLOCK for %c from %s', cid, provider.url)
+
+    const block = await provider.getRawBlock(cid, options.signal)
+    this.log.trace('got block for %c from %s', cid, provider.url)
+
+    await options.validateFn?.(block)
+
+    return block
+  }
+
+  async * findNewProviders (cid: CID, options: AbortOptions = {}): AsyncGenerator<TrustlessGateway> {
+    for await (const provider of this.routing.findProviders(cid, options)) {
+      // require http(s) addresses
+      const httpAddresses = filterMultiaddrs(provider.multiaddrs, this.allowInsecure, this.allowLocal)
+
+      if (httpAddresses.length === 0) {
+        continue
+      }
+
+      // take first address?
+      // /ip4/x.x.x.x/tcp/31337/http
+      // /ip4/x.x.x.x/tcp/31337/https
+      // etc
+      const uri = multiaddrToUri(httpAddresses[0])
+
+      this.log('found http-gateway provider %p %s for cid %c', provider.id, uri, cid)
+      yield new TrustlessGateway(uri, this.logger)
+    }
+  }
+
+  toEvictionKey (provider: TrustlessGateway): Uint8Array | string {
+    return provider.url.toString()
+  }
+
+  equals (providerA: TrustlessGateway, providerB: TrustlessGateway): boolean {
+    return providerA.url.toString() === providerB.url.toString()
+  }
+}
+
+function filterMultiaddrs (multiaddrs: Multiaddr[], allowInsecure: boolean, allowLocal: boolean): Multiaddr[] {
+  return multiaddrs.filter(ma => {
+    if (HTTPS.matches(ma) || (allowInsecure && HTTP.matches(ma))) {
+      if (allowLocal) {
+        return true
+      }
+
+      if (DNS.matches(ma)) {
+        return true
+      }
+
+      return isPrivateIp(ma.toOptions().host) === false
+    }
+
+    return false
+  })
+}
+
+export function createTrustlessGatewaySession (components: TrustlessGatewaySessionComponents, init: CreateTrustlessGatewaySessionOptions): TrustlessGatewaySession {
+  return new TrustlessGatewaySession(components, init)
+}

package/src/trustless-gateway/trustless-gateway.ts

@@ -1,5 +1,15 @@
+import { base64 } from 'multiformats/bases/base64'
+import type { ComponentLogger, Logger } from '@libp2p/interface'
 import type { CID } from 'multiformats/cid'
 
+export interface TrustlessGatewayStats {
+  attempts: number
+  errors: number
+  invalidBlocks: number
+  successes: number
+  pendingResponses?: number
+}
+
 /**
  * A `TrustlessGateway` keeps track of the number of attempts, errors, and
  * successes for a given gateway url so that we can prioritize gateways that
@@ -36,8 +46,32 @@ export class TrustlessGateway {
    */
   #successes = 0
 
-  constructor (url: URL | string) {
+  /**
+   * A map of pending responses for this gateway. This is used to ensure that
+   * only one request per CID is made to a given gateway at a time, and that we
+   * don't make multiple in-flight requests for the same CID to the same gateway.
+   */
+  #pendingResponses = new Map<string, Promise<Uint8Array>>()
+
+  private readonly log: Logger
+
+  constructor (url: URL | string, logger: ComponentLogger) {
     this.url = url instanceof URL ? url : new URL(url)
+    this.log = logger.forComponent(`helia:trustless-gateway-block-broker:${this.url.hostname}`)
+  }
+
+  /**
+   * This function returns a unique string for the multihash.bytes of the CID.
+   *
+   * Some useful resources for why this is needed can be found using the links below:
+   *
+   * - https://github.com/ipfs/helia/pull/503#discussion_r1572451331
+   * - https://github.com/ipfs/kubo/issues/6815
+   * - https://www.notion.so/pl-strflt/Handling-ambiguity-around-CIDs-9d5e14f6516f438980b01ef188efe15d#d9d45cd1ed8b4d349b96285de4aed5ab
+   */
+  #uniqueBlockId (cid: CID): string {
+    const multihashBytes = cid.multihash.bytes
+    return base64.encode(multihashBytes)
   }
 
   /**
@@ -45,7 +79,7 @@ export class TrustlessGateway {
    * https://specs.ipfs.tech/http-gateways/trustless-gateway/
    */
   async getRawBlock (cid: CID, signal?: AbortSignal): Promise<Uint8Array> {
-    const gwUrl = this.url
+    const gwUrl = new URL(this.url.toString())
     gwUrl.pathname = `/ipfs/${cid.toString()}`
 
     // necessary as not every gateway supports dag-cbor, but every should support
@@ -56,23 +90,29 @@ export class TrustlessGateway {
       throw new Error(`Signal to fetch raw block for CID ${cid} from gateway ${this.url} was aborted prior to fetch`)
     }
 
+    const blockId = this.#uniqueBlockId(cid)
     try {
-      this.#attempts++
-      const res = await fetch(gwUrl.toString(), {
-        signal,
-        headers: {
-        // also set header, just in case ?format= is filtered out by some
-        // reverse proxy
-          Accept: 'application/vnd.ipld.raw'
-        },
-        cache: 'force-cache'
-      })
-      if (!res.ok) {
-        this.#errors++
-        throw new Error(`unable to fetch raw block for CID ${cid} from gateway ${this.url}`)
+      let pendingResponse: Promise<Uint8Array> | undefined = this.#pendingResponses.get(blockId)
+      if (pendingResponse == null) {
+        this.#attempts++
+        pendingResponse = fetch(gwUrl.toString(), {
+          signal,
+          headers: {
+            Accept: 'application/vnd.ipld.raw'
+          },
+          cache: 'force-cache'
+        }).then(async (res) => {
+          this.log('GET %s %d', gwUrl, res.status)
+          if (!res.ok) {
+            this.#errors++
+            throw new Error(`unable to fetch raw block for CID ${cid} from gateway ${this.url}`)
+          }
+          this.#successes++
+          return new Uint8Array(await res.arrayBuffer())
+        })
+        this.#pendingResponses.set(blockId, pendingResponse)
       }
-      this.#successes++
-      return new Uint8Array(await res.arrayBuffer())
+      return await pendingResponse
     } catch (cause) {
       // @ts-expect-error - TS thinks signal?.aborted can only be false now
       // because it was checked for true above.
@@ -81,6 +121,8 @@ export class TrustlessGateway {
       }
       this.#errors++
       throw new Error(`unable to fetch raw block for CID ${cid}`)
+    } finally {
+      this.#pendingResponses.delete(blockId)
     }
   }
 
@@ -123,4 +165,14 @@ export class TrustlessGateway {
   incrementInvalidBlocks (): void {
     this.#invalidBlocks++
   }
+
+  getStats (): TrustlessGatewayStats {
+    return {
+      attempts: this.#attempts,
+      errors: this.#errors,
+      invalidBlocks: this.#invalidBlocks,
+      successes: this.#successes,
+      pendingResponses: this.#pendingResponses.size
+    }
+  }
 }

package/dist/typedoc-urls.json

@@ -1,4 +0,0 @@
-{
-  "bitswap": "https://ipfs.github.io/helia/functions/_helia_block_brokers.bitswap.html",
-  "trustlessGateway": "https://ipfs.github.io/helia/functions/_helia_block_brokers.trustlessGateway.html"
-}