@helia/block-brokers 0.0.0

package/src/trustless-gateway/broker.ts

@@ -0,0 +1,65 @@
+import { TrustlessGateway } from './trustless-gateway.js'
+import { DEFAULT_TRUSTLESS_GATEWAYS } from './index.js'
+import type { TrustlessGatewayBlockBrokerInit, TrustlessGatewayComponents, TrustlessGatewayGetBlockProgressEvents } from './index.js'
+import type { BlockRetrievalOptions, BlockRetriever } from '@helia/interface/blocks'
+import type { Logger } from '@libp2p/interface'
+import type { CID } from 'multiformats/cid'
+import type { ProgressOptions } from 'progress-events'
+
+/**
+ * A class that accepts a list of trustless gateways that are queried
+ * for blocks.
+ */
+export class TrustlessGatewayBlockBroker implements BlockRetriever<
+ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
+> {
+  private readonly gateways: TrustlessGateway[]
+  private readonly log: Logger
+
+  constructor (components: TrustlessGatewayComponents, init: TrustlessGatewayBlockBrokerInit = {}) {
+    this.log = components.logger.forComponent('helia:trustless-gateway-block-broker')
+    this.gateways = (init.gateways ?? DEFAULT_TRUSTLESS_GATEWAYS)
+      .map((gatewayOrUrl) => {
+        return new TrustlessGateway(gatewayOrUrl)
+      })
+  }
+
+  async retrieve (cid: CID, options: BlockRetrievalOptions<ProgressOptions<TrustlessGatewayGetBlockProgressEvents>> = {}): Promise<Uint8Array> {
+    // Loop through the gateways until we get a block or run out of gateways
+    // TODO: switch to toSorted when support is better
+    const sortedGateways = this.gateways.sort((a, b) => b.reliability() - a.reliability())
+    const aggregateErrors: Error[] = []
+
+    for (const gateway of sortedGateways) {
+      this.log('getting block for %c from %s', cid, gateway.url)
+      try {
+        const block = await gateway.getRawBlock(cid, options.signal)
+        this.log.trace('got block for %c from %s', cid, gateway.url)
+        try {
+          await options.validateFn?.(block)
+        } catch (err) {
+          this.log.error('failed to validate block for %c from %s', cid, gateway.url, err)
+          gateway.incrementInvalidBlocks()
+
+          throw new Error(`unable to validate block for CID ${cid} from gateway ${gateway.url}`)
+        }
+
+        return block
+      } catch (err: unknown) {
+        this.log.error('failed to get block for %c from %s', cid, gateway.url, err)
+        if (err instanceof Error) {
+          aggregateErrors.push(err)
+        } else {
+          aggregateErrors.push(new Error(`unable to fetch raw block for CID ${cid} from gateway ${gateway.url}`))
+        }
+        // if signal was aborted, exit the loop
+        if (options.signal?.aborted === true) {
+          this.log.trace('request aborted while fetching raw block for CID %c from gateway %s', cid, gateway.url)
+          break
+        }
+      }
+    }
+
+    throw new AggregateError(aggregateErrors, `unable to fetch raw block for CID ${cid} from any gateway`)
+  }
+}

package/src/trustless-gateway/index.ts

@@ -0,0 +1,33 @@
+import { TrustlessGatewayBlockBroker } from './broker.js'
+import type { BlockRetriever } from '@helia/interface/src/blocks.js'
+import type { ComponentLogger } from '@libp2p/interface'
+import type { ProgressEvent } from 'progress-events'
+
+export const DEFAULT_TRUSTLESS_GATEWAYS = [
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://dweb.link',
+
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://cf-ipfs.com',
+
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://4everland.io',
+
+  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
+  'https://w3s.link'
+]
+
+export type TrustlessGatewayGetBlockProgressEvents =
+  ProgressEvent<'trustless-gateway:get-block:fetch', URL>
+
+export interface TrustlessGatewayBlockBrokerInit {
+  gateways?: Array<string | URL>
+}
+
+export interface TrustlessGatewayComponents {
+  logger: ComponentLogger
+}
+
+export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): (components: TrustlessGatewayComponents) => BlockRetriever {
+  return (components) => new TrustlessGatewayBlockBroker(components, init)
+}

package/src/trustless-gateway/trustless-gateway.ts

@@ -0,0 +1,126 @@
+import type { CID } from 'multiformats/cid'
+
+/**
+ * A `TrustlessGateway` keeps track of the number of attempts, errors, and
+ * successes for a given gateway url so that we can prioritize gateways that
+ * have been more reliable in the past, and ensure that requests are distributed
+ * across all gateways within a given `TrustlessGatewayBlockBroker` instance.
+ */
+export class TrustlessGateway {
+  public readonly url: URL
+  /**
+   * The number of times this gateway has been attempted to be used to fetch a
+   * block. This includes successful, errored, and aborted attempts. By counting
+   * even aborted attempts, slow gateways that are out-raced by others will be
+   * considered less reliable.
+   */
+  #attempts = 0
+
+  /**
+   * The number of times this gateway has errored while attempting to fetch a
+   * block. This includes `response.ok === false` and any other errors that
+   * throw while attempting to fetch a block. This does not include aborted
+   * attempts.
+   */
+  #errors = 0
+
+  /**
+   * The number of times this gateway has returned an invalid block. A gateway
+   * that returns the wrong blocks for a CID should be considered for removal
+   * from the list of gateways to fetch blocks from.
+   */
+  #invalidBlocks = 0
+
+  /**
+   * The number of times this gateway has successfully fetched a block.
+   */
+  #successes = 0
+
+  constructor (url: URL | string) {
+    this.url = url instanceof URL ? url : new URL(url)
+  }
+
+  /**
+   * Fetch a raw block from `this.url` following the specification defined at
+   * https://specs.ipfs.tech/http-gateways/trustless-gateway/
+   */
+  async getRawBlock (cid: CID, signal?: AbortSignal): Promise<Uint8Array> {
+    const gwUrl = this.url
+    gwUrl.pathname = `/ipfs/${cid.toString()}`
+
+    // necessary as not every gateway supports dag-cbor, but every should support
+    // sending raw block as-is
+    gwUrl.search = '?format=raw'
+
+    if (signal?.aborted === true) {
+      throw new Error(`Signal to fetch raw block for CID ${cid} from gateway ${this.url} was aborted prior to fetch`)
+    }
+
+    try {
+      this.#attempts++
+      const res = await fetch(gwUrl.toString(), {
+        signal,
+        headers: {
+        // also set header, just in case ?format= is filtered out by some
+        // reverse proxy
+          Accept: 'application/vnd.ipld.raw'
+        },
+        cache: 'force-cache'
+      })
+      if (!res.ok) {
+        this.#errors++
+        throw new Error(`unable to fetch raw block for CID ${cid} from gateway ${this.url}`)
+      }
+      this.#successes++
+      return new Uint8Array(await res.arrayBuffer())
+    } catch (cause) {
+      // @ts-expect-error - TS thinks signal?.aborted can only be false now
+      // because it was checked for true above.
+      if (signal?.aborted === true) {
+        throw new Error(`fetching raw block for CID ${cid} from gateway ${this.url} was aborted`)
+      }
+      this.#errors++
+      throw new Error(`unable to fetch raw block for CID ${cid}`)
+    }
+  }
+
+  /**
+   * Encapsulate the logic for determining whether a gateway is considered
+   * reliable, for prioritization. This is based on the number of successful attempts made
+   * and the number of errors encountered.
+   *
+   * Unused gateways have 100% reliability; They will be prioritized over
+   * gateways with a 100% success rate to ensure that we attempt all gateways.
+   */
+  reliability (): number {
+    /**
+     * if we have never tried to use this gateway, it is considered the most
+     * reliable until we determine otherwise (prioritize unused gateways)
+     */
+    if (this.#attempts === 0) {
+      return 1
+    }
+
+    if (this.#invalidBlocks > 0) {
+      // this gateway may not be trustworthy..
+      return -Infinity
+    }
+
+    /**
+     * We have attempted the gateway, so we need to calculate the reliability
+     * based on the number of attempts, errors, and successes. Gateways that
+     * return a single error should drop their reliability score more than a
+     * single success increases it.
+     *
+     * Play around with the below reliability function at https://www.desmos.com/calculator/d6hfhf5ukm
+     */
+    return this.#successes / (this.#attempts + (this.#errors * 3))
+  }
+
+  /**
+   * Increment the number of invalid blocks returned by this gateway.
+   */
+  incrementInvalidBlocks (): void {
+    this.#invalidBlocks++
+  }
+}

package/src/utils/default-hashers.ts

@@ -0,0 +1,12 @@
+import { identity } from 'multiformats/hashes/identity'
+import { sha256, sha512 } from 'multiformats/hashes/sha2'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+
+export function defaultHashers (hashers: MultihashHasher[] = []): MultihashHasher[] {
+  return [
+    sha256,
+    sha512,
+    identity,
+    ...hashers
+  ]
+}

package/src/utils/networked-storage.ts

@@ -0,0 +1,266 @@
+import { CodeError, start, stop } from '@libp2p/interface'
+import { anySignal } from 'any-signal'
+import filter from 'it-filter'
+import forEach from 'it-foreach'
+import { CustomProgressEvent, type ProgressOptions } from 'progress-events'
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
+import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetriever, BlockAnnouncer, BlockRetrievalOptions } from '@helia/interface/blocks'
+import type { AbortOptions, ComponentLogger, Logger, LoggerOptions, Startable } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { AwaitIterable } from 'interface-store'
+import type { CID } from 'multiformats/cid'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+
+export interface NetworkedStorageStorageInit {
+  blockBrokers?: BlockBroker[]
+  hashers?: MultihashHasher[]
+}
+
+export interface GetOptions extends AbortOptions {
+  progress?(evt: Event): void
+}
+
+function isBlockRetriever (b: any): b is BlockRetriever {
+  return typeof b.retrieve === 'function'
+}
+
+function isBlockAnnouncer (b: any): b is BlockAnnouncer {
+  return typeof b.announce === 'function'
+}
+
+export interface NetworkedStorageComponents {
+  blockstore: Blockstore
+  logger: ComponentLogger
+}
+
+/**
+ * Networked storage wraps a regular blockstore - when getting blocks if the
+ * blocks are not present Bitswap will be used to fetch them from network peers.
+ */
+export class NetworkedStorage implements Blocks, Startable {
+  private readonly child: Blockstore
+  private readonly blockRetrievers: BlockRetriever[]
+  private readonly blockAnnouncers: BlockAnnouncer[]
+  private readonly hashers: MultihashHasher[]
+  private started: boolean
+  private readonly log: Logger
+
+  /**
+   * Create a new BlockStorage
+   */
+  constructor (components: NetworkedStorageComponents, init: NetworkedStorageStorageInit) {
+    this.log = components.logger.forComponent('helia:networked-storage')
+    this.child = components.blockstore
+    this.blockRetrievers = (init.blockBrokers ?? []).filter(isBlockRetriever)
+    this.blockAnnouncers = (init.blockBrokers ?? []).filter(isBlockAnnouncer)
+    this.hashers = init.hashers ?? []
+    this.started = false
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  async start (): Promise<void> {
+    await start(this.child, ...new Set([...this.blockRetrievers, ...this.blockAnnouncers]))
+    this.started = true
+  }
+
+  async stop (): Promise<void> {
+    await stop(this.child, ...new Set([...this.blockRetrievers, ...this.blockAnnouncers]))
+    this.started = false
+  }
+
+  unwrap (): Blockstore {
+    return this.child
+  }
+
+  /**
+   * Put a block to the underlying datastore
+   */
+  async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
+    if (await this.child.has(cid)) {
+      options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:duplicate', cid))
+      return cid
+    }
+
+    options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:providers:notify', cid))
+
+    this.blockAnnouncers.forEach(provider => {
+      provider.announce(cid, block, options)
+    })
+
+    options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:blockstore:put', cid))
+
+    return this.child.put(cid, block, options)
+  }
+
+  /**
+   * Put a multiple blocks to the underlying datastore
+   */
+  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+    const missingBlocks = filter(blocks, async ({ cid }): Promise<boolean> => {
+      const has = await this.child.has(cid)
+
+      if (has) {
+        options.onProgress?.(new CustomProgressEvent<CID>('blocks:put-many:duplicate', cid))
+      }
+
+      return !has
+    })
+
+    const notifyEach = forEach(missingBlocks, ({ cid, block }): void => {
+      options.onProgress?.(new CustomProgressEvent<CID>('blocks:put-many:providers:notify', cid))
+      this.blockAnnouncers.forEach(provider => {
+        provider.announce(cid, block, options)
+      })
+    })
+
+    options.onProgress?.(new CustomProgressEvent('blocks:put-many:blockstore:put-many'))
+    yield * this.child.putMany(notifyEach, options)
+  }
+
+  /**
+   * Get a block by cid
+   */
+  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+    if (options.offline !== true && !(await this.child.has(cid))) {
+      // we do not have the block locally, get it from a block provider
+      options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:get', cid))
+      const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers, {
+        ...options,
+        log: this.log
+      })
+      options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:blockstore:put', cid))
+      await this.child.put(cid, block, options)
+
+      // notify other block providers of the new block
+      options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:notify', cid))
+      this.blockAnnouncers.forEach(provider => {
+        provider.announce(cid, block, options)
+      })
+
+      return block
+    }
+
+    options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:blockstore:get', cid))
+
+    return this.child.get(cid, options)
+  }
+
+  /**
+   * Get multiple blocks back from an (async) iterable of cids
+   */
+  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+    options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:get-many'))
+
+    yield * this.child.getMany(forEach(cids, async (cid): Promise<void> => {
+      if (options.offline !== true && !(await this.child.has(cid))) {
+        // we do not have the block locally, get it from a block provider
+        options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:get', cid))
+        const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers, {
+          ...options,
+          log: this.log
+        })
+        options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:blockstore:put', cid))
+        await this.child.put(cid, block, options)
+
+        // notify other block providers of the new block
+        options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:notify', cid))
+        this.blockAnnouncers.forEach(provider => {
+          provider.announce(cid, block, options)
+        })
+      }
+    }))
+  }
+
+  /**
+   * Delete a block from the blockstore
+   */
+  async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
+    options.onProgress?.(new CustomProgressEvent<CID>('blocks:delete:blockstore:delete', cid))
+
+    await this.child.delete(cid, options)
+  }
+
+  /**
+   * Delete multiple blocks from the blockstore
+   */
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+    options.onProgress?.(new CustomProgressEvent('blocks:delete-many:blockstore:delete-many'))
+    yield * this.child.deleteMany((async function * (): AsyncGenerator<CID> {
+      for await (const cid of cids) {
+        yield cid
+      }
+    }()), options)
+  }
+
+  async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
+    return this.child.has(cid, options)
+  }
+
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AwaitIterable<Pair> {
+    options.onProgress?.(new CustomProgressEvent('blocks:get-all:blockstore:get-many'))
+    yield * this.child.getAll(options)
+  }
+}
+
+export const getCidBlockVerifierFunction = (cid: CID, hashers: MultihashHasher[]): Required<BlockRetrievalOptions>['validateFn'] => {
+  const hasher = hashers.find(hasher => hasher.code === cid.multihash.code)
+
+  if (hasher == null) {
+    throw new CodeError(`No hasher configured for multihash code 0x${cid.multihash.code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`, 'ERR_UNKNOWN_HASH_ALG')
+  }
+
+  return async (block: Uint8Array): Promise<void> => {
+    // verify block
+    const hash = await hasher.digest(block)
+
+    if (!uint8ArrayEquals(hash.digest, cid.multihash.digest)) {
+      // if a hash mismatch occurs for a TrustlessGatewayBlockBroker, we should try another gateway
+      throw new CodeError('Hash of downloaded block did not match multihash from passed CID', 'ERR_HASH_MISMATCH')
+    }
+  }
+}
+
+/**
+ * Race block providers cancelling any pending requests once the block has been
+ * found.
+ */
+async function raceBlockRetrievers (cid: CID, providers: BlockRetriever[], hashers: MultihashHasher[], options: AbortOptions & LoggerOptions): Promise<Uint8Array> {
+  const validateFn = getCidBlockVerifierFunction(cid, hashers)
+
+  const controller = new AbortController()
+  const signal = anySignal([controller.signal, options.signal])
+
+  try {
+    return await Promise.any(
+      providers.map(async provider => {
+        try {
+          let blocksWereValidated = false
+          const block = await provider.retrieve(cid, {
+            ...options,
+            signal,
+            validateFn: async (block: Uint8Array): Promise<void> => {
+              await validateFn(block)
+              blocksWereValidated = true
+            }
+          })
+
+          if (!blocksWereValidated) {
+            // the blockBroker either did not throw an error when attempting to validate the block
+            // or did not call the validateFn at all. We should validate the block ourselves
+            await validateFn(block)
+          }
+
+          return block
+        } catch (err) {
+          options.log.error('could not retrieve verified block for %c', cid, err)
+          throw err
+        }
+      })
+    )
+  } finally {
+    signal.clear()
+  }
+}