@helia/verified-fetch 0.0.0-8a5bc6f

package/src/utils/parse-url-string.ts

@@ -0,0 +1,139 @@
+import { peerIdFromString } from '@libp2p/peer-id'
+import { CID } from 'multiformats/cid'
+import { TLRU } from './tlru.js'
+import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
+import type { ComponentLogger } from '@libp2p/interface'
+import type { ProgressOptions } from 'progress-events'
+
+const ipnsCache = new TLRU<ResolveResult>(1000)
+
+export interface ParseUrlStringInput {
+  urlString: string
+  ipns: IPNS
+  logger: ComponentLogger
+}
+export interface ParseUrlStringOptions extends ProgressOptions<ResolveProgressEvents | IPNSRoutingEvents | ResolveDnsLinkProgressEvents> {
+
+}
+
+export interface ParsedUrlStringResults {
+  protocol: string
+  path: string
+  cid: CID
+  query: Record<string, string>
+}
+
+const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/$?]+)\/?(?<path>[^$?]*)\??(?<queryString>.*)$/
+
+/**
+ * A function that parses ipfs:// and ipns:// URLs, returning an object with easily recognizable properties.
+ *
+ * After determining the protocol successfully, we process the cidOrPeerIdOrDnsLink:
+ * * If it's ipfs, it parses the CID or throws an Aggregate error
+ * * If it's ipns, it attempts to resolve the PeerId and then the DNSLink. If both fail, an Aggregate error is thrown.
+ */
+export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStringInput, options?: ParseUrlStringOptions): Promise<ParsedUrlStringResults> {
+  const log = logger.forComponent('helia:verified-fetch:parse-url-string')
+  const match = urlString.match(URL_REGEX)
+
+  if (match == null || match.groups == null) {
+    throw new TypeError(`Invalid URL: ${urlString}, please use ipfs:// or ipns:// URLs only.`)
+  }
+
+  const { protocol, cidOrPeerIdOrDnsLink, path: urlPath, queryString } = match.groups
+
+  let cid: CID | undefined
+  let resolvedPath: string | undefined
+  const errors: Error[] = []
+
+  if (protocol === 'ipfs') {
+    try {
+      cid = CID.parse(cidOrPeerIdOrDnsLink)
+    } catch (err) {
+      log.error(err)
+      errors.push(new TypeError('Invalid CID for ipfs://<cid> URL'))
+    }
+  } else {
+    let resolveResult = ipnsCache.get(cidOrPeerIdOrDnsLink)
+
+    if (resolveResult != null) {
+      cid = resolveResult.cid
+      resolvedPath = resolveResult.path
+      log.trace('resolved %s to %c from cache', cidOrPeerIdOrDnsLink, cid)
+    } else {
+      // protocol is ipns
+      log.trace('Attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink)
+      let peerId = null
+
+      try {
+        peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
+        resolveResult = await ipns.resolve(peerId, { onProgress: options?.onProgress })
+        cid = resolveResult?.cid
+        resolvedPath = resolveResult?.path
+        log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid)
+        ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
+      } catch (err) {
+        if (peerId == null) {
+          log.error('Could not parse PeerId string "%s"', cidOrPeerIdOrDnsLink, err)
+          errors.push(new TypeError(`Could not parse PeerId in ipns url "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`))
+        } else {
+          log.error('Could not resolve PeerId %c', peerId, err)
+          errors.push(new TypeError(`Could not resolve PeerId "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`))
+        }
+      }
+
+      if (cid == null) {
+        log.trace('Attempting to resolve DNSLink for %s', cidOrPeerIdOrDnsLink)
+
+        try {
+          resolveResult = await ipns.resolveDns(cidOrPeerIdOrDnsLink, { onProgress: options?.onProgress })
+          cid = resolveResult?.cid
+          resolvedPath = resolveResult?.path
+          log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid)
+          ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
+        } catch (err) {
+          log.error('Could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err)
+          errors.push(err as Error)
+        }
+      }
+    }
+  }
+
+  if (cid == null) {
+    throw new AggregateError(errors, `Invalid resource. Cannot determine CID from URL "${urlString}"`)
+  }
+
+  // parse query string
+  const query: Record<string, string> = {}
+
+  if (queryString != null && queryString.length > 0) {
+    const queryParts = queryString.split('&')
+    for (const part of queryParts) {
+      const [key, value] = part.split('=')
+      query[key] = decodeURIComponent(value)
+    }
+  }
+
+  /**
+   * join the path from resolve result & given path.
+   * e.g. /ipns/<peerId>/ that is resolved to /ipfs/<cid>/<path1>, when requested as /ipns/<peerId>/<path2>, should be
+   * resolved to /ipfs/<cid>/<path1>/<path2>
+   */
+  const pathParts = []
+
+  if (urlPath.length > 0) {
+    pathParts.push(urlPath)
+  }
+
+  if (resolvedPath != null && resolvedPath.length > 0) {
+    pathParts.push(resolvedPath)
+  }
+  const path = pathParts.join('/')
+
+  return {
+    protocol,
+    cid,
+    path,
+    query
+  }
+}

package/src/utils/tlru.ts

@@ -0,0 +1,52 @@
+import hashlru from 'hashlru'
+
+/**
+ * Time Aware Least Recent Used Cache
+ *
+ * @see https://arxiv.org/pdf/1801.00390
+ */
+export class TLRU<T> {
+  private readonly lru: ReturnType<typeof hashlru>
+
+  constructor (maxSize: number) {
+    this.lru = hashlru(maxSize)
+  }
+
+  get (key: string): T | undefined {
+    const value = this.lru.get(key)
+
+    if (value != null) {
+      if (value.expire != null && value.expire < Date.now()) {
+        this.lru.remove(key)
+
+        return undefined
+      }
+
+      return value.value
+    }
+
+    return undefined
+  }
+
+  set (key: string, value: T, ttl: number): void {
+    this.lru.set(key, { value, expire: Date.now() + ttl })
+  }
+
+  has (key: string): boolean {
+    const value = this.get(key)
+
+    if (value != null) {
+      return true
+    }
+
+    return false
+  }
+
+  remove (key: string): void {
+    this.lru.remove(key)
+  }
+
+  clear (): void {
+    this.lru.clear()
+  }
+}

package/src/utils/walk-path.ts

@@ -0,0 +1,34 @@
+import { walkPath as exporterWalk, type ExporterOptions, type ReadableStorage, type UnixFSEntry } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+
+export interface PathWalkerOptions extends ExporterOptions {
+
+}
+export interface PathWalkerResponse {
+  ipfsRoots: CID[]
+  terminalElement: UnixFSEntry
+
+}
+
+export interface PathWalkerFn {
+  (blockstore: ReadableStorage, path: string, options?: PathWalkerOptions): Promise<PathWalkerResponse>
+}
+
+export async function walkPath (blockstore: ReadableStorage, path: string, options?: PathWalkerOptions): Promise<PathWalkerResponse> {
+  const ipfsRoots: CID[] = []
+  let terminalElement: UnixFSEntry | undefined
+
+  for await (const entry of exporterWalk(path, blockstore, options)) {
+    ipfsRoots.push(entry.cid)
+    terminalElement = entry
+  }
+
+  if (terminalElement == null) {
+    throw new Error('No terminal element found')
+  }
+
+  return {
+    ipfsRoots,
+    terminalElement
+  }
+}

package/src/verified-fetch.ts

@@ -0,0 +1,323 @@
+import { dagCbor as heliaDagCbor, type DAGCBOR } from '@helia/dag-cbor'
+import { dagJson as heliaDagJson, type DAGJSON } from '@helia/dag-json'
+import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
+import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers'
+import { json as heliaJson, type JSON } from '@helia/json'
+import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs'
+import { code as dagCborCode } from '@ipld/dag-cbor'
+import { code as dagJsonCode } from '@ipld/dag-json'
+import { code as dagPbCode } from '@ipld/dag-pb'
+import { code as jsonCode } from 'multiformats/codecs/json'
+import { decode, code as rawCode } from 'multiformats/codecs/raw'
+import { CustomProgressEvent } from 'progress-events'
+import { getStreamAndContentType } from './utils/get-stream-and-content-type.js'
+import { parseResource } from './utils/parse-resource.js'
+import { walkPath, type PathWalkerFn } from './utils/walk-path.js'
+import type { CIDDetail, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
+import type { Helia } from '@helia/interface'
+import type { AbortOptions, Logger } from '@libp2p/interface'
+import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+
+interface VerifiedFetchComponents {
+  helia: Helia
+  ipns?: IPNS
+  unixfs?: HeliaUnixFs
+  dagJson?: DAGJSON
+  json?: JSON
+  dagCbor?: DAGCBOR
+  pathWalker?: PathWalkerFn
+}
+
+/**
+ * Potential future options for the VerifiedFetch constructor.
+ */
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+interface VerifiedFetchInit {
+
+}
+
+interface FetchHandlerFunctionArg {
+  cid: CID
+  path: string
+  terminalElement?: UnixFSEntry
+  options?: Omit<VerifiedFetchOptions, 'signal'> & AbortOptions
+}
+
+interface FetchHandlerFunction {
+  (options: FetchHandlerFunctionArg): Promise<Response>
+}
+
+function convertOptions (options?: VerifiedFetchOptions): (Omit<VerifiedFetchOptions, 'signal'> & AbortOptions) | undefined {
+  if (options == null) {
+    return undefined
+  }
+
+  let signal: AbortSignal | undefined
+  if (options?.signal === null) {
+    signal = undefined
+  }
+  return {
+    ...options,
+    signal
+  }
+}
+
+export class VerifiedFetch {
+  private readonly helia: Helia
+  private readonly ipns: IPNS
+  private readonly unixfs: HeliaUnixFs
+  private readonly dagJson: DAGJSON
+  private readonly dagCbor: DAGCBOR
+  private readonly json: JSON
+  private readonly pathWalker: PathWalkerFn
+  private readonly log: Logger
+
+  constructor ({ helia, ipns, unixfs, dagJson, json, dagCbor, pathWalker }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
+    this.helia = helia
+    this.log = helia.logger.forComponent('helia:verified-fetch')
+    this.ipns = ipns ?? heliaIpns(helia, {
+      resolvers: [
+        dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'),
+        dnsJsonOverHttps('https://dns.google/resolve')
+      ]
+    })
+    this.unixfs = unixfs ?? heliaUnixFs(helia)
+    this.dagJson = dagJson ?? heliaDagJson(helia)
+    this.json = json ?? heliaJson(helia)
+    this.dagCbor = dagCbor ?? heliaDagCbor(helia)
+    this.pathWalker = pathWalker ?? walkPath
+    this.log.trace('created VerifiedFetch instance')
+  }
+
+  // handle vnd.ipfs.ipns-record
+  private async handleIPNSRecord ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    const response = new Response('vnd.ipfs.ipns-record support is not implemented', { status: 501 })
+    response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
+    return response
+  }
+
+  // handle vnd.ipld.car
+  private async handleIPLDCar ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    const response = new Response('vnd.ipld.car support is not implemented', { status: 501 })
+    response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
+    return response
+  }
+
+  private async handleDagJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    this.log.trace('fetching %c/%s', cid, path)
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid: cid.toString(), path }))
+    const result = await this.dagJson.get(cid, {
+      signal: options?.signal,
+      onProgress: options?.onProgress
+    })
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: cid.toString(), path }))
+    const response = new Response(JSON.stringify(result), { status: 200 })
+    response.headers.set('content-type', 'application/json')
+    return response
+  }
+
+  private async handleJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    this.log.trace('fetching %c/%s', cid, path)
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid: cid.toString(), path }))
+    const result: Record<any, any> = await this.json.get(cid, {
+      signal: options?.signal,
+      onProgress: options?.onProgress
+    })
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: cid.toString(), path }))
+    const response = new Response(JSON.stringify(result), { status: 200 })
+    response.headers.set('content-type', 'application/json')
+    return response
+  }
+
+  private async handleDagCbor ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    this.log.trace('fetching %c/%s', cid, path)
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid: cid.toString(), path }))
+    const result = await this.dagCbor.get(cid, {
+      signal: options?.signal,
+      onProgress: options?.onProgress
+    })
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: cid.toString(), path }))
+    const response = new Response(JSON.stringify(result), { status: 200 })
+    response.headers.set('content-type', 'application/json')
+    return response
+  }
+
+  private async handleDagPb ({ cid, path, options, terminalElement }: FetchHandlerFunctionArg): Promise<Response> {
+    this.log.trace('fetching %c/%s', cid, path)
+    let resolvedCID = terminalElement?.cid ?? cid
+    let stat: UnixFSStats
+    if (terminalElement?.type === 'directory') {
+      const dirCid = terminalElement.cid
+
+      const rootFilePath = 'index.html'
+      try {
+        this.log.trace('found directory at %c/%s, looking for index.html', cid, path)
+        options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid: dirCid.toString(), path: rootFilePath }))
+        stat = await this.unixfs.stat(dirCid, {
+          path: rootFilePath,
+          signal: options?.signal,
+          onProgress: options?.onProgress
+        })
+        this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, stat.cid)
+        path = rootFilePath
+        resolvedCID = stat.cid
+        // terminalElement = stat
+      } catch (err: any) {
+        this.log('error loading path %c/%s', dirCid, rootFilePath, err)
+        return new Response('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented', { status: 501 })
+      } finally {
+        options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: dirCid.toString(), path: rootFilePath }))
+      }
+    }
+
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid: resolvedCID.toString(), path: '' }))
+    const asyncIter = this.unixfs.cat(resolvedCID, {
+      signal: options?.signal,
+      onProgress: options?.onProgress
+    })
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: resolvedCID.toString(), path: '' }))
+    this.log('got async iterator for %c/%s', cid, path)
+
+    const { contentType, stream } = await getStreamAndContentType(asyncIter, path ?? '', this.helia.logger, {
+      onProgress: options?.onProgress
+    })
+    const response = new Response(stream, { status: 200 })
+    response.headers.set('content-type', contentType)
+
+    return response
+  }
+
+  private async handleRaw ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    this.log.trace('fetching %c/%s', cid, path)
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid: cid.toString(), path }))
+    const result = await this.helia.blockstore.get(cid)
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: cid.toString(), path }))
+    const response = new Response(decode(result), { status: 200 })
+    response.headers.set('content-type', 'application/octet-stream')
+    return response
+  }
+
+  /**
+   * Determines the format requested by the client, defaults to `null` if no format is requested.
+   *
+   * @see https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter
+   * @default 'raw'
+   */
+  private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: string | null }): string | null {
+    const formatMap: Record<string, string> = {
+      'vnd.ipld.raw': 'raw',
+      'vnd.ipld.car': 'car',
+      'application/x-tar': 'tar',
+      'application/vnd.ipld.dag-json': 'dag-json',
+      'application/vnd.ipld.dag-cbor': 'dag-cbor',
+      'application/json': 'json',
+      'application/cbor': 'cbor',
+      'vnd.ipfs.ipns-record': 'ipns-record'
+    }
+
+    if (headerFormat != null) {
+      for (const format in formatMap) {
+        if (headerFormat.includes(format)) {
+          return formatMap[format]
+        }
+      }
+    } else if (queryFormat != null) {
+      return queryFormat
+    }
+
+    return null
+  }
+
+  /**
+   * Map of format to specific handlers for that format.
+   * These format handlers should adjust the response headers as specified in https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers
+   */
+  private readonly formatHandlers: Record<string, FetchHandlerFunction> = {
+    raw: async () => new Response('application/vnd.ipld.raw support is not implemented', { status: 501 }),
+    car: this.handleIPLDCar,
+    'ipns-record': this.handleIPNSRecord,
+    tar: async () => new Response('application/x-tar support is not implemented', { status: 501 }),
+    'dag-json': async () => new Response('application/vnd.ipld.dag-json support is not implemented', { status: 501 }),
+    'dag-cbor': async () => new Response('application/vnd.ipld.dag-cbor support is not implemented', { status: 501 }),
+    json: async () => new Response('application/json support is not implemented', { status: 501 }),
+    cbor: async () => new Response('application/cbor support is not implemented', { status: 501 })
+  }
+
+  private readonly codecHandlers: Record<number, FetchHandlerFunction> = {
+    [dagJsonCode]: this.handleDagJson,
+    [dagPbCode]: this.handleDagPb,
+    [jsonCode]: this.handleJson,
+    [dagCborCode]: this.handleDagCbor,
+    [rawCode]: this.handleRaw
+  }
+
+  async fetch (resource: Resource, opts?: VerifiedFetchOptions): Promise<Response> {
+    const options = convertOptions(opts)
+    const { path, query, ...rest } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options)
+    const cid = rest.cid
+    let response: Response | undefined
+
+    const format = this.getFormat({ headerFormat: new Headers(options?.headers).get('accept'), queryFormat: query.format ?? null })
+
+    if (format != null) {
+      // TODO: These should be handled last when they're returning something other than 501
+      const formatHandler = this.formatHandlers[format]
+
+      if (formatHandler != null) {
+        response = await formatHandler.call(this, { cid, path, options })
+
+        if (response.status === 501) {
+          return response
+        }
+      }
+    }
+
+    let terminalElement: UnixFSEntry | undefined
+    let ipfsRoots: CID[] | undefined
+
+    try {
+      const pathDetails = await this.pathWalker(this.helia.blockstore, `${cid.toString()}/${path}`, options)
+      ipfsRoots = pathDetails.ipfsRoots
+      terminalElement = pathDetails.terminalElement
+    } catch (err) {
+      this.log.error('Error walking path %s', path, err)
+      // return new Response(`Error walking path: ${(err as Error).message}`, { status: 500 })
+    }
+
+    if (response == null) {
+      const codecHandler = this.codecHandlers[cid.code]
+
+      if (codecHandler != null) {
+        response = await codecHandler.call(this, { cid, path, options, terminalElement })
+      } else {
+        return new Response(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`, { status: 501 })
+      }
+    }
+
+    response.headers.set('etag', cid.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
+    response.headers.set('cache-cotrol', 'public, max-age=29030400, immutable')
+    response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header
+
+    if (ipfsRoots != null) {
+      response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
+    }
+    // response.headers.set('Content-Disposition', `TODO`) // https://specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header
+
+    return response
+  }
+
+  /**
+   * Start the Helia instance
+   */
+  async start (): Promise<void> {
+    await this.helia.start()
+  }
+
+  /**
+   * Shut down the Helia instance
+   */
+  async stop (): Promise<void> {
+    await this.helia.stop()
+  }
+}