@helia/verified-fetch 0.0.0-8db7792 → 0.0.0-a04e041

package/src/verified-fetch.ts

@@ -1,18 +1,23 @@
+import { car } from '@helia/car'
 import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
 import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers'
 import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs'
-import { code as dagCborCode } from '@ipld/dag-cbor'
-import { code as dagJsonCode } from '@ipld/dag-json'
+import * as ipldDagCbor from '@ipld/dag-cbor'
+import * as ipldDagJson from '@ipld/dag-json'
 import { code as dagPbCode } from '@ipld/dag-pb'
+import toBrowserReadableStream from 'it-to-browser-readablestream'
 import { code as jsonCode } from 'multiformats/codecs/json'
 import { code as rawCode } from 'multiformats/codecs/raw'
 import { identity } from 'multiformats/hashes/identity'
 import { CustomProgressEvent } from 'progress-events'
 import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js'
+import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js'
 import { getETag } from './utils/get-e-tag.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { parseResource } from './utils/parse-resource.js'
-import { walkPath, type PathWalkerFn } from './utils/walk-path.js'
+import { notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
+import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js'
+import { walkPath } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
 import type { RequestFormatShorthand } from './types.js'
 import type { Helia } from '@helia/interface'
@@ -24,7 +29,6 @@ interface VerifiedFetchComponents {
   helia: Helia
   ipns?: IPNS
   unixfs?: HeliaUnixFs
-  pathWalker?: PathWalkerFn
 }
 
 /**
@@ -37,8 +41,13 @@ interface VerifiedFetchInit {
 interface FetchHandlerFunctionArg {
   cid: CID
   path: string
-  terminalElement?: UnixFSEntry
   options?: Omit<VerifiedFetchOptions, 'signal'> & AbortOptions
+
+  /**
+   * If present, the user has sent an accept header with this value - if the
+   * content cannot be represented in this format a 406 should be returned
+   */
+  accept?: string
 }
 
 interface FetchHandlerFunction {
@@ -60,29 +69,48 @@ function convertOptions (options?: VerifiedFetchOptions): (Omit<VerifiedFetchOpt
   }
 }
 
-function okResponse (body?: BodyInit | null): Response {
-  return new Response(body, {
-    status: 200,
-    statusText: 'OK'
-  })
-}
+/**
+ * These are Accept header values that will cause content type sniffing to be
+ * skipped and set to these values.
+ */
+const RAW_HEADERS = [
+  'application/vnd.ipld.raw',
+  'application/octet-stream'
+]
 
-function notSupportedResponse (body?: BodyInit | null): Response {
-  return new Response(body, {
-    status: 501,
-    statusText: 'Not Implemented'
-  })
+/**
+ * if the user has specified an `Accept` header, and it's in our list of
+ * allowable "raw" format headers, use that instead of detecting the content
+ * type. This avoids the user from receiving something different when they
+ * signal that they want to `Accept` a specific mime type.
+ */
+function getOverridenRawContentType (headers?: HeadersInit): string | undefined {
+  const acceptHeader = new Headers(headers).get('accept') ?? ''
+
+  // e.g. "Accept: text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8"
+  const acceptHeaders = acceptHeader.split(',')
+    .map(s => s.split(';')[0])
+    .map(s => s.trim())
+
+  for (const mimeType of acceptHeaders) {
+    if (mimeType === '*/*') {
+      return
+    }
+
+    if (RAW_HEADERS.includes(mimeType ?? '')) {
+      return mimeType
+    }
+  }
 }
 
 export class VerifiedFetch {
   private readonly helia: Helia
   private readonly ipns: IPNS
   private readonly unixfs: HeliaUnixFs
-  private readonly pathWalker: PathWalkerFn
   private readonly log: Logger
   private readonly contentTypeParser: ContentTypeParser | undefined
 
-  constructor ({ helia, ipns, unixfs, pathWalker }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
+  constructor ({ helia, ipns, unixfs }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
     this.helia = helia
     this.log = helia.logger.forComponent('helia:verified-fetch')
     this.ipns = ipns ?? heliaIpns(helia, {
@@ -92,60 +120,128 @@ export class VerifiedFetch {
       ]
     })
     this.unixfs = unixfs ?? heliaUnixFs(helia)
-    this.pathWalker = pathWalker ?? walkPath
     this.contentTypeParser = init?.contentTypeParser
     this.log.trace('created VerifiedFetch instance')
   }
 
-  // handle vnd.ipfs.ipns-record
-  private async handleIPNSRecord ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
-    const response = notSupportedResponse('vnd.ipfs.ipns-record support is not implemented')
-    response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
-    return response
+  /**
+   * Accepts an `ipns://...` URL as a string and returns a `Response` containing
+   * a raw IPNS record.
+   */
+  private async handleIPNSRecord (resource: string, opts?: VerifiedFetchOptions): Promise<Response> {
+    return notSupportedResponse('vnd.ipfs.ipns-record support is not implemented')
   }
 
-  // handle vnd.ipld.car
-  private async handleIPLDCar ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
-    const response = notSupportedResponse('vnd.ipld.car support is not implemented')
-    response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
+  /**
+   * Accepts a `CID` and returns a `Response` with a body stream that is a CAR
+   * of the `DAG` referenced by the `CID`.
+   */
+  private async handleCar ({ cid, options }: FetchHandlerFunctionArg): Promise<Response> {
+    const c = car(this.helia)
+    const stream = toBrowserReadableStream(c.stream(cid, options))
+
+    const response = okResponse(stream)
+    response.headers.set('content-type', 'application/vnd.ipld.car; version=1')
+
     return response
   }
 
-  private async handleJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+  /**
+   * Accepts a UnixFS `CID` and returns a `.tar` file containing the file or
+   * directory structure referenced by the `CID`.
+   */
+  private async handleTar ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    if (cid.code !== dagPbCode) {
+      return notAcceptableResponse('only dag-pb CIDs can be returned in TAR files')
+    }
+
+    return notSupportedResponse('application/tar support is not implemented')
+  }
+
+  private async handleJson ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
     this.log.trace('fetching %c/%s', cid, path)
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid, path }))
-    const result = await this.helia.blockstore.get(cid, {
-      signal: options?.signal,
-      onProgress: options?.onProgress
-    })
-    const response = okResponse(result)
-    response.headers.set('content-type', 'application/json')
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid, path }))
+    const block = await this.helia.blockstore.get(cid, options)
+    let body: string | Uint8Array
+
+    if (accept === 'application/vnd.ipld.dag-cbor' || accept === 'application/cbor') {
+      try {
+        // if vnd.ipld.dag-cbor has been specified, convert to the format - note
+        // that this supports more data types than regular JSON, the content-type
+        // response header is set so the user knows to process it differently
+        const obj = ipldDagJson.decode(block)
+        body = ipldDagCbor.encode(obj)
+      } catch (err) {
+        this.log.error('could not transform %c to application/vnd.ipld.dag-cbor', err)
+        return notAcceptableResponse()
+      }
+    } else {
+      // skip decoding
+      body = block
+    }
+
+    const response = okResponse(body)
+    response.headers.set('content-type', accept ?? 'application/json')
     return response
   }
 
-  private async handleDagCbor ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleDagCbor ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
     this.log.trace('fetching %c/%s', cid, path)
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid, path }))
-    // return body as binary
-    const block = await this.helia.blockstore.get(cid)
+
+    const block = await this.helia.blockstore.get(cid, options)
     let body: string | Uint8Array
 
-    try {
-      body = dagCborToSafeJSON(block)
-    } catch (err) {
-      this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err)
+    if (accept === 'application/octet-stream' || accept === 'application/vnd.ipld.dag-cbor' || accept === 'application/cbor') {
+      // skip decoding
       body = block
+    } else if (accept === 'application/vnd.ipld.dag-json') {
+      try {
+        // if vnd.ipld.dag-json has been specified, convert to the format - note
+        // that this supports more data types than regular JSON, the content-type
+        // response header is set so the user knows to process it differently
+        const obj = ipldDagCbor.decode(block)
+        body = ipldDagJson.encode(obj)
+      } catch (err) {
+        this.log.error('could not transform %c to application/vnd.ipld.dag-json', err)
+        return notAcceptableResponse()
+      }
+    } else {
+      try {
+        body = dagCborToSafeJSON(block)
+      } catch (err) {
+        if (accept === 'application/json') {
+          this.log('could not decode DAG-CBOR as JSON-safe, but the client sent "Accept: application/json"', err)
+
+          return notAcceptableResponse()
+        }
+
+        this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err)
+        body = block
+      }
     }
 
     const response = okResponse(body)
-    response.headers.set('content-type', body instanceof Uint8Array ? 'application/octet-stream' : 'application/json')
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid, path }))
+
+    if (accept == null) {
+      accept = body instanceof Uint8Array ? 'application/octet-stream' : 'application/json'
+    }
+
+    response.headers.set('content-type', accept)
+
     return response
   }
 
-  private async handleDagPb ({ cid, path, options, terminalElement }: FetchHandlerFunctionArg): Promise<Response> {
-    this.log.trace('fetching %c/%s', cid, path)
+  private async handleDagPb ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    let terminalElement: UnixFSEntry | undefined
+    let ipfsRoots: CID[] | undefined
+
+    try {
+      const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options)
+      ipfsRoots = pathDetails.ipfsRoots
+      terminalElement = pathDetails.terminalElement
+    } catch (err) {
+      this.log.error('Error walking path %s', path, err)
+    }
+
     let resolvedCID = terminalElement?.cid ?? cid
     let stat: UnixFSStats
     if (terminalElement?.type === 'directory') {
@@ -154,7 +250,6 @@ export class VerifiedFetch {
       const rootFilePath = 'index.html'
       try {
         this.log.trace('found directory at %c/%s, looking for index.html', cid, path)
-        options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid: dirCid, path: rootFilePath }))
         stat = await this.unixfs.stat(dirCid, {
           path: rootFilePath,
           signal: options?.signal,
@@ -172,7 +267,6 @@ export class VerifiedFetch {
       }
     }
 
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid: resolvedCID, path: '' }))
     const asyncIter = this.unixfs.cat(resolvedCID, {
       signal: options?.signal,
       onProgress: options?.onProgress
@@ -185,19 +279,27 @@ export class VerifiedFetch {
     const response = okResponse(stream)
     await this.setContentType(firstChunk, path, response)
 
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: resolvedCID, path: '' }))
+    if (ipfsRoots != null) {
+      response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
+    }
 
     return response
   }
 
   private async handleRaw ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
-    this.log.trace('fetching %c/%s', cid, path)
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid, path }))
-    const result = await this.helia.blockstore.get(cid)
+    const result = await this.helia.blockstore.get(cid, options)
     const response = okResponse(result)
-    await this.setContentType(result, path, response)
 
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid, path }))
+    // if the user has specified an `Accept` header that corresponds to a raw
+    // type, honour that header, so for example they don't request
+    // `application/vnd.ipld.raw` but get `application/octet-stream`
+    const overriddenContentType = getOverridenRawContentType(options?.headers)
+    if (overriddenContentType != null) {
+      response.headers.set('content-type', overriddenContentType)
+    } else {
+      await this.setContentType(result, path, response)
+    }
+
     return response
   }
 
@@ -228,111 +330,112 @@ export class VerifiedFetch {
   }
 
   /**
-   * Determines the format requested by the client, defaults to `null` if no format is requested.
-   *
-   * @see https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter
-   * @default 'raw'
-   */
-  private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: RequestFormatShorthand | null }): RequestFormatShorthand | null {
-    const formatMap: Record<string, RequestFormatShorthand> = {
-      'vnd.ipld.raw': 'raw',
-      'vnd.ipld.car': 'car',
-      'application/x-tar': 'tar',
-      'application/vnd.ipld.dag-json': 'dag-json',
-      'application/vnd.ipld.dag-cbor': 'dag-cbor',
-      'application/json': 'json',
-      'application/cbor': 'cbor',
-      'vnd.ipfs.ipns-record': 'ipns-record'
-    }
-
-    if (headerFormat != null) {
-      for (const format in formatMap) {
-        if (headerFormat.includes(format)) {
-          return formatMap[format]
-        }
-      }
-    } else if (queryFormat != null) {
-      return queryFormat
-    }
-
-    return null
-  }
-
-  /**
-   * Map of format to specific handlers for that format.
-   * These format handlers should adjust the response headers as specified in https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers
+   * If the user has not specified an Accept header or format query string arg,
+   * use the CID codec to choose an appropriate handler for the block data.
    */
-  private readonly formatHandlers: Record<string, FetchHandlerFunction> = {
-    raw: async () => notSupportedResponse('application/vnd.ipld.raw support is not implemented'),
-    car: this.handleIPLDCar,
-    'ipns-record': this.handleIPNSRecord,
-    tar: async () => notSupportedResponse('application/x-tar support is not implemented'),
-    'dag-json': async () => notSupportedResponse('application/vnd.ipld.dag-json support is not implemented'),
-    'dag-cbor': async () => notSupportedResponse('application/vnd.ipld.dag-cbor support is not implemented'),
-    json: async () => notSupportedResponse('application/json support is not implemented'),
-    cbor: async () => notSupportedResponse('application/cbor support is not implemented')
-  }
-
   private readonly codecHandlers: Record<number, FetchHandlerFunction> = {
     [dagPbCode]: this.handleDagPb,
-    [dagJsonCode]: this.handleJson,
+    [ipldDagJson.code]: this.handleJson,
     [jsonCode]: this.handleJson,
-    [dagCborCode]: this.handleDagCbor,
+    [ipldDagCbor.code]: this.handleDagCbor,
     [rawCode]: this.handleRaw,
     [identity.code]: this.handleRaw
   }
 
   async fetch (resource: Resource, opts?: VerifiedFetchOptions): Promise<Response> {
+    this.log('fetch %s', resource)
+
     const options = convertOptions(opts)
-    const { path, query, ...rest } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options)
-    const cid = rest.cid
-    let response: Response | undefined
 
-    const format = this.getFormat({ headerFormat: new Headers(options?.headers).get('accept'), queryFormat: query.format ?? null })
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { resource }))
 
-    if (format != null) {
-      // TODO: These should be handled last when they're returning something other than 501
-      const formatHandler = this.formatHandlers[format]
+    // resolve the CID/path from the requested resource
+    const { path, query, cid } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options)
 
-      if (formatHandler != null) {
-        response = await formatHandler.call(this, { cid, path, options })
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:resolve', { cid, path }))
 
-        if (response.status === 501) {
-          return response
-        }
-      }
+    const requestHeaders = new Headers(options?.headers)
+    const incomingAcceptHeader = requestHeaders.get('accept')
+
+    if (incomingAcceptHeader != null) {
+      this.log('incoming accept header "%s"', incomingAcceptHeader)
     }
 
-    let terminalElement: UnixFSEntry | undefined
-    let ipfsRoots: CID[] | undefined
+    const queryFormatMapping = queryFormatToAcceptHeader(query.format)
 
-    try {
-      const pathDetails = await this.pathWalker(this.helia.blockstore, `${cid.toString()}/${path}`, options)
-      ipfsRoots = pathDetails.ipfsRoots
-      terminalElement = pathDetails.terminalElement
-    } catch (err) {
-      this.log.error('Error walking path %s', path, err)
-      // return new Response(`Error walking path: ${(err as Error).message}`, { status: 500 })
+    if (query.format != null) {
+      this.log('incoming query format "%s", mapped to %s', query.format, queryFormatMapping)
+    }
+
+    const acceptHeader = incomingAcceptHeader ?? queryFormatMapping
+    const accept = selectOutputType(cid, acceptHeader)
+    this.log('output type %s', accept)
+
+    if (acceptHeader != null && accept == null) {
+      return notAcceptableResponse()
     }
 
-    if (response == null) {
+    let response: Response
+    let reqFormat: RequestFormatShorthand | undefined
+
+    if (accept === 'application/vnd.ipfs.ipns-record') {
+      // the user requested a raw IPNS record
+      reqFormat = 'ipns-record'
+      response = await this.handleIPNSRecord(resource.toString(), options)
+    } else if (accept === 'application/vnd.ipld.car') {
+      // the user requested a CAR file
+      reqFormat = 'car'
+      query.download = true
+      query.filename = query.filename ?? `${cid.toString()}.car`
+      response = await this.handleCar({ cid, path, options })
+    } else if (accept === 'application/vnd.ipld.raw') {
+      // the user requested a raw block
+      reqFormat = 'raw'
+      query.download = true
+      query.filename = query.filename ?? `${cid.toString()}.bin`
+      response = await this.handleRaw({ cid, path, options })
+    } else if (accept === 'application/x-tar') {
+      // the user requested a TAR file
+      reqFormat = 'tar'
+      response = await this.handleTar({ cid, path, options })
+    } else {
+      // derive the handler from the CID type
       const codecHandler = this.codecHandlers[cid.code]
 
-      if (codecHandler != null) {
-        response = await codecHandler.call(this, { cid, path, options, terminalElement })
-      } else {
+      if (codecHandler == null) {
         return notSupportedResponse(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`)
       }
+
+      response = await codecHandler.call(this, { cid, path, accept, options })
     }
 
-    response.headers.set('etag', getETag({ cid, reqFormat: format ?? undefined, weak: false }))
+    response.headers.set('etag', getETag({ cid, reqFormat, weak: false }))
     response.headers.set('cache-control', 'public, max-age=29030400, immutable')
-    response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header
+    // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header
+    response.headers.set('X-Ipfs-Path', resource.toString())
 
-    if (ipfsRoots != null) {
-      response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
+    // set Content-Disposition header
+    let contentDisposition: string | undefined
+
+    // force download if requested
+    if (query.download === true) {
+      contentDisposition = 'attachment'
+    }
+
+    // override filename if requested
+    if (query.filename != null) {
+      if (contentDisposition == null) {
+        contentDisposition = 'inline'
+      }
+
+      contentDisposition = `${contentDisposition}; ${getContentDispositionFilename(query.filename)}`
     }
-    // response.headers.set('Content-Disposition', `TODO`) // https://specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header
+
+    if (contentDisposition != null) {
+      response.headers.set('Content-Disposition', contentDisposition)
+    }
+
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid, path }))
 
     return response
   }