@helia/verified-fetch 1.0.2 → 1.1.1

package/src/index.ts

@@ -379,6 +379,79 @@
  * const buf = await res.arrayBuffer() // raw bytes, not processed as JSON
  * ```
  *
+ * ## Redirects
+ *
+ * If a requested URL contains a path component, that path component resolves to
+ * a UnixFS directory, but the URL does not have a trailing slash, one will be
+ * added to form a canonical URL for that resource, otherwise the request will
+ * be resolved as normal.
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir')
+ *
+ * console.info(res.url) // ipfs://bafyfoo/path/to/dir/
+ * ```
+ *
+ * It's possible to prevent this behaviour and/or handle a redirect manually
+ * through use of the [redirect](https://developer.mozilla.org/en-US/docs/Web/API/fetch#redirect)
+ * option.
+ *
+ * @example Redirect: follow
+ *
+ * This is the default value and is what happens if no value is specified.
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir', {
+ *   redirect: 'follow'
+ * })
+ *
+ * console.info(res.status) // 200
+ * console.info(res.url) // ipfs://bafyfoo/path/to/dir/
+ * console.info(res.redirected) // true
+ * ```
+ *
+ * @example Redirect: error
+ *
+ * This causes a `TypeError` to be thrown if a URL would cause a redirect.
+ *
+ * ```typescript
+ *
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir', {
+ *   redirect: 'error'
+ * })
+ * // throw TypeError('Failed to fetch')
+ * ```
+ *
+ * @example Redirect: manual
+ *
+ * Manual redirects allow the user to process the redirect. A [301](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/301)
+ * is returned, and the location to redirect to is available as the "location"
+ * response header.
+ *
+ * This differs slightly from HTTP fetch which returns an opaque response as the
+ * browser itself is expected to process the redirect and hide all details from
+ * the user.
+ *
+ * ```typescript
+ *
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir', {
+ *   redirect: 'manual'
+ * })
+ *
+ * console.info(res.status) // 301
+ * console.info(res.url) // ipfs://bafyfoo/path/to/dir
+ * console.info(res.redirected) // false
+ * console.info(res.headers.get('location')) // ipfs://bafyfoo/path/to/dir/
+ * ```
+ *
  * ## Comparison to fetch
  *
  * This module attempts to act as similarly to the `fetch()` API as possible.

package/src/utils/parse-url-string.ts

@@ -30,7 +30,22 @@ export interface ParsedUrlStringResults {
   query: ParsedUrlQuery
 }
 
-const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/$?]+)\/?(?<path>[^$?]*)\??(?<queryString>.*)$/
+const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
+const PATH_REGEX = /^\/(?<protocol>ip[fn]s)\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
+const PATH_GATEWAY_REGEX = /^https?:\/\/(.*[^/])\/(?<protocol>ip[fn]s)\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
+const SUBDOMAIN_GATEWAY_REGEX = /^https?:\/\/(?<cidOrPeerIdOrDnsLink>[^/.?]+)\.(?<protocol>ip[fn]s)\.([^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
+
+function matchURLString (urlString: string): Record<string, string> {
+  for (const pattern of [URL_REGEX, PATH_REGEX, PATH_GATEWAY_REGEX, SUBDOMAIN_GATEWAY_REGEX]) {
+    const match = urlString.match(pattern)
+
+    if (match?.groups != null) {
+      return match.groups
+    }
+  }
+
+  throw new TypeError(`Invalid URL: ${urlString}, please use ipfs://, ipns://, or gateway URLs only`)
+}
 
 /**
  * A function that parses ipfs:// and ipns:// URLs, returning an object with easily recognizable properties.
@@ -41,13 +56,7 @@ const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/$?]+)\/?
  */
 export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStringInput, options?: ParseUrlStringOptions): Promise<ParsedUrlStringResults> {
   const log = logger.forComponent('helia:verified-fetch:parse-url-string')
-  const match = urlString.match(URL_REGEX)
-
-  if (match == null || match.groups == null) {
-    throw new TypeError(`Invalid URL: ${urlString}, please use ipfs:// or ipns:// URLs only.`)
-  }
-
-  const { protocol, cidOrPeerIdOrDnsLink, path: urlPath, queryString } = match.groups
+  const { protocol, cidOrPeerIdOrDnsLink, path: urlPath, queryString } = matchURLString(urlString)
 
   let cid: CID | undefined
   let resolvedPath: string | undefined

package/src/utils/responses.ts

@@ -1,29 +1,98 @@
-export function okResponse (body?: BodyInit | null): Response {
-  return new Response(body, {
+function setField (response: Response, name: string, value: string | boolean): void {
+  Object.defineProperty(response, name, {
+    enumerable: true,
+    configurable: false,
+    set: () => {},
+    get: () => value
+  })
+}
+
+function setType (response: Response, value: 'basic' | 'cors' | 'error' | 'opaque' | 'opaqueredirect'): void {
+  setField(response, 'type', value)
+}
+
+function setUrl (response: Response, value: string): void {
+  setField(response, 'url', value)
+}
+
+function setRedirected (response: Response): void {
+  setField(response, 'redirected', true)
+}
+
+export interface ResponseOptions extends ResponseInit {
+  redirected?: boolean
+}
+
+export function okResponse (url: string, body?: BodyInit | null, init?: ResponseOptions): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
     status: 200,
     statusText: 'OK'
   })
+
+  if (init?.redirected === true) {
+    setRedirected(response)
+  }
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
 }
 
-export function notSupportedResponse (body?: BodyInit | null): Response {
+export function notSupportedResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
   const response = new Response(body, {
+    ...(init ?? {}),
     status: 501,
     statusText: 'Not Implemented'
   })
   response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
   return response
 }
 
-export function notAcceptableResponse (body?: BodyInit | null): Response {
-  return new Response(body, {
+export function notAcceptableResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
     status: 406,
     statusText: 'Not Acceptable'
   })
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
 }
 
-export function badRequestResponse (body?: BodyInit | null): Response {
-  return new Response(body, {
+export function badRequestResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
     status: 400,
     statusText: 'Bad Request'
   })
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
+}
+
+export function movedPermanentlyResponse (url: string, location: string, init?: ResponseInit): Response {
+  const response = new Response(null, {
+    ...(init ?? {}),
+    status: 301,
+    statusText: 'Moved Permanently',
+    headers: {
+      ...(init?.headers ?? {}),
+      location
+    }
+  })
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
 }

package/src/verified-fetch.ts

@@ -22,7 +22,7 @@ import { getETag } from './utils/get-e-tag.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { tarStream } from './utils/get-tar-stream.js'
 import { parseResource } from './utils/parse-resource.js'
-import { badRequestResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
+import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
 import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js'
 import { walkPath } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
@@ -167,7 +167,7 @@ export class VerifiedFetch {
     const buf = await this.helia.datastore.get(datastoreKey, options)
     const record = DHTRecord.deserialize(buf)
 
-    const response = okResponse(record.value)
+    const response = okResponse(resource, record.value)
     response.headers.set('content-type', 'application/vnd.ipfs.ipns-record')
 
     return response
@@ -177,11 +177,11 @@ export class VerifiedFetch {
    * Accepts a `CID` and returns a `Response` with a body stream that is a CAR
    * of the `DAG` referenced by the `CID`.
    */
-  private async handleCar ({ cid, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleCar ({ resource, cid, options }: FetchHandlerFunctionArg): Promise<Response> {
     const c = car(this.helia)
     const stream = toBrowserReadableStream(c.stream(cid, options))
 
-    const response = okResponse(stream)
+    const response = okResponse(resource, stream)
     response.headers.set('content-type', 'application/vnd.ipld.car; version=1')
 
     return response
@@ -191,20 +191,20 @@ export class VerifiedFetch {
    * Accepts a UnixFS `CID` and returns a `.tar` file containing the file or
    * directory structure referenced by the `CID`.
    */
-  private async handleTar ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleTar ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
     if (cid.code !== dagPbCode && cid.code !== rawCode) {
       return notAcceptableResponse('only UnixFS data can be returned in a TAR file')
     }
 
     const stream = toBrowserReadableStream<Uint8Array>(tarStream(`/ipfs/${cid}/${path}`, this.helia.blockstore, options))
 
-    const response = okResponse(stream)
+    const response = okResponse(resource, stream)
     response.headers.set('content-type', 'application/x-tar')
 
     return response
   }
 
-  private async handleJson ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleJson ({ resource, cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
     this.log.trace('fetching %c/%s', cid, path)
     const block = await this.helia.blockstore.get(cid, options)
     let body: string | Uint8Array
@@ -218,19 +218,19 @@ export class VerifiedFetch {
         body = ipldDagCbor.encode(obj)
       } catch (err) {
         this.log.error('could not transform %c to application/vnd.ipld.dag-cbor', err)
-        return notAcceptableResponse()
+        return notAcceptableResponse(resource)
       }
     } else {
       // skip decoding
       body = block
     }
 
-    const response = okResponse(body)
+    const response = okResponse(resource, body)
     response.headers.set('content-type', accept ?? 'application/json')
     return response
   }
 
-  private async handleDagCbor ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleDagCbor ({ resource, cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
     this.log.trace('fetching %c/%s', cid, path)
 
     const block = await this.helia.blockstore.get(cid, options)
@@ -248,7 +248,7 @@ export class VerifiedFetch {
         body = ipldDagJson.encode(obj)
       } catch (err) {
         this.log.error('could not transform %c to application/vnd.ipld.dag-json', err)
-        return notAcceptableResponse()
+        return notAcceptableResponse(resource)
       }
     } else {
       try {
@@ -257,7 +257,7 @@ export class VerifiedFetch {
         if (accept === 'application/json') {
           this.log('could not decode DAG-CBOR as JSON-safe, but the client sent "Accept: application/json"', err)
 
-          return notAcceptableResponse()
+          return notAcceptableResponse(resource)
         }
 
         this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err)
@@ -265,7 +265,7 @@ export class VerifiedFetch {
       }
     }
 
-    const response = okResponse(body)
+    const response = okResponse(resource, body)
 
     if (accept == null) {
       accept = body instanceof Uint8Array ? 'application/octet-stream' : 'application/json'
@@ -276,9 +276,10 @@ export class VerifiedFetch {
     return response
   }
 
-  private async handleDagPb ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleDagPb ({ cid, path, resource, options }: FetchHandlerFunctionArg): Promise<Response> {
     let terminalElement: UnixFSEntry | undefined
     let ipfsRoots: CID[] | undefined
+    let redirected = false
 
     try {
       const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options)
@@ -293,6 +294,21 @@ export class VerifiedFetch {
     if (terminalElement?.type === 'directory') {
       const dirCid = terminalElement.cid
 
+      // https://specs.ipfs.tech/http-gateways/path-gateway/#use-in-directory-url-normalization
+      if (path !== '' && !path.endsWith('/')) {
+        if (options?.redirect === 'error') {
+          this.log('could not redirect to %s/ as redirect option was set to "error"', resource)
+          throw new TypeError('Failed to fetch')
+        } else if (options?.redirect === 'manual') {
+          this.log('returning 301 permanent redirect to %s/', resource)
+          return movedPermanentlyResponse(resource, `${resource}/`)
+        }
+
+        // fall-through simulates following the redirect?
+        resource = `${resource}/`
+        redirected = true
+      }
+
       const rootFilePath = 'index.html'
       try {
         this.log.trace('found directory at %c/%s, looking for index.html', cid, path)
@@ -304,7 +320,6 @@ export class VerifiedFetch {
         this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, stat.cid)
         path = rootFilePath
         resolvedCID = stat.cid
-        // terminalElement = stat
       } catch (err: any) {
         this.log('error loading path %c/%s', dirCid, rootFilePath, err)
         return notSupportedResponse('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented')
@@ -322,7 +337,9 @@ export class VerifiedFetch {
     const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
       onProgress: options?.onProgress
     })
-    const response = okResponse(stream)
+    const response = okResponse(resource, stream, {
+      redirected
+    })
     await this.setContentType(firstChunk, path, response)
 
     if (ipfsRoots != null) {
@@ -332,9 +349,9 @@ export class VerifiedFetch {
     return response
   }
 
-  private async handleRaw ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleRaw ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
     const result = await this.helia.blockstore.get(cid, options)
-    const response = okResponse(result)
+    const response = okResponse(resource, result)
 
     // if the user has specified an `Accept` header that corresponds to a raw
     // type, honour that header, so for example they don't request
@@ -418,7 +435,7 @@ export class VerifiedFetch {
     this.log('output type %s', accept)
 
     if (acceptHeader != null && accept == null) {
-      return notAcceptableResponse()
+      return notAcceptableResponse(resource.toString())
     }
 
     let response: Response