@helia/verified-fetch 1.1.2 → 1.2.0

package/src/utils/responses.ts

@@ -1,3 +1,7 @@
+import type { ByteRangeContext } from './byte-range-context'
+import type { SupportedBodyTypes } from '../types.js'
+import type { Logger } from '@libp2p/interface'
+
 function setField (response: Response, name: string, value: string | boolean): void {
   Object.defineProperty(response, name, {
     enumerable: true,
@@ -23,7 +27,7 @@ export interface ResponseOptions extends ResponseInit {
   redirected?: boolean
 }
 
-export function okResponse (url: string, body?: BodyInit | null, init?: ResponseOptions): Response {
+export function okResponse (url: string, body?: SupportedBodyTypes, init?: ResponseOptions): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 200,
@@ -34,13 +38,27 @@ export function okResponse (url: string, body?: BodyInit | null, init?: Response
     setRedirected(response)
   }
 
+  setType(response, 'basic')
+  setUrl(response, url)
+  response.headers.set('Accept-Ranges', 'bytes')
+
+  return response
+}
+
+export function badGatewayResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
+    status: 502,
+    statusText: 'Bad Gateway'
+  })
+
   setType(response, 'basic')
   setUrl(response, url)
 
   return response
 }
 
-export function notSupportedResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+export function notSupportedResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 501,
@@ -54,7 +72,7 @@ export function notSupportedResponse (url: string, body?: BodyInit | null, init?
   return response
 }
 
-export function notAcceptableResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+export function notAcceptableResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 406,
@@ -67,7 +85,7 @@ export function notAcceptableResponse (url: string, body?: BodyInit | null, init
   return response
 }
 
-export function badRequestResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+export function badRequestResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 400,
@@ -96,3 +114,63 @@ export function movedPermanentlyResponse (url: string, location: string, init?:
 
   return response
 }
+
+interface RangeOptions {
+  byteRangeContext: ByteRangeContext
+  log?: Logger
+}
+
+export function okRangeResponse (url: string, body: SupportedBodyTypes, { byteRangeContext, log }: RangeOptions, init?: ResponseOptions): Response {
+  if (!byteRangeContext.isRangeRequest) {
+    return okResponse(url, body, init)
+  }
+
+  if (!byteRangeContext.isValidRangeRequest) {
+    return badRangeResponse(url, body, init)
+  }
+
+  let response: Response
+  try {
+    response = new Response(body, {
+      ...(init ?? {}),
+      status: 206,
+      statusText: 'Partial Content',
+      headers: {
+        ...(init?.headers ?? {}),
+        'content-range': byteRangeContext.contentRangeHeaderValue
+      }
+    })
+  } catch (e: any) {
+    log?.error('failed to create range response', e)
+    return badRangeResponse(url, body, init)
+  }
+
+  if (init?.redirected === true) {
+    setRedirected(response)
+  }
+
+  setType(response, 'basic')
+  setUrl(response, url)
+  response.headers.set('Accept-Ranges', 'bytes')
+
+  return response
+}
+
+/**
+ * We likely need to catch errors handled by upstream helia libraries if range-request throws an error. Some examples:
+ * * The range is out of bounds
+ * * The range is invalid
+ * * The range is not supported for the given type
+ */
+export function badRangeResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
+    status: 416,
+    statusText: 'Requested Range Not Satisfiable'
+  })
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
+}

package/src/verified-fetch.ts

@@ -1,7 +1,6 @@
 import { car } from '@helia/car'
-import { ipns as heliaIpns, type DNSResolver, type IPNS } from '@helia/ipns'
-import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers'
-import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs'
+import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
+import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs } from '@helia/unixfs'
 import * as ipldDagCbor from '@ipld/dag-cbor'
 import * as ipldDagJson from '@ipld/dag-json'
 import { code as dagPbCode } from '@ipld/dag-pb'
@@ -16,19 +15,22 @@ import { CustomProgressEvent } from 'progress-events'
 import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { ByteRangeContext } from './utils/byte-range-context.js'
 import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js'
 import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js'
 import { getETag } from './utils/get-e-tag.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { tarStream } from './utils/get-tar-stream.js'
 import { parseResource } from './utils/parse-resource.js'
-import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
+import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse } from './utils/responses.js'
 import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js'
 import { walkPath } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
 import type { RequestFormatShorthand } from './types.js'
+import type { ParsedUrlStringResults } from './utils/parse-url-string'
 import type { Helia } from '@helia/interface'
 import type { AbortOptions, Logger, PeerId } from '@libp2p/interface'
+import type { DNSResolver } from '@multiformats/dns/resolvers'
 import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
 import type { CID } from 'multiformats/cid'
 
@@ -126,12 +128,7 @@ export class VerifiedFetch {
   constructor ({ helia, ipns, unixfs }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
     this.helia = helia
     this.log = helia.logger.forComponent('helia:verified-fetch')
-    this.ipns = ipns ?? heliaIpns(helia, {
-      resolvers: init?.dnsResolvers ?? [
-        dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'),
-        dnsJsonOverHttps('https://dns.google/resolve')
-      ]
-    })
+    this.ipns = ipns ?? heliaIpns(helia)
     this.unixfs = unixfs ?? heliaUnixFs(helia)
     this.contentTypeParser = init?.contentTypeParser
     this.log.trace('created VerifiedFetch instance')
@@ -280,17 +277,19 @@ export class VerifiedFetch {
     let terminalElement: UnixFSEntry | undefined
     let ipfsRoots: CID[] | undefined
     let redirected = false
+    const byteRangeContext = new ByteRangeContext(this.helia.logger, options?.headers)
 
     try {
       const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options)
       ipfsRoots = pathDetails.ipfsRoots
       terminalElement = pathDetails.terminalElement
     } catch (err) {
-      this.log.error('Error walking path %s', path, err)
+      this.log.error('error walking path %s', path, err)
+
+      return badGatewayResponse('Error walking path')
     }
 
     let resolvedCID = terminalElement?.cid ?? cid
-    let stat: UnixFSStats
     if (terminalElement?.type === 'directory') {
       const dirCid = terminalElement.cid
 
@@ -312,7 +311,7 @@ export class VerifiedFetch {
       const rootFilePath = 'index.html'
       try {
         this.log.trace('found directory at %c/%s, looking for index.html', cid, path)
-        stat = await this.unixfs.stat(dirCid, {
+        const stat = await this.unixfs.stat(dirCid, {
           path: rootFilePath,
           signal: options?.signal,
           onProgress: options?.onProgress
@@ -328,30 +327,56 @@ export class VerifiedFetch {
       }
     }
 
+    // we have a validRangeRequest & terminalElement is a file, we know the size and should set it
+    if (byteRangeContext.isRangeRequest && byteRangeContext.isValidRangeRequest && terminalElement.type === 'file') {
+      byteRangeContext.setFileSize(terminalElement.unixfs.fileSize())
+
+      this.log.trace('fileSize for rangeRequest %d', byteRangeContext.getFileSize())
+    }
+    const offset = byteRangeContext.offset
+    const length = byteRangeContext.length
+    this.log.trace('calling unixfs.cat for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length)
     const asyncIter = this.unixfs.cat(resolvedCID, {
       signal: options?.signal,
-      onProgress: options?.onProgress
+      onProgress: options?.onProgress,
+      offset,
+      length
     })
     this.log('got async iterator for %c/%s', cid, path)
 
-    const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
-      onProgress: options?.onProgress
-    })
-    const response = okResponse(resource, stream, {
-      redirected
-    })
-    await this.setContentType(firstChunk, path, response)
+    try {
+      const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
+        onProgress: options?.onProgress
+      })
+      byteRangeContext.setBody(stream)
+      // if not a valid range request, okRangeRequest will call okResponse
+      const response = okRangeResponse(resource, byteRangeContext.getBody(), { byteRangeContext, log: this.log }, {
+        redirected
+      })
+
+      await this.setContentType(firstChunk, path, response)
+
+      if (ipfsRoots != null) {
+        response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
+      }
 
-    if (ipfsRoots != null) {
-      response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
+      return response
+    } catch (err: any) {
+      this.log.error('error streaming %c/%s', cid, path, err)
+      if (byteRangeContext.isRangeRequest && err.code === 'ERR_INVALID_PARAMS') {
+        return badRangeResponse(resource)
+      }
+      return badGatewayResponse('Unable to stream content')
     }
-
-    return response
   }
 
   private async handleRaw ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    const byteRangeContext = new ByteRangeContext(this.helia.logger, options?.headers)
     const result = await this.helia.blockstore.get(cid, options)
-    const response = okResponse(resource, result)
+    byteRangeContext.setBody(result)
+    const response = okRangeResponse(resource, byteRangeContext.getBody(), { byteRangeContext, log: this.log }, {
+      redirected: false
+    })
 
     // if the user has specified an `Accept` header that corresponds to a raw
     // type, honour that header, so for example they don't request
@@ -385,10 +410,10 @@ export class VerifiedFetch {
           contentType = parsed
         }
       } catch (err) {
-        this.log.error('Error parsing content type', err)
+        this.log.error('error parsing content type', err)
       }
     }
-
+    this.log.trace('setting content type to "%s"', contentType)
     response.headers.set('content-type', contentType)
   }
 
@@ -413,7 +438,19 @@ export class VerifiedFetch {
     options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { resource }))
 
     // resolve the CID/path from the requested resource
-    const { path, query, cid } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options)
+    let cid: ParsedUrlStringResults['cid']
+    let path: ParsedUrlStringResults['path']
+    let query: ParsedUrlStringResults['query']
+    try {
+      const result = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options)
+      cid = result.cid
+      path = result.path
+      query = result.query
+    } catch (err) {
+      this.log.error('error parsing resource %s', resource, err)
+
+      return badRequestResponse('Invalid resource')
+    }
 
     options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:resolve', { cid, path }))
 
@@ -466,12 +503,14 @@ export class VerifiedFetch {
       query.filename = query.filename ?? `${cid.toString()}.tar`
       response = await this.handleTar(handlerArgs)
     } else {
+      this.log.trace('finding handler for cid code "%s" and output type "%s"', cid.code, accept)
       // derive the handler from the CID type
       const codecHandler = this.codecHandlers[cid.code]
 
       if (codecHandler == null) {
         return notSupportedResponse(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`)
       }
+      this.log.trace('calling handler "%s"', codecHandler.name)
 
       response = await codecHandler.call(this, handlerArgs)
     }