@helia/verified-fetch 1.1.3 → 1.2.1

package/src/utils/parse-url-string.ts

@@ -106,7 +106,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
       log.trace('resolved %s to %c from cache', cidOrPeerIdOrDnsLink, cid)
     } else {
       // protocol is ipns
-      log.trace('Attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink)
+      log.trace('attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink)
       let peerId = null
       try {
         peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
@@ -117,10 +117,10 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
         ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
       } catch (err) {
         if (peerId == null) {
-          log.error('Could not parse PeerId string "%s"', cidOrPeerIdOrDnsLink, err)
+          log.error('could not parse PeerId string "%s"', cidOrPeerIdOrDnsLink, err)
           errors.push(new TypeError(`Could not parse PeerId in ipns url "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`))
         } else {
-          log.error('Could not resolve PeerId %c', peerId, err)
+          log.error('could not resolve PeerId %c', peerId, err)
           errors.push(new TypeError(`Could not resolve PeerId "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`))
         }
       }
@@ -140,7 +140,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
           log.trace('resolved %s to %c', decodedDnsLinkLabel, cid)
           ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
         } catch (err: any) {
-          log.error('Could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err)
+          log.error('could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err)
           errors.push(err)
         }
       }

package/src/utils/request-headers.ts

@@ -0,0 +1,63 @@
+export function getHeader (headers: HeadersInit | undefined, header: string): string | undefined {
+  if (headers == null) {
+    return undefined
+  }
+  if (headers instanceof Headers) {
+    return headers.get(header) ?? undefined
+  }
+  if (Array.isArray(headers)) {
+    const entry = headers.find(([key]) => key.toLowerCase() === header.toLowerCase())
+    return entry?.[1]
+  }
+  const key = Object.keys(headers).find(k => k.toLowerCase() === header.toLowerCase())
+  if (key == null) {
+    return undefined
+  }
+
+  return headers[key]
+}
+
+/**
+ * Given two ints from a Range header, and potential fileSize, returns:
+ * 1. number of bytes the response should contain.
+ * 2. the start index of the range. // inclusive
+ * 3. the end index of the range. // inclusive
+ */
+// eslint-disable-next-line complexity
+export function calculateByteRangeIndexes (start: number | undefined, end: number | undefined, fileSize?: number): { byteSize?: number, start?: number, end?: number } {
+  if ((start ?? 0) > (end ?? Infinity)) {
+    throw new Error('Invalid range: Range-start index is greater than range-end index.')
+  }
+  if (start != null && (end ?? 0) >= (fileSize ?? Infinity)) {
+    throw new Error('Invalid range: Range-end index is greater than or equal to the size of the file.')
+  }
+  if (start == null && (end ?? 0) > (fileSize ?? Infinity)) {
+    throw new Error('Invalid range: Range-end index is greater than the size of the file.')
+  }
+  if (start != null && start < 0) {
+    throw new Error('Invalid range: Range-start index cannot be negative.')
+  }
+
+  if (start != null && end != null) {
+    return { byteSize: end - start + 1, start, end }
+  } else if (start == null && end != null) {
+    // suffix byte range requested
+    if (fileSize == null) {
+      return { end }
+    }
+    if (end === fileSize) {
+      return { byteSize: fileSize, start: 0, end: fileSize - 1 }
+    }
+    return { byteSize: end, start: fileSize - end, end: fileSize - 1 }
+  } else if (start != null && end == null) {
+    if (fileSize == null) {
+      // we only have the start index, and no fileSize, so we can't return a valid range.
+      return { start }
+    }
+    const end = fileSize - 1
+    const byteSize = fileSize - start
+    return { byteSize, start, end }
+  }
+
+  return { byteSize: fileSize, start: 0, end: fileSize != null ? fileSize - 1 : 0 }
+}

package/src/utils/response-headers.ts

@@ -0,0 +1,42 @@
+/**
+ * This function returns the value of the `Content-Range` header for a given range.
+ * If you know the total size of the body, pass it as `byteSize`
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range
+ */
+export function getContentRangeHeader ({ byteStart, byteEnd, byteSize }: { byteStart: number | undefined, byteEnd: number | undefined, byteSize: number | undefined }): string {
+  const total = byteSize ?? '*' // if we don't know the total size, we should use *
+
+  if ((byteEnd ?? 0) >= (byteSize ?? Infinity)) {
+    throw new Error('Invalid range: Range-end index is greater than or equal to the size of the file.')
+  }
+  if ((byteStart ?? 0) >= (byteSize ?? Infinity)) {
+    throw new Error('Invalid range: Range-start index is greater than or equal to the size of the file.')
+  }
+
+  if (byteStart != null && byteEnd == null) {
+    // only byteStart in range
+    if (byteSize == null) {
+      return `bytes */${total}`
+    }
+    return `bytes ${byteStart}-${byteSize - 1}/${byteSize}`
+  }
+
+  if (byteStart == null && byteEnd != null) {
+    // only byteEnd in range
+    if (byteSize == null) {
+      return `bytes */${total}`
+    }
+    const end = byteSize - 1
+    const start = end - byteEnd + 1
+
+    return `bytes ${start}-${end}/${byteSize}`
+  }
+
+  if (byteStart == null && byteEnd == null) {
+    // neither are provided, we can't return a valid range.
+    return `bytes */${total}`
+  }
+
+  return `bytes ${byteStart}-${byteEnd}/${total}`
+}

package/src/utils/responses.ts

@@ -1,3 +1,7 @@
+import type { ByteRangeContext } from './byte-range-context'
+import type { SupportedBodyTypes } from '../types.js'
+import type { Logger } from '@libp2p/interface'
+
 function setField (response: Response, name: string, value: string | boolean): void {
   Object.defineProperty(response, name, {
     enumerable: true,
@@ -23,7 +27,7 @@ export interface ResponseOptions extends ResponseInit {
   redirected?: boolean
 }
 
-export function okResponse (url: string, body?: BodyInit | null, init?: ResponseOptions): Response {
+export function okResponse (url: string, body?: SupportedBodyTypes, init?: ResponseOptions): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 200,
@@ -34,13 +38,27 @@ export function okResponse (url: string, body?: BodyInit | null, init?: Response
     setRedirected(response)
   }
 
+  setType(response, 'basic')
+  setUrl(response, url)
+  response.headers.set('Accept-Ranges', 'bytes')
+
+  return response
+}
+
+export function badGatewayResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
+    status: 502,
+    statusText: 'Bad Gateway'
+  })
+
   setType(response, 'basic')
   setUrl(response, url)
 
   return response
 }
 
-export function notSupportedResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+export function notSupportedResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 501,
@@ -54,7 +72,7 @@ export function notSupportedResponse (url: string, body?: BodyInit | null, init?
   return response
 }
 
-export function notAcceptableResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+export function notAcceptableResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 406,
@@ -67,7 +85,7 @@ export function notAcceptableResponse (url: string, body?: BodyInit | null, init
   return response
 }
 
-export function badRequestResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+export function badRequestResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 400,
@@ -96,3 +114,63 @@ export function movedPermanentlyResponse (url: string, location: string, init?:
 
   return response
 }
+
+interface RangeOptions {
+  byteRangeContext: ByteRangeContext
+  log?: Logger
+}
+
+export function okRangeResponse (url: string, body: SupportedBodyTypes, { byteRangeContext, log }: RangeOptions, init?: ResponseOptions): Response {
+  if (!byteRangeContext.isRangeRequest) {
+    return okResponse(url, body, init)
+  }
+
+  if (!byteRangeContext.isValidRangeRequest) {
+    return badRangeResponse(url, body, init)
+  }
+
+  let response: Response
+  try {
+    response = new Response(body, {
+      ...(init ?? {}),
+      status: 206,
+      statusText: 'Partial Content',
+      headers: {
+        ...(init?.headers ?? {}),
+        'content-range': byteRangeContext.contentRangeHeaderValue
+      }
+    })
+  } catch (e: any) {
+    log?.error('failed to create range response', e)
+    return badRangeResponse(url, body, init)
+  }
+
+  if (init?.redirected === true) {
+    setRedirected(response)
+  }
+
+  setType(response, 'basic')
+  setUrl(response, url)
+  response.headers.set('Accept-Ranges', 'bytes')
+
+  return response
+}
+
+/**
+ * We likely need to catch errors handled by upstream helia libraries if range-request throws an error. Some examples:
+ * * The range is out of bounds
+ * * The range is invalid
+ * * The range is not supported for the given type
+ */
+export function badRangeResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
+    status: 416,
+    statusText: 'Requested Range Not Satisfiable'
+  })
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
+}

package/src/verified-fetch.ts

@@ -1,6 +1,6 @@
 import { car } from '@helia/car'
 import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
-import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs'
+import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs } from '@helia/unixfs'
 import * as ipldDagCbor from '@ipld/dag-cbor'
 import * as ipldDagJson from '@ipld/dag-json'
 import { code as dagPbCode } from '@ipld/dag-pb'
@@ -15,17 +15,19 @@ import { CustomProgressEvent } from 'progress-events'
 import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { ByteRangeContext } from './utils/byte-range-context.js'
 import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js'
 import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js'
 import { getETag } from './utils/get-e-tag.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { tarStream } from './utils/get-tar-stream.js'
 import { parseResource } from './utils/parse-resource.js'
-import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
+import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse } from './utils/responses.js'
 import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js'
 import { walkPath } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
 import type { RequestFormatShorthand } from './types.js'
+import type { ParsedUrlStringResults } from './utils/parse-url-string'
 import type { Helia } from '@helia/interface'
 import type { AbortOptions, Logger, PeerId } from '@libp2p/interface'
 import type { DNSResolver } from '@multiformats/dns/resolvers'
@@ -275,17 +277,19 @@ export class VerifiedFetch {
     let terminalElement: UnixFSEntry | undefined
     let ipfsRoots: CID[] | undefined
     let redirected = false
+    const byteRangeContext = new ByteRangeContext(this.helia.logger, options?.headers)
 
     try {
       const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options)
       ipfsRoots = pathDetails.ipfsRoots
       terminalElement = pathDetails.terminalElement
     } catch (err) {
-      this.log.error('Error walking path %s', path, err)
+      this.log.error('error walking path %s', path, err)
+
+      return badGatewayResponse('Error walking path')
     }
 
     let resolvedCID = terminalElement?.cid ?? cid
-    let stat: UnixFSStats
     if (terminalElement?.type === 'directory') {
       const dirCid = terminalElement.cid
 
@@ -307,7 +311,7 @@ export class VerifiedFetch {
       const rootFilePath = 'index.html'
       try {
         this.log.trace('found directory at %c/%s, looking for index.html', cid, path)
-        stat = await this.unixfs.stat(dirCid, {
+        const stat = await this.unixfs.stat(dirCid, {
           path: rootFilePath,
           signal: options?.signal,
           onProgress: options?.onProgress
@@ -323,30 +327,56 @@ export class VerifiedFetch {
       }
     }
 
+    // we have a validRangeRequest & terminalElement is a file, we know the size and should set it
+    if (byteRangeContext.isRangeRequest && byteRangeContext.isValidRangeRequest && terminalElement.type === 'file') {
+      byteRangeContext.setFileSize(terminalElement.unixfs.fileSize())
+
+      this.log.trace('fileSize for rangeRequest %d', byteRangeContext.getFileSize())
+    }
+    const offset = byteRangeContext.offset
+    const length = byteRangeContext.length
+    this.log.trace('calling unixfs.cat for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length)
     const asyncIter = this.unixfs.cat(resolvedCID, {
       signal: options?.signal,
-      onProgress: options?.onProgress
+      onProgress: options?.onProgress,
+      offset,
+      length
     })
     this.log('got async iterator for %c/%s', cid, path)
 
-    const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
-      onProgress: options?.onProgress
-    })
-    const response = okResponse(resource, stream, {
-      redirected
-    })
-    await this.setContentType(firstChunk, path, response)
+    try {
+      const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
+        onProgress: options?.onProgress
+      })
+      byteRangeContext.setBody(stream)
+      // if not a valid range request, okRangeRequest will call okResponse
+      const response = okRangeResponse(resource, byteRangeContext.getBody(), { byteRangeContext, log: this.log }, {
+        redirected
+      })
+
+      await this.setContentType(firstChunk, path, response)
+
+      if (ipfsRoots != null) {
+        response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
+      }
 
-    if (ipfsRoots != null) {
-      response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
+      return response
+    } catch (err: any) {
+      this.log.error('error streaming %c/%s', cid, path, err)
+      if (byteRangeContext.isRangeRequest && err.code === 'ERR_INVALID_PARAMS') {
+        return badRangeResponse(resource)
+      }
+      return badGatewayResponse('Unable to stream content')
     }
-
-    return response
   }
 
   private async handleRaw ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+    const byteRangeContext = new ByteRangeContext(this.helia.logger, options?.headers)
     const result = await this.helia.blockstore.get(cid, options)
-    const response = okResponse(resource, result)
+    byteRangeContext.setBody(result)
+    const response = okRangeResponse(resource, byteRangeContext.getBody(), { byteRangeContext, log: this.log }, {
+      redirected: false
+    })
 
     // if the user has specified an `Accept` header that corresponds to a raw
     // type, honour that header, so for example they don't request
@@ -380,10 +410,10 @@ export class VerifiedFetch {
           contentType = parsed
         }
       } catch (err) {
-        this.log.error('Error parsing content type', err)
+        this.log.error('error parsing content type', err)
       }
     }
-
+    this.log.trace('setting content type to "%s"', contentType)
     response.headers.set('content-type', contentType)
   }
 
@@ -408,7 +438,19 @@ export class VerifiedFetch {
     options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { resource }))
 
     // resolve the CID/path from the requested resource
-    const { path, query, cid } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options)
+    let cid: ParsedUrlStringResults['cid']
+    let path: ParsedUrlStringResults['path']
+    let query: ParsedUrlStringResults['query']
+    try {
+      const result = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options)
+      cid = result.cid
+      path = result.path
+      query = result.query
+    } catch (err) {
+      this.log.error('error parsing resource %s', resource, err)
+
+      return badRequestResponse('Invalid resource')
+    }
 
     options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:resolve', { cid, path }))
 
@@ -461,12 +503,14 @@ export class VerifiedFetch {
       query.filename = query.filename ?? `${cid.toString()}.tar`
       response = await this.handleTar(handlerArgs)
     } else {
+      this.log.trace('finding handler for cid code "%s" and output type "%s"', cid.code, accept)
       // derive the handler from the CID type
       const codecHandler = this.codecHandlers[cid.code]
 
       if (codecHandler == null) {
         return notSupportedResponse(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`)
       }
+      this.log.trace('calling handler "%s"', codecHandler.name)
 
       response = await codecHandler.call(this, handlerArgs)
     }