@helia/verified-fetch 1.1.2 → 1.2.0

package/src/index.ts

@@ -148,7 +148,7 @@
  *
  * ```typescript
  * import { createVerifiedFetch } from '@helia/verified-fetch'
- * import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers'
+ * import { dnsJsonOverHttps, dnsOverHttps } from '@multiformats/dns/resolvers'
  *
  * const fetch = await createVerifiedFetch({
  *   gateways: ['https://trustless-gateway.link'],
@@ -160,6 +160,29 @@
  * })
  * ```
  *
+ * @example Customizing DNS per-TLD resolvers
+ *
+ * DNS resolvers can be configured to only service DNS queries for specific
+ * TLDs:
+ *
+ * ```typescript
+ * import { createVerifiedFetch } from '@helia/verified-fetch'
+ * import { dnsJsonOverHttps, dnsOverHttps } from '@multiformats/dns/resolvers'
+ *
+ * const fetch = await createVerifiedFetch({
+ *   gateways: ['https://trustless-gateway.link'],
+ *   routers: ['http://delegated-ipfs.dev'],
+ *   dnsResolvers: {
+ *     // this resolver will only be used for `.com` domains (note - this could
+ *     // also be an array of resolvers)
+ *     'com.': dnsJsonOverHttps('https://my-dns-resolver.example.com/dns-json'),
+ *     // this resolver will be used for everything else (note - this could
+ *     // also be an array of resolvers)
+ *     '.': dnsOverHttps('https://my-dns-resolver.example.com/dns-query')
+ *   }
+ * })
+ * ```
+ *
  * ### IPLD codec handling
  *
  * IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption.
@@ -569,10 +592,13 @@
 import { trustlessGateway } from '@helia/block-brokers'
 import { createHeliaHTTP } from '@helia/http'
 import { delegatedHTTPRouting } from '@helia/routers'
+import { dns } from '@multiformats/dns'
 import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js'
 import type { Helia } from '@helia/interface'
-import type { DNSResolver, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns'
+import type { ResolveDNSLinkProgressEvents } from '@helia/ipns'
 import type { GetEvents } from '@helia/unixfs'
+import type { DNSResolvers, DNS } from '@multiformats/dns'
+import type { DNSResolver } from '@multiformats/dns/resolvers'
 import type { CID } from 'multiformats/cid'
 import type { ProgressEvent, ProgressOptions } from 'progress-events'
 
@@ -618,7 +644,7 @@ export interface CreateVerifiedFetchInit {
    *
    * @default [dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'),dnsJsonOverHttps('https://dns.google/resolve')]
    */
-  dnsResolvers?: DNSResolver[]
+  dnsResolvers?: DNSResolver[] | DNSResolvers
 }
 
 export interface CreateVerifiedFetchOptions {
@@ -651,7 +677,7 @@ export type BubbledProgressEvents =
   // unixfs
   GetEvents |
   // ipns
-  ResolveProgressEvents | ResolveDnsLinkProgressEvents | IPNSRoutingEvents
+  ResolveDNSLinkProgressEvents
 
 export type VerifiedFetchProgressEvents =
   ProgressEvent<'verified-fetch:request:start', CIDDetail> |
@@ -674,20 +700,19 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions<BubbledP
  * Create and return a Helia node
  */
 export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchInit, options?: CreateVerifiedFetchOptions): Promise<VerifiedFetch> {
-  let dnsResolvers: DNSResolver[] | undefined
   if (!isHelia(init)) {
-    dnsResolvers = init?.dnsResolvers
     init = await createHeliaHTTP({
       blockBrokers: [
         trustlessGateway({
           gateways: init?.gateways
         })
       ],
-      routers: (init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl))
+      routers: (init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl)),
+      dns: createDns(init?.dnsResolvers)
     })
   }
 
-  const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, { dnsResolvers, ...options })
+  const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, options)
   async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise<Response> {
     return verifiedFetchInstance.fetch(resource, options)
   }
@@ -707,3 +732,19 @@ function isHelia (obj: any): obj is Helia {
     obj?.stop != null &&
     obj?.start != null
 }
+
+function createDns (resolvers?: DNSResolver[] | DNSResolvers): DNS | undefined {
+  if (resolvers == null) {
+    return
+  }
+
+  if (Array.isArray(resolvers)) {
+    return dns({
+      resolvers: {
+        '.': resolvers
+      }
+    })
+  }
+
+  return dns({ resolvers })
+}

package/src/types.ts

@@ -1 +1,3 @@
 export type RequestFormatShorthand = 'raw' | 'car' | 'tar' | 'ipns-record' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor'
+
+export type SupportedBodyTypes = string | ArrayBuffer | Blob | ReadableStream<Uint8Array> | null

package/src/utils/byte-range-context.ts

@@ -0,0 +1,303 @@
+import { calculateByteRangeIndexes, getHeader } from './request-headers.js'
+import { getContentRangeHeader } from './response-headers.js'
+import type { SupportedBodyTypes } from '../types.js'
+import type { ComponentLogger, Logger } from '@libp2p/interface'
+
+type SliceableBody = Exclude<SupportedBodyTypes, ReadableStream<Uint8Array> | null>
+
+/**
+ * Gets the body size of a given body if it's possible to calculate it synchronously.
+ */
+function getBodySizeSync (body: SupportedBodyTypes): number | null {
+  if (typeof body === 'string') {
+    return body.length
+  }
+  if (body instanceof ArrayBuffer || body instanceof Uint8Array) {
+    return body.byteLength
+  }
+  if (body instanceof Blob) {
+    return body.size
+  }
+
+  if (body instanceof ReadableStream) {
+    return null
+  }
+
+  return null
+}
+
+function getByteRangeFromHeader (rangeHeader: string): { start: string, end: string } {
+  /**
+   * Range: bytes=<start>-<end> | bytes=<start2>- | bytes=-<end2>
+   */
+  const match = rangeHeader.match(/^bytes=(?<start>\d+)?-(?<end>\d+)?$/)
+  if (match?.groups == null) {
+    throw new Error('Invalid range request')
+  }
+
+  const { start, end } = match.groups
+
+  return { start, end }
+}
+
+export class ByteRangeContext {
+  public readonly isRangeRequest: boolean
+
+  /**
+   * This property is purposefully only set in `set fileSize` and should not be set directly.
+   */
+  private _fileSize: number | null | undefined
+  private _body: SupportedBodyTypes = null
+  private readonly rangeRequestHeader: string | undefined
+  private readonly log: Logger
+  private readonly requestRangeStart: number | null
+  private readonly requestRangeEnd: number | null
+  private byteStart: number | undefined
+  private byteEnd: number | undefined
+  private byteSize: number | undefined
+
+  constructor (logger: ComponentLogger, private readonly headers?: HeadersInit) {
+    this.log = logger.forComponent('helia:verified-fetch:byte-range-context')
+    this.rangeRequestHeader = getHeader(this.headers, 'Range')
+    if (this.rangeRequestHeader != null) {
+      this.isRangeRequest = true
+      this.log.trace('range request detected')
+      try {
+        const { start, end } = getByteRangeFromHeader(this.rangeRequestHeader)
+        this.requestRangeStart = start != null ? parseInt(start) : null
+        this.requestRangeEnd = end != null ? parseInt(end) : null
+      } catch (e) {
+        this.log.error('error parsing range request header: %o', e)
+        this.requestRangeStart = null
+        this.requestRangeEnd = null
+      }
+
+      this.setOffsetDetails()
+    } else {
+      this.log.trace('no range request detected')
+      this.isRangeRequest = false
+      this.requestRangeStart = null
+      this.requestRangeEnd = null
+    }
+  }
+
+  public setBody (body: SupportedBodyTypes): void {
+    this._body = body
+    // if fileSize was already set, don't recalculate it
+    this.setFileSize(this._fileSize ?? getBodySizeSync(body))
+
+    this.log.trace('set request body with fileSize %o', this._fileSize)
+  }
+
+  public getBody (): SupportedBodyTypes {
+    const body = this._body
+    if (body == null) {
+      this.log.trace('body is null')
+      return body
+    }
+    if (!this.isRangeRequest || !this.isValidRangeRequest) {
+      this.log.trace('returning body unmodified for non-range, or invalid range, request')
+      return body
+    }
+    const byteStart = this.byteStart
+    const byteEnd = this.byteEnd
+    const byteSize = this.byteSize
+    if (byteStart != null || byteEnd != null) {
+      this.log.trace('returning body with byteStart=%o, byteEnd=%o, byteSize=%o', byteStart, byteEnd, byteSize)
+      if (body instanceof ReadableStream) {
+        // stream should already be spliced by `unixfs.cat`
+        return body
+      }
+      return this.getSlicedBody(body)
+    }
+
+    // we should not reach this point, but return body untouched.
+    this.log.error('returning unmodified body for valid range request')
+    return body
+  }
+
+  private getSlicedBody <T extends SliceableBody>(body: T): SliceableBody {
+    if (this.isPrefixLengthRequest) {
+      this.log.trace('sliced body with byteStart %o', this.byteStart)
+      return body.slice(this.offset) satisfies SliceableBody
+    }
+    if (this.isSuffixLengthRequest && this.length != null) {
+      this.log.trace('sliced body with length %o', -this.length)
+      return body.slice(-this.length) satisfies SliceableBody
+    }
+    const offset = this.byteStart ?? 0
+    const length = this.byteEnd == null ? undefined : this.byteEnd + 1
+    this.log.trace('returning body with offset %o and length %o', offset, length)
+
+    return body.slice(offset, length) satisfies SliceableBody
+  }
+
+  private get isSuffixLengthRequest (): boolean {
+    return this.requestRangeStart == null && this.requestRangeEnd != null
+  }
+
+  private get isPrefixLengthRequest (): boolean {
+    return this.requestRangeStart != null && this.requestRangeEnd == null
+  }
+
+  /**
+   * Sometimes, we need to set the fileSize explicitly because we can't calculate
+   * the size of the body (e.g. for unixfs content where we call .stat).
+   *
+   * This fileSize should otherwise only be called from `setBody`.
+   */
+  public setFileSize (size: number | bigint | null): void {
+    this._fileSize = size != null ? Number(size) : null
+    this.log.trace('set _fileSize to %o', this._fileSize)
+    // when fileSize changes, we need to recalculate the offset details
+    this.setOffsetDetails()
+  }
+
+  public getFileSize (): number | null | undefined {
+    return this._fileSize
+  }
+
+  private isValidByteStart (): boolean {
+    if (this.byteStart != null) {
+      if (this.byteStart < 0) {
+        return false
+      }
+      if (this._fileSize != null && this.byteStart > this._fileSize) {
+        return false
+      }
+    }
+    return true
+  }
+
+  private isValidByteEnd (): boolean {
+    if (this.byteEnd != null) {
+      if (this.byteEnd < 0) {
+        return false
+      }
+      if (this._fileSize != null && this.byteEnd > this._fileSize) {
+        return false
+      }
+    }
+    return true
+  }
+
+  /**
+   * We may get the values required to determine if this is a valid range request at different times
+   * so we need to calculate it when asked.
+   */
+  public get isValidRangeRequest (): boolean {
+    if (!this.isRangeRequest) {
+      return false
+    }
+    if (this.requestRangeStart == null && this.requestRangeEnd == null) {
+      this.log.trace('invalid range request, range request values not provided')
+      return false
+    }
+    if (!this.isValidByteStart()) {
+      this.log.trace('invalid range request, byteStart is less than 0 or greater than fileSize')
+      return false
+    }
+    if (!this.isValidByteEnd()) {
+      this.log.trace('invalid range request, byteEnd is less than 0 or greater than fileSize')
+      return false
+    }
+    if (this.requestRangeEnd != null && this.requestRangeStart != null) {
+      // we may not have enough info.. base check on requested bytes
+      if (this.requestRangeStart > this.requestRangeEnd) {
+        this.log.trace('invalid range request, start is greater than end')
+        return false
+      } else if (this.requestRangeStart < 0) {
+        this.log.trace('invalid range request, start is less than 0')
+        return false
+      } else if (this.requestRangeEnd < 0) {
+        this.log.trace('invalid range request, end is less than 0')
+        return false
+      }
+    }
+
+    return true
+  }
+
+  /**
+   * Given all the information we have, this function returns the offset that will be used when:
+   * 1. calling unixfs.cat
+   * 2. slicing the body
+   */
+  public get offset (): number {
+    if (this.byteStart === 0) {
+      return 0
+    }
+    if (this.isPrefixLengthRequest || this.isSuffixLengthRequest) {
+      if (this.byteStart != null) {
+        // we have to subtract by 1 because the offset is inclusive
+        return this.byteStart - 1
+      }
+    }
+
+    return this.byteStart ?? 0
+  }
+
+  /**
+   * Given all the information we have, this function returns the length that will be used when:
+   * 1. calling unixfs.cat
+   * 2. slicing the body
+   */
+  public get length (): number | undefined {
+    return this.byteSize ?? undefined
+  }
+
+  /**
+   * Converts a range request header into helia/unixfs supported range options
+   * Note that the gateway specification says we "MAY" support multiple ranges (https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header) but we don't
+   *
+   * Also note that @helia/unixfs and ipfs-unixfs-exporter expect length and offset to be numbers, the range header is a string, and the size of the resource is likely a bigint.
+   *
+   * SUPPORTED:
+   * Range: bytes=<range-start>-<range-end>
+   * Range: bytes=<range-start>-
+   * Range: bytes=-<suffix-length> // must pass size so we can calculate the offset. suffix-length is the number of bytes from the end of the file.
+   *
+   * NOT SUPPORTED:
+   * Range: bytes=<range-start>-<range-end>, <range-start>-<range-end>
+   * Range: bytes=<range-start>-<range-end>, <range-start>-<range-end>, <range-start>-<range-end>
+   *
+   * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range#directives
+   */
+  private setOffsetDetails (): void {
+    if (this.requestRangeStart == null && this.requestRangeEnd == null) {
+      this.log.trace('requestRangeStart and requestRangeEnd are null')
+      return
+    }
+
+    const { start, end, byteSize } = calculateByteRangeIndexes(this.requestRangeStart ?? undefined, this.requestRangeEnd ?? undefined, this._fileSize ?? undefined)
+    this.log.trace('set byteStart to %o, byteEnd to %o, byteSize to %o', start, end, byteSize)
+    this.byteStart = start
+    this.byteEnd = end
+    this.byteSize = byteSize
+  }
+
+  /**
+   * This function returns the value of the "content-range" header.
+   *
+   * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range
+   *
+   * Returns a string representing the following content ranges:
+   *
+   * @example
+   * - Content-Range: <unit> <byteStart>-<byteEnd>/<byteSize>
+   * - Content-Range: <unit> <byteStart>-<byteEnd>/*
+   */
+  // - Content-Range: <unit> */<byteSize> // this is purposefully not in jsdoc block
+  public get contentRangeHeaderValue (): string {
+    if (!this.isValidRangeRequest) {
+      this.log.error('cannot get contentRangeHeaderValue for invalid range request')
+      throw new Error('Invalid range request')
+    }
+
+    return getContentRangeHeader({
+      byteStart: this.byteStart,
+      byteEnd: this.byteEnd,
+      byteSize: this._fileSize ?? undefined
+    })
+  }
+}

package/src/utils/get-stream-from-async-iterable.ts

@@ -11,7 +11,7 @@ export async function getStreamFromAsyncIterable (iterator: AsyncIterable<Uint8A
   const { value: firstChunk, done } = await reader.next()
 
   if (done === true) {
-    log.error('No content found for path', path)
+    log.error('no content found for path', path)
     throw new Error('No content found')
   }
 

package/src/utils/parse-resource.ts

@@ -2,7 +2,7 @@ import { CID } from 'multiformats/cid'
 import { parseUrlString } from './parse-url-string.js'
 import type { ParsedUrlStringResults } from './parse-url-string.js'
 import type { Resource } from '../index.js'
-import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns'
+import type { IPNS, IPNSRoutingEvents, ResolveDNSLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressOptions } from 'progress-events'
 
@@ -11,7 +11,7 @@ export interface ParseResourceComponents {
   logger: ComponentLogger
 }
 
-export interface ParseResourceOptions extends ProgressOptions<ResolveProgressEvents | IPNSRoutingEvents | ResolveDnsLinkProgressEvents> {
+export interface ParseResourceOptions extends ProgressOptions<ResolveProgressEvents | IPNSRoutingEvents | ResolveDNSLinkProgressEvents> {
 
 }
 /**

package/src/utils/parse-url-string.ts

@@ -2,7 +2,7 @@ import { peerIdFromString } from '@libp2p/peer-id'
 import { CID } from 'multiformats/cid'
 import { TLRU } from './tlru.js'
 import type { RequestFormatShorthand } from '../types.js'
-import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
+import type { IPNS, ResolveDNSLinkProgressEvents, ResolveResult } from '@helia/ipns'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressOptions } from 'progress-events'
 
@@ -13,7 +13,7 @@ export interface ParseUrlStringInput {
   ipns: IPNS
   logger: ComponentLogger
 }
-export interface ParseUrlStringOptions extends ProgressOptions<ResolveProgressEvents | IPNSRoutingEvents | ResolveDnsLinkProgressEvents> {
+export interface ParseUrlStringOptions extends ProgressOptions<ResolveDNSLinkProgressEvents> {
 
 }
 
@@ -106,7 +106,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
       log.trace('resolved %s to %c from cache', cidOrPeerIdOrDnsLink, cid)
     } else {
       // protocol is ipns
-      log.trace('Attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink)
+      log.trace('attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink)
       let peerId = null
       try {
         peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
@@ -117,10 +117,10 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
         ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
       } catch (err) {
         if (peerId == null) {
-          log.error('Could not parse PeerId string "%s"', cidOrPeerIdOrDnsLink, err)
+          log.error('could not parse PeerId string "%s"', cidOrPeerIdOrDnsLink, err)
           errors.push(new TypeError(`Could not parse PeerId in ipns url "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`))
         } else {
-          log.error('Could not resolve PeerId %c', peerId, err)
+          log.error('could not resolve PeerId %c', peerId, err)
           errors.push(new TypeError(`Could not resolve PeerId "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`))
         }
       }
@@ -134,13 +134,13 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
         log.trace('Attempting to resolve DNSLink for %s', decodedDnsLinkLabel)
 
         try {
-          resolveResult = await ipns.resolveDns(decodedDnsLinkLabel, { onProgress: options?.onProgress })
+          resolveResult = await ipns.resolveDNSLink(decodedDnsLinkLabel, { onProgress: options?.onProgress })
           cid = resolveResult?.cid
           resolvedPath = resolveResult?.path
           log.trace('resolved %s to %c', decodedDnsLinkLabel, cid)
           ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
         } catch (err: any) {
-          log.error('Could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err)
+          log.error('could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err)
           errors.push(err)
         }
       }

package/src/utils/request-headers.ts

@@ -0,0 +1,51 @@
+export function getHeader (headers: HeadersInit | undefined, header: string): string | undefined {
+  if (headers == null) {
+    return undefined
+  }
+  if (headers instanceof Headers) {
+    return headers.get(header) ?? undefined
+  }
+  if (Array.isArray(headers)) {
+    const entry = headers.find(([key]) => key.toLowerCase() === header.toLowerCase())
+    return entry?.[1]
+  }
+  const key = Object.keys(headers).find(k => k.toLowerCase() === header.toLowerCase())
+  if (key == null) {
+    return undefined
+  }
+
+  return headers[key]
+}
+
+/**
+ * Given two ints from a Range header, and potential fileSize, returns:
+ * 1. number of bytes the response should contain.
+ * 2. the start index of the range. // inclusive
+ * 3. the end index of the range. // inclusive
+ */
+export function calculateByteRangeIndexes (start: number | undefined, end: number | undefined, fileSize?: number): { byteSize?: number, start?: number, end?: number } {
+  if (start != null && end != null) {
+    if (start > end) {
+      throw new Error('Invalid range')
+    }
+
+    return { byteSize: end - start + 1, start, end }
+  } else if (start == null && end != null) {
+    // suffix byte range requested
+    if (fileSize == null) {
+      return { end }
+    }
+    const result = { byteSize: end, start: fileSize - end + 1, end: fileSize }
+    return result
+  } else if (start != null && end == null) {
+    if (fileSize == null) {
+      return { start }
+    }
+    const byteSize = fileSize - start + 1
+    const end = fileSize
+    return { byteSize, start, end }
+  }
+
+  // both start and end are undefined
+  return { byteSize: fileSize }
+}

package/src/utils/response-headers.ts

@@ -0,0 +1,32 @@
+/**
+ * This function returns the value of the `Content-Range` header for a given range.
+ * If you know the total size of the body, pass it as `byteSize`
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range
+ */
+export function getContentRangeHeader ({ byteStart, byteEnd, byteSize }: { byteStart: number | undefined, byteEnd: number | undefined, byteSize: number | undefined }): string {
+  const total = byteSize ?? '*' // if we don't know the total size, we should use *
+
+  if (byteStart != null && byteEnd == null) {
+    // only byteStart in range
+    if (byteSize == null) {
+      return `bytes */${total}`
+    }
+    return `bytes ${byteStart}-${byteSize}/${byteSize}`
+  }
+
+  if (byteStart == null && byteEnd != null) {
+    // only byteEnd in range
+    if (byteSize == null) {
+      return `bytes */${total}`
+    }
+    return `bytes ${byteSize - byteEnd + 1}-${byteSize}/${byteSize}`
+  }
+
+  if (byteStart == null && byteEnd == null) {
+    // neither are provided, we can't return a valid range.
+    return `bytes */${total}`
+  }
+
+  return `bytes ${byteStart}-${byteEnd}/${total}`
+}