@helia/verified-fetch 4.0.0 → 4.0.2

package/src/plugins/plugin-handle-raw.ts

@@ -3,7 +3,6 @@ import { code as rawCode } from 'multiformats/codecs/raw'
 import { identity } from 'multiformats/hashes/identity'
 import { getContentType } from '../utils/get-content-type.js'
 import { notFoundResponse, okRangeResponse } from '../utils/responses.js'
-import { PluginFatalError } from './errors.js'
 import { BasePlugin } from './plugin-base.js'
 import type { PluginContext } from './types.js'
 import type { AcceptHeader } from '../utils/select-output-type.ts'
@@ -49,10 +48,10 @@ export class RawPlugin extends BasePlugin {
   codes: number[] = [rawCode, identity.code]
 
   canHandle ({ cid, accept, query, byteRangeContext }: PluginContext): boolean {
-    this.log('checking if we can handle %c with accept %s', cid, accept)
     if (byteRangeContext == null) {
       return false
     }
+
     return accept?.mimeType === 'application/vnd.ipld.raw' || query.format === 'raw'
   }
 
@@ -66,21 +65,20 @@ export class RawPlugin extends BasePlugin {
       context.reqFormat = 'raw'
       context.query.download = true
       context.query.filename = context.query.filename ?? `${cid.toString()}.bin`
-      log.trace('Set content disposition...')
+      log.trace('set content disposition to force download')
     } else {
-      log.trace('Did NOT set content disposition...')
+      log.trace('did not set content disposition, raw block will display inline')
     }
 
-    if (path !== '' && cid.code === rawCode) {
+    if (path.length > 0 && cid.code === rawCode) {
       log.trace('404-ing raw codec request for %c/%s', cid, path)
-      // throw new PluginError('ERR_RAW_PATHS_NOT_SUPPORTED', 'Raw codec does not support paths')
-      // return notFoundResponse(resource, 'Raw codec does not support paths')
-      throw new PluginFatalError('ERR_RAW_PATHS_NOT_SUPPORTED', 'Raw codec does not support paths', { response: notFoundResponse(resource, 'Raw codec does not support paths') })
+      return notFoundResponse(resource)
     }
 
     const terminalCid = context.pathDetails?.terminalElement.cid ?? context.cid
     const blockstore = getBlockstore(terminalCid, resource, session, options)
     const result = await toBuffer(blockstore.get(terminalCid, options))
+
     context.byteRangeContext.setBody(result)
 
     // if the user has specified an `Accept` header that corresponds to a raw
@@ -94,6 +92,7 @@ export class RawPlugin extends BasePlugin {
       contentTypeParser,
       log
     })
+
     const response = okRangeResponse(resource, context.byteRangeContext.getBody(contentType), { byteRangeContext: context.byteRangeContext, log }, {
       redirected: false
     })

package/src/plugins/plugin-handle-tar.ts

@@ -14,11 +14,12 @@ import type { PluginContext } from './types.js'
 export class TarPlugin extends BasePlugin {
   readonly id = 'tar-plugin'
   readonly codes = []
+
   canHandle ({ cid, accept, query, byteRangeContext }: PluginContext): boolean {
-    this.log('checking if we can handle %c with accept %s', cid, accept)
     if (byteRangeContext == null) {
       return false
     }
+
     return accept?.mimeType === 'application/x-tar' || query.format === 'tar'
   }
 

package/src/plugins/types.ts

@@ -1,11 +1,10 @@
-import type { PluginError } from './errors.js'
-import type { ResolveURLResult, UrlQuery, VerifiedFetchInit, ContentTypeParser, RequestFormatShorthand } from '../index.js'
+import type { ResolveURLResult, VerifiedFetchInit, ContentTypeParser, RequestFormatShorthand } from '../index.js'
 import type { ByteRangeContext } from '../utils/byte-range-context.js'
 import type { AcceptHeader } from '../utils/select-output-type.ts'
 import type { ServerTiming } from '../utils/server-timing.ts'
 import type { PathWalkerResponse } from '../utils/walk-path.js'
 import type { IPNSResolver } from '@helia/ipns'
-import type { AbortOptions, ComponentLogger, Logger } from '@libp2p/interface'
+import type { AbortOptions, Logger } from '@libp2p/interface'
 import type { Helia } from 'helia'
 import type { Blockstore } from 'interface-blockstore'
 import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
@@ -18,7 +17,7 @@ import type { CustomProgressEvent } from 'progress-events'
  * - Persistent: Relevant even after the request completes (e.g., logging or metrics).
  */
 export interface PluginOptions {
-  logger: ComponentLogger
+  logger: Logger
   getBlockstore(cid: CID, resource: string | CID, useSession?: boolean, options?: AbortOptions): Blockstore
   contentTypeParser?: ContentTypeParser
   helia: Helia
@@ -32,8 +31,6 @@ export interface PluginOptions {
  * - Ephemeral: Typically discarded once fetch(...) completes.
  */
 export interface PluginContext extends ResolveURLResult {
-  readonly cid: CID
-  readonly path: string
   readonly resource: string
   readonly accept?: AcceptHeader
 
@@ -51,10 +48,8 @@ export interface PluginContext extends ResolveURLResult {
   options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions
   isDirectory?: boolean
   directoryEntries?: UnixFSEntry[]
-  errors?: PluginError[]
   reqFormat?: RequestFormatShorthand
   pathDetails?: PathWalkerResponse
-  query: UrlQuery
 
   /**
    * ByteRangeContext contains information about the size of the content and range requests.
@@ -65,6 +60,10 @@ export interface PluginContext extends ResolveURLResult {
   byteRangeContext?: ByteRangeContext
   serverTiming: ServerTiming
   ipfsPath: string
+
+  /**
+   * Allow arbitrary keys/values
+   */
   [key: string]: unknown
 }
 
@@ -85,7 +84,3 @@ export interface PluginErrorOptions {
   details?: Record<string, any>
   response?: Response
 }
-
-export interface FatalPluginErrorOptions extends PluginErrorOptions {
-  response: Response
-}

package/src/url-resolver.ts

@@ -1,7 +1,8 @@
 import { peerIdFromCID, peerIdFromString } from '@libp2p/peer-id'
 import { CID } from 'multiformats/cid'
-import { matchURLString } from './utils/parse-url-string.ts'
+import { parseURLString } from './utils/parse-url-string.ts'
 import type { ResolveURLOptions, ResolveURLResult, Resource, URLResolver as URLResolverInterface } from './index.ts'
+import type { ParsedURL } from './utils/parse-url-string.ts'
 import type { ServerTiming } from './utils/server-timing.ts'
 import type { DNSLink } from '@helia/dnslink'
 import type { IPNSResolver } from '@helia/ipns'
@@ -15,29 +16,6 @@ export interface URLResolverComponents {
   timing: ServerTiming
 }
 
-function toQuery (query?: string): Record<string, any> {
-  if (query == null) {
-    return {}
-  }
-
-  const params = new URLSearchParams(query)
-  const output: Record<string, any> = {}
-
-  for (const [key, value] of params.entries()) {
-    output[key] = value
-
-    if (value === 'true') {
-      output[key] = true
-    }
-
-    if (value === 'false') {
-      output[key] = false
-    }
-  }
-
-  return output
-}
-
 export class URLResolver implements URLResolverInterface {
   private readonly components: URLResolverComponents
 
@@ -53,56 +31,60 @@ export class URLResolver implements URLResolverInterface {
     const cid = CID.asCID(resource)
 
     if (cid != null) {
-      return this.resolveCIDResource(cid, '', {}, options)
+      return this.resolveCIDResource(cid, {
+        url: new URL(`ipfs://${cid}`)
+      }, options)
     }
 
     throw new TypeError(`Invalid resource. Cannot determine CID from resource: ${resource}`)
   }
 
   async parseUrlString (urlString: string, options: ResolveURLOptions = {}): Promise<ResolveURLResult> {
-    const { protocol, cidOrPeerIdOrDnsLink, path, query } = matchURLString(urlString)
+    const result = parseURLString(urlString)
 
-    if (protocol === 'ipfs') {
-      const cid = CID.parse(cidOrPeerIdOrDnsLink)
+    if (result.protocol === 'ipfs') {
+      const cid = CID.parse(result.cidOrPeerIdOrDnsLink)
 
-      return this.resolveCIDResource(cid, path ?? '', toQuery(query), options)
+      return this.resolveCIDResource(cid, result, options)
     }
 
-    if (protocol === 'ipns') {
+    if (result.protocol === 'ipns') {
       // try to parse target as peer id
       let peerId: PeerId
 
       try {
-        peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
+        peerId = peerIdFromString(result.cidOrPeerIdOrDnsLink)
       } catch {
         // fall back to DNSLink (e.g. /ipns/example.com)
-        return this.resolveDNSLink(cidOrPeerIdOrDnsLink, path ?? '', toQuery(query), options)
+        return this.resolveDNSLink(result.cidOrPeerIdOrDnsLink, result, options)
       }
 
       // parse multihash from string (e.g. /ipns/QmFoo...)
-      return this.resolveIPNSName(cidOrPeerIdOrDnsLink, peerId, path ?? '', toQuery(query), options)
+      return this.resolveIPNSName(result.cidOrPeerIdOrDnsLink, peerId, result, options)
     }
 
     throw new TypeError(`Invalid resource. Cannot determine CID from resource: ${urlString}`)
   }
 
-  async resolveCIDResource (cid: CID, path: string, query: Record<string, any>, options: ResolveURLOptions = {}): Promise<ResolveURLResult> {
+  async resolveCIDResource (cid: CID, parsed: Partial<ParsedURL> & Pick<ParsedURL, 'url'>, options: ResolveURLOptions = {}): Promise<ResolveURLResult> {
     if (cid.code === CODEC_LIBP2P_KEY) {
       // special case - peer id encoded as a CID
-      return this.resolveIPNSName(cid.toString(), peerIdFromCID(cid), path, query, options)
+      return this.resolveIPNSName(cid.toString(), peerIdFromCID(cid), parsed, options)
     }
 
     return {
+      url: parsed.url,
       cid,
       protocol: 'ipfs',
-      query,
-      path,
+      query: parsed.query ?? {},
+      path: parsed.path ?? [],
+      fragment: parsed.fragment ?? '',
       ttl: 29030400, // 1 year for ipfs content
-      ipfsPath: `/ipfs/${cid}${path === '' ? '' : `/${path}`}`
+      ipfsPath: `/ipfs/${cid}${parsed.url.pathname}`
     }
   }
 
-  async resolveDNSLink (domain: string, path: string, query: Record<string, any>, options?: ResolveURLOptions): Promise<ResolveURLResult> {
+  async resolveDNSLink (domain: string, parsed: ParsedURL, options?: ResolveURLOptions): Promise<ResolveURLResult> {
     const results = await this.components.timing.time('dnsLink.resolve', `Resolve DNSLink ${domain}`, this.components.dnsLink.resolve(domain, options))
     const result = results?.[0]
 
@@ -112,7 +94,7 @@ export class URLResolver implements URLResolverInterface {
 
     // dnslink resolved to IPNS name
     if (result.namespace === 'ipns') {
-      return this.resolveIPNSName(domain, result.peerId, path, query, options)
+      return this.resolveIPNSName(domain, result.peerId, parsed, options)
     }
 
     // dnslink resolved to CID
@@ -122,38 +104,37 @@ export class URLResolver implements URLResolverInterface {
     }
 
     return {
+      url: parsed.url,
       cid: result.cid,
-      path: concatPaths(result.path, path),
+      path: concatPaths(...(result.path ?? '').split('/'), ...(parsed.path ?? [])),
+      fragment: parsed.fragment,
       // dnslink is mutable so return 'ipns' protocol so we do not include immutable in cache-control header
       protocol: 'ipns',
       ttl: result.answer.TTL,
-      query,
-      ipfsPath: `/ipns/${domain}${path === '' ? '' : `/${path}`}`
+      query: parsed.query,
+      ipfsPath: `/ipns/${domain}${parsed.url.pathname}`
     }
   }
 
-  async resolveIPNSName (resource: string, key: PeerId, path: string, query: Record<string, any>, options?: AbortOptions): Promise<ResolveURLResult> {
+  async resolveIPNSName (resource: string, key: PeerId, parsed: Partial<ParsedURL> & Pick<ParsedURL, 'url'>, options?: AbortOptions): Promise<ResolveURLResult> {
     const result = await this.components.timing.time('ipns.resolve', `Resolve IPNS name ${key}`, this.components.ipnsResolver.resolve(key, options))
 
     return {
+      url: parsed.url,
       cid: result.cid,
-      path: concatPaths(result.path, path),
-      query,
+      path: concatPaths(...(result.path ?? '').split('/'), ...(parsed.path ?? [])),
+      query: parsed.query ?? {},
+      fragment: parsed.fragment ?? '',
       protocol: 'ipns',
       // IPNS ttl is in nanoseconds, convert to seconds
       ttl: Number((result.record.ttl ?? 0n) / BigInt(1e9)),
-      ipfsPath: `/ipns/${resource}${path === '' ? '' : `/${path}`}`
+      ipfsPath: `/ipns/${resource}${parsed.url.pathname}`
     }
   }
 }
 
-function concatPaths (...paths: Array<string | undefined>): string {
-  return `${
-    paths
-      .filter(p => p != null && p !== '')
-      .join('/')
-      .replaceAll(/(\/+)/g, '/')
-      .replace(/^(\/)+/, '')
-      .replace(/(\/)+$/, '/')
-  }`
+function concatPaths (...paths: Array<string | undefined>): string[] {
+  // @ts-expect-error undefined is filtered out
+  return paths
+    .filter(p => p != null && p !== '')
 }

package/src/utils/byte-range-context.ts

@@ -3,7 +3,7 @@ import { InvalidRangeError } from '../errors.js'
 import { calculateByteRangeIndexes, getHeader } from './request-headers.js'
 import { getContentRangeHeader } from './response-headers.js'
 import type { SupportedBodyTypes } from '../index.js'
-import type { ComponentLogger, Logger } from '@libp2p/interface'
+import type { Logger } from '@libp2p/interface'
 
 type SliceableBody = Exclude<SupportedBodyTypes, ReadableStream<Uint8Array> | null>
 
@@ -90,8 +90,8 @@ export class ByteRangeContext {
   // to be set by isValidRangeRequest so that we don't need to re-check the byteRanges
   private _isValidRangeRequest: boolean = false
 
-  constructor (logger: ComponentLogger, private readonly headers?: HeadersInit) {
-    this.log = logger.forComponent('helia:verified-fetch:byte-range-context')
+  constructor (logger: Logger, private readonly headers?: HeadersInit) {
+    this.log = logger.newScope('byte-range-context')
     this.rangeRequestHeader = getHeader(this.headers, 'Range')
 
     if (this.rangeRequestHeader != null) {

package/src/utils/content-type-parser.ts

@@ -1,28 +1,22 @@
-import { logger } from '@libp2p/logger'
 import { fileTypeFromBuffer } from 'file-type'
 
-const log = logger('helia:verified-fetch:content-type-parser')
-
 export const defaultMimeType = 'application/octet-stream'
 function checkForSvg (text: string): boolean {
-  log('checking for svg')
   return /^(<\?xml[^>]+>)?[^<^\w]+<svg/ig.test(text)
 }
 
 async function checkForJson (text: string): Promise<boolean> {
-  log('checking for json')
   try {
     JSON.parse(text)
     return true
   } catch (err) {
-    log('failed to parse as json', err)
     return false
   }
 }
 
 function getText (bytes: Uint8Array): string | null {
-  log('checking for text')
   const decoder = new TextDecoder('utf-8', { fatal: true })
+
   try {
     return decoder.decode(bytes)
   } catch (err) {
@@ -31,25 +25,24 @@ function getText (bytes: Uint8Array): string | null {
 }
 
 async function checkForHtml (text: string): Promise<boolean> {
-  log('checking for html')
   return /^\s*<(?:!doctype\s+html|html|head|body)\b/i.test(text)
 }
 
 export async function contentTypeParser (bytes: Uint8Array, fileName?: string): Promise<string> {
-  log('contentTypeParser called for fileName: %s, byte size=%s', fileName, bytes.length)
   const detectedType = (await fileTypeFromBuffer(bytes))?.mime
+
   if (detectedType != null) {
-    log('detectedType: %s', detectedType)
     if (detectedType === 'application/xml' && fileName?.toLowerCase().endsWith('.svg')) {
       return 'image/svg+xml'
     }
+
     return detectedType
   }
-  log('no detectedType')
 
   if (fileName == null) {
     // it's likely text... no other way to determine file-type.
     const text = getText(bytes)
+
     if (text != null) {
       // check for svg, json, html, or it's plain text.
       if (checkForSvg(text)) {
@@ -62,6 +55,7 @@ export async function contentTypeParser (bytes: Uint8Array, fileName?: string):
         return 'text/plain; charset=utf-8'
       }
     }
+
     return defaultMimeType
   }
 

package/src/utils/dnslink-label.ts

@@ -0,0 +1,38 @@
+/**
+ * For DNSLink see https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header
+ * DNSLink names include . which means they must be inlined into a single DNS label to provide unique origin and work with wildcard TLS certificates.
+ */
+
+// DNS label can have up to 63 characters, consisting of alphanumeric
+// characters or hyphens -, but it must not start or end with a hyphen.
+const dnsLabelRegex = /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/
+
+/**
+ * Checks if label looks like inlined DNSLink.
+ * (https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header)
+ */
+export function isInlinedDnsLink (label: string): boolean {
+  return dnsLabelRegex.test(label) && label.includes('-') && !label.includes('.')
+}
+
+/**
+ * DNSLink label decoding
+ * - Every standalone - is replaced with .
+ * - Every remaining -- is replaced with -
+ *
+ * @example en-wikipedia--on--ipfs-org -> en.wikipedia-on-ipfs.org
+ */
+export function decodeDNSLinkLabel (label: string): string {
+  return label.replace(/--/g, '%').replace(/-/g, '.').replace(/%/g, '-')
+}
+
+/**
+ * DNSLink label encoding
+ * - Every - is replaced with --
+ * - Every . is replaced with -
+ *
+ * @example en.wikipedia-on-ipfs.org -> en-wikipedia--on--ipfs-org
+ */
+export function encodeDNSLinkLabel (name: string): string {
+  return name.replace(/-/g, '--').replace(/\./g, '-')
+}

package/src/utils/error-to-object.ts

@@ -0,0 +1,22 @@
+function isAggregateError (err?: any): err is AggregateError {
+  return err instanceof AggregateError || (err?.name === 'AggregateError' && Array.isArray(err?.errors))
+}
+
+/**
+ * Error instance properties are not enumerable so we must transform the error
+ * into a plain object if we want to pass it to `JSON.stringify` or similar.
+ */
+export function errorToObject (err: Error): any {
+  let errors
+
+  if (isAggregateError(err)) {
+    errors = err.errors.map(err => errorToObject(err))
+  }
+
+  return {
+    name: err.name,
+    message: err.message,
+    stack: err.stack,
+    errors
+  }
+}

package/src/utils/get-content-type.ts

@@ -5,7 +5,7 @@ import type { Logger } from '@libp2p/interface'
 
 export interface GetContentTypeOptions {
   bytes: Uint8Array
-  path?: string
+  path?: string[]
   defaultContentType?: string
   contentTypeParser?: ContentTypeParser
   log: Logger
@@ -25,7 +25,7 @@ export async function getContentType ({ bytes, path, contentTypeParser, log, def
     try {
       let fileName
       if (filenameParam == null) {
-        fileName = path?.split('/').pop()?.trim()
+        fileName = path?.[path.length - 1]?.trim()
         fileName = (fileName === '' || fileName?.split('.').length === 1) ? undefined : fileName
       } else {
         fileName = filenameParam

package/src/utils/get-stream-from-async-iterable.ts

@@ -2,18 +2,15 @@ import { AbortError } from '@libp2p/interface'
 import { CustomProgressEvent } from 'progress-events'
 import { NoContentError } from '../errors.js'
 import type { VerifiedFetchInit } from '../index.js'
-import type { ComponentLogger } from '@libp2p/interface'
 
 /**
  * Converts an async iterator of Uint8Array bytes to a stream and returns the first chunk of bytes
  */
-export async function getStreamFromAsyncIterable (iterator: AsyncIterable<Uint8Array>, path: string, logger: ComponentLogger, options?: Pick<VerifiedFetchInit, 'onProgress' | 'signal'>): Promise<{ stream: ReadableStream<Uint8Array>, firstChunk: Uint8Array }> {
-  const log = logger.forComponent('helia:verified-fetch:get-stream-from-async-iterable')
+export async function getStreamFromAsyncIterable (iterator: AsyncIterable<Uint8Array>, options?: Pick<VerifiedFetchInit, 'onProgress' | 'signal'>): Promise<{ stream: ReadableStream<Uint8Array>, firstChunk: Uint8Array }> {
   const reader = iterator[Symbol.asyncIterator]()
   const { value: firstChunk, done } = await reader.next()
 
   if (done === true) {
-    log.error('no content found for path', path)
     throw new NoContentError()
   }
 

package/src/utils/handle-redirects.ts

@@ -1,5 +1,5 @@
 import { SubdomainNotSupportedError } from '../errors.js'
-import { matchURLString } from './parse-url-string.js'
+import { parseURLString } from './parse-url-string.js'
 import { movedPermanentlyResponse } from './responses.js'
 import type { VerifiedFetchInit, Resource } from '../index.js'
 import type { AbortOptions, ComponentLogger } from '@libp2p/interface'
@@ -47,7 +47,7 @@ export async function getRedirectResponse ({ resource, options, logger, cid, fet
   // if x-forwarded-host is passed, we need to set the location header to the
   // subdomain so that the browser can redirect to the correct subdomain
   try {
-    const urlParts = matchURLString(resource)
+    const urlParts = parseURLString(resource)
     const reqUrl = new URL(resource)
     const actualHost = forwardedHost ?? reqUrl.host
     const subdomainUrl = new URL(reqUrl)

package/src/utils/ipfs-path-to-url.ts

@@ -0,0 +1,54 @@
+import { InvalidParametersError } from '@libp2p/interface'
+
+/**
+ * Turns an IPFS or IPNS path into a HTTP URL. Path gateway syntax is used to
+ * preserve any case sensitivity
+ *
+ * - `/ipfs/cid` -> `https://example.org/ipfs/cid`
+ * - `/ipns/name` -> `https://example.org/ipns/name`
+ */
+export function ipfsPathToUrl (path: string): string {
+  if (!path.startsWith('/ipfs/') && !path.startsWith('/ipns/')) {
+    throw new InvalidParametersError(`Path ${path} did not start with /ipfs/ or /ipns/`)
+  }
+
+  // trim fragment
+  const fragmentIndex = path.indexOf('#')
+  let fragment = ''
+
+  if (fragmentIndex > -1) {
+    fragment = path.substring(fragmentIndex)
+    path = path.substring(0, fragmentIndex)
+  }
+
+  // trim query
+  const queryIndex = path.indexOf('?')
+  let query = ''
+
+  if (queryIndex > -1) {
+    query = path.substring(queryIndex)
+    path = path.substring(0, queryIndex)
+  }
+
+  const type = path.substring(1, 5)
+  const rest = path.substring(6)
+
+  return `https://example.org/${type}/${rest}${query}${fragment}`
+}
+
+/**
+ * Turns an IPFS or IPNS URL into a HTTP URL. Path gateway syntax is used to
+ * preserve and case sensitivity
+ *
+ * `ipfs://cid` -> `https://example.org/ipfs/cid`
+ */
+export function ipfsUrlToUrl (url: string): string {
+  if (!url.startsWith('ipfs://') && !url.startsWith('ipns://')) {
+    throw new InvalidParametersError(`URL ${url} did not start with ipfs:// or ipns://`)
+  }
+
+  const type = url.substring(0, 4)
+  const rest = url.substring(7)
+
+  return `https://example.org/${type}/${rest}`
+}