@helia/verified-fetch 4.0.1 → 4.0.2

package/src/utils/parse-url-string.ts

@@ -1,77 +1,194 @@
-const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<query>.*)$/
-const PATH_REGEX = /^\/(?<protocol>ip[fn]s)\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<query>.*)$/
-const PATH_GATEWAY_REGEX = /^https?:\/\/(.*[^/])\/(?<protocol>ip[fn]s)\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<query>.*)$/
-const SUBDOMAIN_GATEWAY_REGEX = /^https?:\/\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\.(?<protocol>ip[fn]s)\.([^/?]+)\/?(?<path>[^?]*)\??(?<query>.*)$/
+import { InvalidParametersError } from '@libp2p/interface'
+import { decodeDNSLinkLabel, isInlinedDnsLink } from './dnslink-label.ts'
+import { ipfsPathToUrl, ipfsUrlToUrl } from './ipfs-path-to-url.ts'
 
-interface MatchUrlGroups {
+interface SubdomainMatchGroups {
   protocol: 'ipfs' | 'ipns'
   cidOrPeerIdOrDnsLink: string
-  path?: string
-  query?: string
 }
 
-function matchUrlGroupsGuard (groups?: null | { [key in string]: string; } | MatchUrlGroups): groups is MatchUrlGroups {
+const SUBDOMAIN_GATEWAY_REGEX = /^(?<cidOrPeerIdOrDnsLink>[^/?]+)\.(?<protocol>ip[fn]s)\.([^/?]+)$/
+
+function matchSubdomainGroupsGuard (groups?: null | { [key in string]: string; } | SubdomainMatchGroups): groups is SubdomainMatchGroups {
   const protocol = groups?.protocol
-  if (protocol == null) { return false }
+
+  if (protocol !== 'ipfs' && protocol !== 'ipns') {
+    return false
+  }
+
   const cidOrPeerIdOrDnsLink = groups?.cidOrPeerIdOrDnsLink
-  if (cidOrPeerIdOrDnsLink == null) { return false }
-  const path = groups?.path
-  const query = groups?.query
-
-  return ['ipns', 'ipfs'].includes(protocol) &&
-    typeof cidOrPeerIdOrDnsLink === 'string' &&
-    (path == null || typeof path === 'string') &&
-    (query == null || typeof query === 'string')
+
+  if (cidOrPeerIdOrDnsLink == null) {
+    return false
+  }
+
+  return true
 }
 
-// TODO: can this be replaced with `new URL`?
-export function matchURLString (urlString: string): MatchUrlGroups {
-  for (const pattern of [SUBDOMAIN_GATEWAY_REGEX, URL_REGEX, PATH_GATEWAY_REGEX, PATH_REGEX]) {
-    const match = urlString.match(pattern)
+export interface ParsedURL {
+  url: URL,
+  protocol: 'ipfs' | 'ipns'
+  cidOrPeerIdOrDnsLink: string
+  path: string[]
+  query: Record<string, any>
+  fragment: string
+}
 
-    if (matchUrlGroupsGuard(match?.groups)) {
-      const groups = match.groups satisfies MatchUrlGroups
+function toQuery (query?: URLSearchParams): Record<string, any> {
+  if (query == null) {
+    return {}
+  }
 
-      if (groups.path != null) {
-        groups.path = decodeURIComponent(groups.path)
-      }
+  const output: Record<string, any> = {}
 
-      // decode inline dnslink domain if present
-      if (pattern === SUBDOMAIN_GATEWAY_REGEX && groups.protocol === 'ipns' && isInlinedDnsLink(groups.cidOrPeerIdOrDnsLink)) {
-        groups.cidOrPeerIdOrDnsLink = dnsLinkLabelDecoder(groups.cidOrPeerIdOrDnsLink)
-      }
+  for (const [key, value] of query.entries()) {
+    output[key] = value
+
+    if (value === 'true') {
+      output[key] = true
+    }
 
-      return groups
+    if (value === 'false') {
+      output[key] = false
     }
   }
 
-  throw new TypeError(`Invalid URL: ${urlString}, please use ipfs://, ipns://, or gateway URLs only`)
+  return output
 }
 
-/**
- * For DNSLink see https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header
- * DNSLink names include . which means they must be inlined into a single DNS label to provide unique origin and work with wildcard TLS certificates.
- */
+function stripLeadingHash (pathname: string): string {
+  return stripLeading(pathname, '#')
+}
 
-// DNS label can have up to 63 characters, consisting of alphanumeric
-// characters or hyphens -, but it must not start or end with a hyphen.
-const dnsLabelRegex = /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/
+function stripLeading (str: string, char: string): string {
+  while (str.startsWith(char)) {
+    str = str.substring(1)
+  }
+
+  return str
+}
 
 /**
- * Checks if label looks like inlined DNSLink.
- * (https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header)
+ * If the caller has passed a case-sensitive identifier (like a base58btc
+ * encoded CID or PeerId) in a case-insensitive location (like a subdomain),
+ * be nice and return the original identifier from the passed string
  */
-function isInlinedDnsLink (label: string): boolean {
-  return dnsLabelRegex.test(label) && label.includes('-') && !label.includes('.')
+function findOriginalCidOrPeer (needle: string, haystack: string): string {
+  const start = haystack.toLowerCase().indexOf(needle)
+
+  if (start === -1) {
+    return needle
+  }
+
+  return haystack.substring(start, start + needle.length)
+}
+
+function toUrl (urlString: string): URL {
+  // turn IPFS/IPNS path into gateway URL string
+  if (urlString.startsWith('/ipfs/') || urlString.startsWith('/ipns/')) {
+    urlString = ipfsPathToUrl(urlString)
+  }
+
+  // turn IPFS/IPNS URL into gateway URL string
+  if (urlString.startsWith('ipfs://') || urlString.startsWith('ipns://')) {
+    urlString = ipfsUrlToUrl(urlString)
+  }
+
+  if (urlString.startsWith('http://') || urlString.startsWith('https://')) {
+    return new URL(urlString)
+  }
+
+  throw new InvalidParametersError(`Invalid URL: ${urlString}`)
 }
 
 /**
- * DNSLink label decoding
- * - Every standalone - is replaced with .
- * - Every remaining -- is replaced with -
+ * Accepts the following url strings:
  *
- * @example en-wikipedia--on--ipfs-org.ipns.example.net -> example.net/ipns/en.wikipedia-on-ipfs.org
+ * - /ipfs/Qmfoo/path
+ * - /ipns/Qmfoo/path
+ * - ipfs://cid/path
+ * - ipns://name/path
+ * - http://cid.ipfs.example.com/path
+ * - http://name.ipns.example.com/path
+ * - http://example.com/ipfs/cid/path
+ * - http://example.com/ipns/name/path
  */
-function dnsLinkLabelDecoder (linkLabel: string): string {
-  return linkLabel.replace(/--/g, '%').replace(/-/g, '.').replace(/%/g, '-')
+export function parseURLString (urlString: string): ParsedURL {
+  // validate url
+  const url = toUrl(urlString)
+
+  // test for subdomain gateway URL
+  const subdomainMatch = url.hostname.match(SUBDOMAIN_GATEWAY_REGEX)
+
+  if (matchSubdomainGroupsGuard(subdomainMatch?.groups)) {
+    const groups = subdomainMatch.groups
+
+    if (groups.protocol === 'ipns' && isInlinedDnsLink(groups.cidOrPeerIdOrDnsLink)) {
+      // decode inline dnslink domain if present
+      groups.cidOrPeerIdOrDnsLink = decodeDNSLinkLabel(groups.cidOrPeerIdOrDnsLink)
+    }
+
+    const cidOrPeerIdOrDnsLink = findOriginalCidOrPeer(groups.cidOrPeerIdOrDnsLink, urlString)
+
+    // parse url as not http(s):// - this is necessary because URL makes
+    // `.pathname` default to `/` for http URLs, even if no trailing slash was
+    // present in the string URL and we need to be able to round-trip the user's
+    // input while also maintaining a sane canonical URL for the resource. Phew.
+    const wat = new URL(`not-${urlString}`)
+
+    return {
+      url: new URL(`${groups.protocol}://${cidOrPeerIdOrDnsLink}${wat.pathname}${url.search}${url.hash}`),
+      protocol: groups.protocol,
+      cidOrPeerIdOrDnsLink,
+      path: url.pathname.split('/')
+        .filter(str => str !== '')
+        .map(str => decodeURIComponent(str)),
+      query: toQuery(url.searchParams),
+      fragment: stripLeadingHash(url.hash)
+    }
+  }
+
+  // test for IPFS path gateway URL
+  if (url.pathname.startsWith('/ipfs/')) {
+    const parts = url.pathname.substring(6).split('/')
+    const cid = parts.shift()
+
+    if (cid == null) {
+      throw new InvalidParametersError(`Path gateway URL ${urlString} had no CID`)
+    }
+
+    return {
+      url: new URL(`ipfs://${cid}${url.pathname.replace(`/ipfs/${cid}`, '')}${url.search}${url.hash}`),
+      protocol: 'ipfs',
+      cidOrPeerIdOrDnsLink: cid,
+      path: parts
+        .filter(str => str !== '')
+        .map(str => decodeURIComponent(str)),
+      query: toQuery(url.searchParams),
+      fragment: stripLeadingHash(url.hash)
+    }
+  }
+
+  // test for IPNS path gateway URL
+  if (url.pathname.startsWith('/ipns/')) {
+    const parts = url.pathname.substring(6).split('/')
+    const name = parts.shift()
+
+    if (name == null) {
+      throw new InvalidParametersError(`Path gateway URL ${urlString} had no name`)
+    }
+
+    return {
+      url: new URL(`ipns://${name}${url.pathname.replace(`/ipns/${name}`, '')}${url.search}${url.hash}`),
+      protocol: 'ipns',
+      cidOrPeerIdOrDnsLink: name,
+      path: parts
+        .filter(str => str !== '')
+        .map(str => decodeURIComponent(str)),
+      query: toQuery(url.searchParams),
+      fragment: stripLeadingHash(url.hash)
+    }
+  }
+
+  throw new TypeError(`Invalid URL: ${urlString}, please use ipfs://, ipns://, or gateway URLs only`)
 }

package/src/utils/resource-to-cache-key.ts

@@ -1,5 +1,5 @@
 import { CID } from 'multiformats/cid'
-import { matchURLString } from './parse-url-string.js'
+import { parseURLString } from './parse-url-string.js'
 
 /**
  * Takes a resource and returns a session cache key as an IPFS or IPNS path with
@@ -24,7 +24,7 @@ export function resourceToSessionCacheKey (url: string | CID): string {
     return `ipfs://${CID.parse(url.toString())}`
   } catch {}
 
-  const { protocol, cidOrPeerIdOrDnsLink } = matchURLString(url.toString())
+  const { protocol, cidOrPeerIdOrDnsLink } = parseURLString(url.toString())
 
   return `${protocol}://${cidOrPeerIdOrDnsLink}`
 }

package/src/utils/responses.ts

@@ -16,6 +16,12 @@ function setType (response: Response, value: 'basic' | 'cors' | 'error' | 'opaqu
 }
 
 function setUrl (response: Response, value: string): void {
+  const fragmentStart = value.indexOf('#')
+
+  if (fragmentStart > -1) {
+    value = value.substring(0, fragmentStart)
+  }
+
   setField(response, 'url', value)
 }
 

package/src/utils/walk-path.ts

@@ -52,7 +52,7 @@ export function isObjectNode (node: UnixFSEntry): node is ObjectNode {
  */
 export async function handlePathWalking ({ cid, path, resource, options, blockstore, log }: PluginContext & { blockstore: Blockstore, log: Logger }): Promise<PathWalkerResponse | Response> {
   try {
-    return await walkPath(blockstore, `${cid}/${path}`, options)
+    return await walkPath(blockstore, `${cid}/${path.join('/')}`, options)
   } catch (err: any) {
     if (options?.signal?.aborted) {
       throw new AbortError(options?.signal?.reason)