@helia/verified-fetch 5.1.1 → 6.1.0

package/src/plugins/plugin-handle-unixfs.ts

@@ -1,7 +1,8 @@
 import { code as dagPbCode } from '@ipld/dag-pb'
 import { isPromise } from '@libp2p/utils'
-import { exporter } from 'ipfs-unixfs-exporter'
+import { exporter, walkPath } from 'ipfs-unixfs-exporter'
 import first from 'it-first'
+import last from 'it-last'
 import itToBrowserReadableStream from 'it-to-browser-readablestream'
 import toBuffer from 'it-to-buffer'
 import * as raw from 'multiformats/codecs/raw'
@@ -9,27 +10,28 @@ import { MEDIA_TYPE_OCTET_STREAM, MEDIA_TYPE_DAG_PB } from '../utils/content-typ
 import { getContentDispositionFilename } from '../utils/get-content-disposition-filename.ts'
 import { badGatewayResponse, movedPermanentlyResponse, partialContentResponse, okResponse } from '../utils/responses.js'
 import { BasePlugin } from './plugin-base.js'
-import type { PluginContext } from '../index.js'
+import type { PluginContext, Resource } from '../index.js'
 import type { RangeHeader } from '../utils/get-range-header.ts'
 import type { AbortOptions } from '@libp2p/interface'
 import type { IdentityNode, RawNode, UnixFSEntry, UnixFSFile } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
 
 /**
  * @see https://specs.ipfs.tech/http-gateways/path-gateway/#use-in-directory-url-normalization
  */
-function getRedirectUrl (resource: string, url: URL, terminalElement: UnixFSEntry): string | undefined {
+function getDirectoryRedirectUrl (resource: Resource, url: URL): string | undefined {
   let uri: URL
 
   try {
     // try the requested resource
-    uri = new URL(resource)
+    uri = new URL(resource.toString())
   } catch {
     // fall back to the canonical URL
     uri = url
   }
 
   // directories must be requested with a trailing slash
-  if (terminalElement?.type === 'directory' && !uri.pathname.endsWith('/')) {
+  if (!uri.pathname.endsWith('/')) {
     // make sure we append slash to end of the path
     uri.pathname += '/'
 
@@ -54,12 +56,27 @@ export class UnixFSPlugin extends BasePlugin {
   }
 
   async handle (context: PluginContext): Promise<Response> {
-    let { url, resource, terminalElement, ipfsRoots } = context
+    let { url, resource, terminalElement, ipfsRoots, blockstore } = context
     let filename = url.searchParams.get('filename') ?? terminalElement.name
     let redirected: undefined | true
+    let entry: UnixFSEntry
 
-    if (terminalElement.type === 'directory') {
-      const redirectUrl = getRedirectUrl(resource, url, terminalElement)
+    try {
+      entry = await exporter(terminalElement.cid, blockstore, context.options)
+    } catch (err: any) {
+      // throw abort error if signal was aborted
+      context?.options?.signal?.throwIfAborted()
+
+      if (err.name === 'BlockNotFoundWhileOfflineError') {
+        throw err
+      }
+
+      this.log.error('could not export %c - $e', terminalElement.cid, err)
+      return badGatewayResponse(resource, 'Unable to stream content')
+    }
+
+    if (entry.type === 'directory') {
+      const redirectUrl = getDirectoryRedirectUrl(resource, url)
 
       if (redirectUrl != null) {
         this.log.trace('directory url normalization spec requires redirect')
@@ -86,20 +103,26 @@ export class UnixFSPlugin extends BasePlugin {
         const rootFilePath = 'index.html'
 
         try {
-          this.log.trace('found directory at %c/%s, looking for index.html', dirCid, url.pathname)
+          this.log.trace('found directory at %c%s, looking for index.html', dirCid, url.pathname)
 
-          const entry = await context.serverTiming.time('exporter-dir', '', exporter(`/ipfs/${dirCid}/${rootFilePath}`, context.blockstore, context.options))
+          const indexFile = await context.serverTiming.time('exporter-dir', '', last(walkPath(`/ipfs/${dirCid}/${rootFilePath}`, context.blockstore, context.options)))
 
-          if (entry.type === 'directory' || entry.type === 'object') {
+          if (indexFile == null) {
+            return badGatewayResponse(resource, 'Unable to stream content')
+          }
+
+          const indexFileEntry = await context.serverTiming.time('exporter-dir', '', exporter(indexFile.cid, context.blockstore, context.options))
+
+          if (indexFileEntry.type !== 'file' && indexFileEntry.type !== 'raw' && indexFileEntry.type !== 'identity') {
             return badGatewayResponse(resource, 'Unable to stream content')
           }
 
           // use `index.html` as the file name to help with content types
           filename = rootFilePath
 
-          this.log.trace('found directory index at %c/%s with cid %c', dirCid, rootFilePath, entry.cid)
+          this.log.trace('found directory index at %c%s with cid %c', dirCid, rootFilePath, entry.cid)
 
-          return await this.streamFile(resource, entry, filename, redirected, context.range, context.options)
+          return await this.streamFile(resource, indexFileEntry, filename, ipfsRoots, redirected, context.range, context.options)
         } catch (err: any) {
           if (err.name !== 'NotFoundError') {
             this.log.error('error loading path %c/%s - %e', dirCid, rootFilePath, err)
@@ -123,16 +146,16 @@ export class UnixFSPlugin extends BasePlugin {
         },
         redirected
       })
-    } else if (terminalElement.type === 'file' || terminalElement.type === 'raw' || terminalElement.type === 'identity') {
+    } else if (entry.type === 'file' || entry.type === 'raw' || entry.type === 'identity') {
       this.log('streaming file')
-      return this.streamFile(resource, terminalElement, filename, redirected, context.range, context.options)
+      return this.streamFile(resource, entry, filename, ipfsRoots, redirected, context.range, context.options)
     } else {
-      this.log.error('cannot stream terminal element type %s', terminalElement.type)
+      this.log.error('cannot stream terminal element type %s', entry.type)
       return badGatewayResponse(resource, 'Unable to stream content')
     }
   }
 
-  private async streamFile (resource: string, entry: UnixFSFile | RawNode | IdentityNode, filename: string, redirected?: boolean, rangeHeader?: RangeHeader, options?: AbortOptions): Promise<Response> {
+  private async streamFile (resource: Resource, entry: UnixFSFile | RawNode | IdentityNode, filename: string, ipfsRoots: CID[], redirected?: boolean, rangeHeader?: RangeHeader, options?: AbortOptions): Promise<Response> {
     let contentType = MEDIA_TYPE_OCTET_STREAM
 
     // only detect content type for non-range requests to avoid loading blocks
@@ -154,7 +177,7 @@ export class UnixFSPlugin extends BasePlugin {
           'content-disposition': `inline; ${
             getContentDispositionFilename(filename)
           }`,
-          'x-ipfs-roots': entry.cid.toString(),
+          'x-ipfs-roots': ipfsRoots.map(cid => cid.toV1()).join(','),
           'accept-ranges': 'bytes'
         },
         redirected
@@ -170,7 +193,7 @@ export class UnixFSPlugin extends BasePlugin {
         'content-disposition': `inline; ${
           getContentDispositionFilename(filename)
         }`,
-        'x-ipfs-roots': entry.cid.toString(),
+        'x-ipfs-roots': ipfsRoots.map(cid => cid.toV1()).join(','),
         'accept-ranges': 'bytes'
       },
       redirected

package/src/url-resolver.ts

@@ -1,16 +1,9 @@
 import { DoesNotExistError } from '@helia/unixfs/errors'
-import * as dagCbor from '@ipld/dag-cbor'
-import * as dagJson from '@ipld/dag-json'
-import * as dagPb from '@ipld/dag-pb'
 import { peerIdFromString } from '@libp2p/peer-id'
 import { InvalidParametersError, walkPath } from 'ipfs-unixfs-exporter'
-import toBuffer from 'it-to-buffer'
 import { CID } from 'multiformats/cid'
-import * as json from 'multiformats/codecs/json'
-import * as raw from 'multiformats/codecs/raw'
 import QuickLRU from 'quick-lru'
-import { SESSION_CACHE_MAX_SIZE, SESSION_CACHE_TTL_MS, CODEC_CBOR, CODEC_IDENTITY } from './constants.ts'
-import { resourceToSessionCacheKey } from './utils/resource-to-cache-key.ts'
+import { SESSION_CACHE_MAX_SIZE, SESSION_CACHE_TTL_MS } from './constants.ts'
 import { ServerTiming } from './utils/server-timing.ts'
 import type { ResolveURLResult, URLResolver as URLResolverInterface } from './index.ts'
 import type { DNSLink } from '@helia/dnslink'
@@ -18,49 +11,30 @@ import type { IPNSResolver } from '@helia/ipns'
 import type { AbortOptions } from '@libp2p/interface'
 import type { Helia, SessionBlockstore } from 'helia'
 import type { Blockstore } from 'interface-blockstore'
-import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
+import type { PathEntry } from 'ipfs-unixfs-exporter'
 
 // 1 year in seconds for ipfs content
 const IPFS_CONTENT_TTL = 29030400
 
-const ENTITY_CODECS = [
-  CODEC_CBOR,
-  json.code,
-  raw.code
-]
-
-/**
- * These are supported by the UnixFS exporter
- */
-const EXPORTABLE_CODECS = [
-  dagPb.code,
-  dagCbor.code,
-  dagJson.code,
-  raw.code
-]
-
 interface GetBlockstoreOptions extends AbortOptions {
   session?: boolean
 }
 
 export interface WalkPathResult {
   ipfsRoots: CID[]
-  terminalElement: UnixFSEntry
+  terminalElement: PathEntry
   blockstore: Blockstore
 }
 
-function basicEntry (type: 'raw' | 'object', cid: CID, bytes: Uint8Array): UnixFSEntry {
+function basicEntry (cid: CID): PathEntry {
   return {
+    cid,
     name: cid.toString(),
     path: cid.toString(),
-    depth: 0,
-    type,
-    node: bytes,
-    cid,
-    size: BigInt(bytes.byteLength),
-    content: async function * () {
-      yield bytes
-    }
+    roots: [
+      cid
+    ],
+    remainder: []
   }
 }
 
@@ -78,7 +52,6 @@ export interface URLResolverInit {
 export interface ResolveURLOptions extends AbortOptions {
   session?: boolean
   isRawBlockRequest?: boolean
-  onlyIfCached?: boolean
 }
 
 export class URLResolver implements URLResolverInterface {
@@ -118,7 +91,7 @@ export class URLResolver implements URLResolverInterface {
       return this.components.helia.blockstore
     }
 
-    const key = resourceToSessionCacheKey(root)
+    const key = `ipfs:${root}`
     let session = this.blockstoreSessions.get(key)
 
     if (session == null) {
@@ -196,16 +169,14 @@ export class URLResolver implements URLResolverInterface {
     const cid = CID.parse(url.hostname)
     const blockstore = this.getBlockstore(cid, options)
 
-    if (EXPORTABLE_CODECS.includes(cid.code)) {
+    try {
       const ipfsRoots: CID[] = []
-      let terminalElement: UnixFSEntry | undefined
+      let terminalElement: PathEntry | undefined
       const ipfsPath = toIPFSPath(url)
 
-      // @ts-expect-error offline is a helia option
       for await (const entry of walkPath(ipfsPath, blockstore, {
         ...options,
-        offline: options.onlyIfCached === true,
-        extended: options.isRawBlockRequest !== true
+        yieldSubShards: true
       })) {
         ipfsRoots.push(entry.cid)
         terminalElement = entry
@@ -220,31 +191,17 @@ export class URLResolver implements URLResolverInterface {
         terminalElement,
         blockstore
       }
-    }
-
-    let bytes: Uint8Array
-
-    if (cid.multihash.code === CODEC_IDENTITY) {
-      bytes = cid.multihash.digest
-    } else {
-      bytes = await toBuffer(blockstore.get(cid, options))
-    }
-
-    // entity codecs contain all the bytes for an entity in one block and no
-    // path walking outside of that block is possible
-    if (ENTITY_CODECS.includes(cid.code)) {
-      return {
-        ipfsRoots: [cid],
-        terminalElement: basicEntry('object', cid, bytes),
-        blockstore
+    } catch (err: any) {
+      if (err.name === 'NoResolverError') {
+        // may be an unknown codec
+        return {
+          ipfsRoots: [cid],
+          terminalElement: basicEntry(cid),
+          blockstore
+        }
       }
-    }
 
-    // may be an unknown codec
-    return {
-      ipfsRoots: [cid],
-      terminalElement: basicEntry('raw', cid, bytes),
-      blockstore
+      throw err
     }
   }
 }

package/src/utils/error-to-response.ts

@@ -1,45 +1,48 @@
 import { badGatewayResponse, gatewayTimeoutResponse, internalServerErrorResponse, notFoundResponse, preconditionFailedResponse } from './responses.js'
 import type { Resource } from '../index.js'
 
-export function errorToResponse (resource: Resource | string, err: any): Response {
-  // if a signal abort caused the error, throw the error
-  if (err.name === 'AbortError') {
+export function errorToResponse (resource: Resource | string, err: any, init?: RequestInit): Response {
+  // throw an AbortError if the passed signal has aborted
+  init?.signal?.throwIfAborted()
+
+  // rethrow these errors
+  if (['AbortError', 'InvalidParametersError'].includes(err.name)) {
     throw err
   }
 
   // could not reach an upstream server, bad connection or offline
   if (err.code === 'ECONNREFUSED' || err.code === 'ECANCELLED' || err.name === 'DNSQueryFailedError') {
-    return gatewayTimeoutResponse(resource.toString(), err)
+    return gatewayTimeoutResponse(resource, err)
   }
 
   // data was not parseable, user may be able to request raw block
   if (['NotUnixFSError'].includes(err.name)) {
-    return badGatewayResponse(resource.toString(), err)
+    return badGatewayResponse(resource, err)
   }
 
   // an upstream server didn't respond in time but inside the signal timeout
   if (err.code === 'ETIMEOUT' || err.name === 'TimeoutError') {
-    return gatewayTimeoutResponse(resource.toString(), err)
+    return gatewayTimeoutResponse(resource, err)
   }
 
   // path was not under DAG root
-  if (['ERR_NO_PROP', 'ERR_NO_TERMINAL_ELEMENT', 'ERR_NOT_FOUND'].includes(err.code)) {
-    return notFoundResponse(resource.toString())
+  if (['ERR_BAD_PATH', 'ERR_NO_TERMINAL_ELEMENT', 'ERR_NOT_FOUND'].includes(err.code)) {
+    return notFoundResponse(resource)
   }
 
   // path was not under DAG root
   if (['DoesNotExistError'].includes(err.name)) {
-    return notFoundResponse(resource.toString())
+    return notFoundResponse(resource)
   }
 
   if (['BlockNotFoundWhileOfflineError'].includes(err.name)) {
-    return preconditionFailedResponse(resource.toString())
+    return preconditionFailedResponse(resource)
   }
 
   if (['RecordNotFoundError', 'LoadBlockFailedError'].includes(err.name)) {
-    return gatewayTimeoutResponse(resource.toString(), err)
+    return gatewayTimeoutResponse(resource, err)
   }
 
   // can't tell what went wrong, return a generic error
-  return internalServerErrorResponse(resource.toString(), err)
+  return internalServerErrorResponse(resource, err)
 }

package/src/utils/get-range-header.ts

@@ -1,5 +1,6 @@
 import { InvalidRangeError } from '../errors.ts'
 import { badRequestResponse, notSatisfiableResponse } from './responses.ts'
+import type { Resource } from '../index.ts'
 
 /**
  * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Range
@@ -78,7 +79,7 @@ function getByteRangeFromHeader (rangeHeader?: string): Range[] {
   return ranges
 }
 
-export function getRangeHeader (resource: string, headers: Headers): RangeHeader | undefined | Response {
+export function getRangeHeader (resource: Resource, headers: Headers): RangeHeader | undefined | Response {
   const header = headers.get('range')
 
   // not a range request

package/src/utils/get-tar-stream.ts

@@ -1,5 +1,7 @@
 import { NotUnixFSError } from '@helia/unixfs/errors'
-import { exporter, recursive } from 'ipfs-unixfs-exporter'
+import { InvalidParametersError } from '@libp2p/interface'
+import { exporter, recursive, walkPath } from 'ipfs-unixfs-exporter'
+import last from 'it-last'
 import map from 'it-map'
 import { pipe } from 'it-pipe'
 import { pack } from 'it-tar'
@@ -10,9 +12,14 @@ import type { TarEntryHeader, TarImportCandidate } from 'it-tar'
 
 const EXPORTABLE = ['file', 'raw', 'directory']
 
-function toHeader (file: UnixFSEntry): Partial<TarEntryHeader> & { name: string } {
+function toHeader (file: UnixFSEntry, path: string): Partial<TarEntryHeader> & { name: string } {
   let mode: number | undefined
   let mtime: Date | undefined
+  let size = 0n
+
+  if (file.type === 'file' || file.type === 'raw' || file.type === 'identity') {
+    size = file.size
+  }
 
   if (file.type === 'file' || file.type === 'directory') {
     mode = file.unixfs.mode
@@ -20,21 +27,21 @@ function toHeader (file: UnixFSEntry): Partial<TarEntryHeader> & { name: string
   }
 
   return {
-    name: file.path,
+    name: path,
     mode,
     mtime,
-    size: Number(file.size),
+    size: Number(size),
     type: file.type === 'directory' ? 'directory' : 'file'
   }
 }
 
-function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate {
+function toTarImportCandidate (entry: UnixFSEntry, path: string): TarImportCandidate {
   if (!EXPORTABLE.includes(entry.type)) {
     throw new NotUnixFSError(`${entry.type} is not a UnixFS node`)
   }
 
   const candidate: TarImportCandidate = {
-    header: toHeader(entry)
+    header: toHeader(entry, path)
   }
 
   if (entry.type === 'file' || entry.type === 'raw') {
@@ -45,11 +52,17 @@ function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate {
 }
 
 export async function * tarStream (ipfsPath: string, blockstore: Blockstore, options?: AbortOptions): AsyncGenerator<Uint8Array> {
-  const file = await exporter(ipfsPath, blockstore, options)
+  const entry = await last(walkPath(ipfsPath, blockstore, options))
+
+  if (entry == null) {
+    throw new InvalidParametersError(`Could not walk path "${ipfsPath}"`)
+  }
+
+  const file = await exporter(entry.cid, blockstore, options)
 
   if (file.type === 'file' || file.type === 'raw') {
     yield * pipe(
-      [toTarImportCandidate(file)],
+      [toTarImportCandidate(file, entry.path)],
       pack()
     )
 
@@ -58,8 +71,11 @@ export async function * tarStream (ipfsPath: string, blockstore: Blockstore, opt
 
   if (file.type === 'directory') {
     yield * pipe(
-      recursive(ipfsPath, blockstore, options),
-      (source) => map(source, (entry) => toTarImportCandidate(entry)),
+      recursive(file.cid, blockstore, options),
+      (source) => map(source, async (entry) => {
+        const file = await exporter(entry.cid, blockstore, options)
+        return toTarImportCandidate(file, entry.path)
+      }),
       pack()
     )
 

package/src/utils/parse-resource.ts

@@ -0,0 +1,62 @@
+import { InvalidParametersError } from '@libp2p/interface'
+import { peerIdFromString } from '@libp2p/peer-id'
+import { CID } from 'multiformats/cid'
+
+/**
+ * Turns an IPFS or IPNS path into a URL
+ *
+ * - `/ipfs/cid` -> `ipfs://cid`
+ * - `/ipns/name` -> `ipns://name`
+ */
+function ipfsPathToIpfsUrl (path: string): string {
+  if (!path.startsWith('/ipfs/') && !path.startsWith('/ipns/')) {
+    throw new InvalidParametersError(`Path ${path} did not start with /ipfs/ or /ipns/`)
+  }
+
+  // use non-http protocol as otherwise an empty path will become "/"
+  const url = new URL(`not-http://example.com${path}`)
+  const [
+    ,
+    protocol,
+    name,
+    ...rest
+  ] = url.pathname.split('/')
+
+  return `${protocol}://${name}${rest.length > 0 ? `/${rest.join('/')}` : ''}${url.search}${url.hash}`
+}
+
+/**
+ * Accepts the following url strings:
+ *
+ * - /ipfs/Qmfoo/path
+ * - /ipns/Qmfoo/path
+ * - ipfs://cid/path
+ * - ipns://name/path
+ */
+export function stringToIpfsUrl (urlString: string): URL {
+  if (urlString.startsWith('/ipfs/') || urlString.startsWith('/ipns/')) {
+    urlString = ipfsPathToIpfsUrl(urlString)
+  }
+
+  if (urlString.startsWith('ipfs://') || urlString.startsWith('ipns://') || urlString.startsWith('dnslink://')) {
+    const url = new URL(urlString)
+
+    // ensure IPNS name can be parsed as a CID or peer id, otherwise treat as
+    // dnslink
+    if (url.protocol === 'ipns:') {
+      try {
+        CID.parse(url.hostname)
+      } catch {
+        try {
+          peerIdFromString(url.hostname)
+        } catch {
+          url.protocol = 'dnslink'
+        }
+      }
+    }
+
+    return url
+  }
+
+  throw new InvalidParametersError(`URL did not start with ipfs:// or ipns:// - ${urlString}`)
+}