@helia/verified-fetch 4.1.0 → 5.0.0

package/src/plugins/plugin-handle-dag-pb.ts

@@ -1,248 +0,0 @@
-import { unixfs } from '@helia/unixfs'
-import { code as dagPbCode } from '@ipld/dag-pb'
-import { AbortError } from '@libp2p/interface'
-import { exporter } from 'ipfs-unixfs-exporter'
-import { CustomProgressEvent } from 'progress-events'
-import { getContentType } from '../utils/get-content-type.js'
-import { getStreamFromAsyncIterable } from '../utils/get-stream-from-async-iterable.js'
-import { setIpfsRoots } from '../utils/response-headers.js'
-import { badGatewayResponse, badRangeResponse, movedPermanentlyResponse, okRangeResponse } from '../utils/responses.js'
-import { BasePlugin } from './plugin-base.js'
-import type { PluginContext } from './types.js'
-import type { CIDDetail } from '../index.js'
-import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
-
-/**
- * Handles UnixFS and dag-pb content.
- */
-export class DagPbPlugin extends BasePlugin {
-  readonly id = 'dag-pb-plugin'
-  readonly codes = [dagPbCode]
-
-  canHandle ({ cid, accept, pathDetails, byteRangeContext }: PluginContext): boolean {
-    if (pathDetails == null) {
-      return false
-    }
-
-    if (byteRangeContext == null) {
-      return false
-    }
-
-    // TODO: this may be too restrictive?
-    if (accept != null && accept.mimeType !== 'application/octet-stream') {
-      return false
-    }
-
-    return cid.code === dagPbCode
-  }
-
-  /**
-   * @see https://specs.ipfs.tech/http-gateways/path-gateway/#use-in-directory-url-normalization
-   */
-  getRedirectUrl (context: PluginContext): string | null {
-    const { resource, url, isDirectory } = context
-
-    let uri: URL
-
-    try {
-      // try the requested resource
-      uri = new URL(resource)
-    } catch {
-      // fall back to the canonical URL
-      uri = url
-    }
-
-    // directories must be requested with a trailing slash
-    if (isDirectory && !uri.pathname.endsWith('/')) {
-      // make sure we append slash to end of the path
-      uri.pathname += '/'
-      return uri.toString()
-    }
-
-    return null
-  }
-
-  async handle (context: PluginContext & Required<Pick<PluginContext, 'byteRangeContext' | 'pathDetails'>>): Promise<Response | null> {
-    const { cid, options, pathDetails, query } = context
-    const { contentTypeParser, helia, getBlockstore } = this.pluginOptions
-    const log = this.log
-    let resource = context.resource
-    const path = context.path
-    let filename = query?.filename
-
-    let redirected = false
-
-    const byteRangeContext = context.byteRangeContext
-    const ipfsRoots = pathDetails.ipfsRoots
-    const terminalElement = pathDetails.terminalElement
-    let resolvedCID = terminalElement.cid
-    const fs = unixfs({ ...helia, blockstore: getBlockstore(context.cid, context.resource, options?.session ?? true, options) })
-
-    context.isDirectory = terminalElement?.type === 'directory'
-
-    if (terminalElement?.type === 'directory') {
-      const redirectUrl = this.getRedirectUrl(context)
-
-      if (redirectUrl != null) {
-        log.trace('directory url normalization spec requires redirect...')
-        if (options?.redirect === 'error') {
-          log('could not redirect to %s as redirect option was set to "error"', redirectUrl)
-          throw new TypeError('Failed to fetch')
-        } else if (options?.redirect === 'manual') {
-          log('returning 301 permanent redirect to %s', redirectUrl)
-          return movedPermanentlyResponse(resource, redirectUrl)
-        }
-        log('following redirect to %s', redirectUrl)
-
-        // fall-through simulates following the redirect?
-        resource = redirectUrl
-        redirected = true
-      }
-
-      const dirCid = terminalElement.cid
-      const rootFilePath = 'index.html'
-
-      try {
-        log.trace('found directory at %c/%s, looking for index.html', cid, path)
-
-        const entry = await context.serverTiming.time('exporter-dir', '', exporter(`/ipfs/${dirCid}/${rootFilePath}`, helia.blockstore, {
-          signal: options?.signal,
-          onProgress: options?.onProgress
-        }))
-
-        log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, entry.cid)
-        resolvedCID = entry.cid
-
-        // use `index.html` as the file name to help with content types
-        filename = rootFilePath
-      } catch (err: any) {
-        if (options?.signal?.aborted) {
-          throw new AbortError(options?.signal?.reason)
-        }
-
-        this.log.error('error loading path %c/%s - %e', dirCid, rootFilePath, err)
-
-        context.isDirectory = true
-        context.directoryEntries = []
-        context.modified++
-
-        this.log.trace('attempting to get directory entries because index.html was not found')
-        for await (const dirItem of fs.ls(dirCid, { signal: options?.signal, onProgress: options?.onProgress, extended: false })) {
-          context.directoryEntries.push(dirItem)
-        }
-
-        // dir-index-html plugin or dir-index-json (future idea?) plugin should handle this
-        return null
-      } finally {
-        options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: dirCid, path: [rootFilePath] }))
-      }
-    }
-
-    try {
-      // attempt to get the exact file size, but timeout quickly.
-      const stat = await fs.stat(resolvedCID, {
-        extended: true,
-        signal: AbortSignal.timeout(500)
-      })
-
-      byteRangeContext.setFileSize(stat.size)
-    } catch (err: any) {
-      log.error('error getting exact file size for %c/%s - %e', cid, path, err)
-      byteRangeContext.setFileSize(pathDetails.terminalElement.size)
-      log.trace('using terminal element size of %d for %c/%s', pathDetails.terminalElement.size, cid, path)
-    }
-
-    try {
-      const entry = await context.serverTiming.time('exporter-file', '', exporter(resolvedCID, helia.blockstore, {
-        signal: options?.signal,
-        onProgress: options?.onProgress
-      }))
-
-      let firstChunk: Uint8Array
-      let contentType: string
-      if (byteRangeContext.isValidRangeRequest) {
-        contentType = await this.handleRangeRequest(context, entry)
-      } else {
-        const asyncIter = entry.content({
-          signal: options?.signal,
-          onProgress: options?.onProgress
-        })
-        log('got async iterator for %c/%s', cid, path)
-
-        const streamAndFirstChunk = await context.serverTiming.time('stream-and-chunk', '', getStreamFromAsyncIterable(asyncIter, {
-          onProgress: options?.onProgress,
-          signal: options?.signal
-        }))
-        const stream = streamAndFirstChunk.stream
-        firstChunk = streamAndFirstChunk.firstChunk
-        contentType = await context.serverTiming.time('get-content-type', '', getContentType({
-          path,
-          filename,
-          bytes: firstChunk,
-          contentTypeParser,
-          log
-        }))
-
-        byteRangeContext.setBody(stream)
-      }
-
-      // if not a valid range request, okRangeRequest will call okResponse
-      const response = okRangeResponse(resource, byteRangeContext.getBody(contentType), { byteRangeContext, log }, {
-        redirected
-      })
-
-      response.headers.set('Content-Type', byteRangeContext.getContentType() ?? contentType)
-
-      setIpfsRoots(response, ipfsRoots)
-
-      return response
-    } catch (err: any) {
-      if (options?.signal?.aborted) {
-        throw new AbortError(options?.signal?.reason)
-      }
-
-      log.error('error streaming %c/%s - %e', cid, path, err)
-
-      if (byteRangeContext.isRangeRequest && err.code === 'ERR_INVALID_PARAMS') {
-        return badRangeResponse(resource)
-      }
-
-      return badGatewayResponse(resource, 'Unable to stream content')
-    }
-  }
-
-  private async handleRangeRequest (context: PluginContext & Required<Pick<PluginContext, 'byteRangeContext' | 'pathDetails'>>, entry: UnixFSEntry): Promise<string> {
-    const { path, byteRangeContext, options } = context
-    const { contentTypeParser } = this.pluginOptions
-    const log = this.log
-
-    // get the first chunk in order to determine the content type
-    const asyncIter = entry.content({
-      signal: options?.signal,
-      onProgress: options?.onProgress,
-      offset: 0,
-      // 8kb in order to determine the content type
-      length: 8192
-    })
-
-    const { firstChunk } = await getStreamFromAsyncIterable(asyncIter, {
-      onProgress: options?.onProgress,
-      signal: options?.signal
-    })
-    const contentType = await context.serverTiming.time('get-content-type', '', getContentType({ bytes: firstChunk, path, contentTypeParser, log }))
-
-    byteRangeContext?.setBody((range): AsyncGenerator<Uint8Array, void, unknown> => {
-      if (options?.signal?.aborted) {
-        throw new AbortError(options?.signal?.reason ?? 'aborted while streaming')
-      }
-      return entry.content({
-        signal: options?.signal,
-        onProgress: options?.onProgress,
-        offset: range.start ?? 0,
-        length: byteRangeContext.getLength(range)
-      })
-    }, contentType)
-
-    return contentType
-  }
-}

package/src/plugins/plugin-handle-dag-walk.ts

@@ -1,110 +0,0 @@
-import * as dagCbor from '@ipld/dag-cbor'
-import * as dagJson from '@ipld/dag-json'
-import * as dagPb from '@ipld/dag-pb'
-import toBuffer from 'it-to-buffer'
-import * as json from 'multiformats/codecs/json'
-import * as raw from 'multiformats/codecs/raw'
-import { CODEC_CBOR, CODEC_IDENTITY } from '../constants.ts'
-import { handlePathWalking } from '../utils/walk-path.js'
-import { BasePlugin } from './plugin-base.js'
-import type { PluginContext } from './types.js'
-import type { PathWalkerResponse } from '../utils/walk-path.js'
-
-const ENTITY_CODECS = [
-  CODEC_CBOR,
-  json.code,
-  raw.code
-]
-
-const WALKABLE_CODECS = [
-  dagPb.code,
-  dagCbor.code,
-  dagJson.code,
-  ...ENTITY_CODECS
-]
-
-/**
- * This plugin should almost always run first because it's going to handle path
- * walking if needed, and will only say it can handle the request if path
- * walking is possible (path is not empty, terminalCid is unknown, and the path
- * has not been walked yet).
- *
- * Once this plugin has run, the PluginContext will be updated and then this
- * plugin will return false for canHandle, so it won't run again.
- */
-export class DagWalkPlugin extends BasePlugin {
-  readonly id = 'dag-walk-plugin'
-
-  /**
-   * Return false if the path has already been walked, otherwise return true if
-   * the CID is encoded with a codec that supports pathing.
-   */
-  canHandle (context: PluginContext): boolean {
-    const { pathDetails, cid } = context
-
-    if (pathDetails != null) {
-      // path has already been walked
-      return false
-    }
-
-    return WALKABLE_CODECS.includes(cid.code)
-  }
-
-  async handle (context: PluginContext): Promise<Response | null> {
-    const { cid, resource, options } = context
-    const { getBlockstore } = this.pluginOptions
-    const blockstore = getBlockstore(cid, resource, options?.session ?? true, options)
-
-    let pathDetails: PathWalkerResponse | Response
-
-    // entity codecs contain all the bytes for an entity in one block and no
-    // path walking outside of that block is possible
-    if (ENTITY_CODECS.includes(cid.code)) {
-      let bytes: Uint8Array
-
-      if (cid.multihash.code === CODEC_IDENTITY) {
-        bytes = cid.multihash.digest
-      } else {
-        bytes = await toBuffer(blockstore.get(cid, context.options))
-      }
-
-      pathDetails = {
-        ipfsRoots: [cid],
-        terminalElement: {
-          name: cid.toString(),
-          path: cid.toString(),
-          depth: 0,
-          type: 'object',
-          node: bytes,
-          cid,
-          size: BigInt(bytes.byteLength),
-          content: async function * () {
-            yield bytes
-          }
-        }
-      }
-    } else {
-      // TODO: migrate handlePathWalking into this plugin
-      pathDetails = await context.serverTiming.time('path-walking', '', handlePathWalking({ ...context, blockstore, log: this.log }))
-    }
-
-    if (pathDetails instanceof Response) {
-      this.log.trace('path walking failed')
-
-      if (pathDetails.status === 404) {
-        // invalid or incorrect path.. we walked the path but nothing is there
-        // send the 404 response
-        return pathDetails
-      }
-
-      // some other error walking the path (codec doesn't support pathing,
-      // etc..), let the next plugin try to handle it
-      return null
-    }
-
-    context.modified++
-    context.pathDetails = pathDetails
-
-    return null
-  }
-}

package/src/plugins/plugin-handle-dir-index-html.ts

@@ -1,72 +0,0 @@
-import { code as dagPbCode } from '@ipld/dag-pb'
-import { base32 } from 'multiformats/bases/base32'
-import { sha256 } from 'multiformats/hashes/sha2'
-import { dirIndexHtml } from '../utils/dir-index-html.js'
-import { getETag } from '../utils/get-e-tag.js'
-import { getIpfsRoots } from '../utils/response-headers.js'
-import { okRangeResponse } from '../utils/responses.js'
-import { BasePlugin } from './plugin-base.js'
-import type { PluginContext, VerifiedFetchPluginFactory } from './types.js'
-import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
-
-/**
- * Converts a list of directory entries into a small hash that can be used in the etag header.
- *
- * @see https://github.com/ipfs/boxo/blob/dc60fe747c375c631a92fcfd6c7456f44a760d24/gateway/assets/assets.go#L84
- * @see https://github.com/ipfs/boxo/blob/dc60fe747c375c631a92fcfd6c7456f44a760d24/gateway/handler_unixfs_dir.go#L233-L235
- */
-async function getAssetHash (directoryEntries: UnixFSEntry[]): Promise<string> {
-  const entryDetails = directoryEntries.reduce((acc, entry) => {
-    return `${acc}${entry.name}${entry.cid.toString()}`
-  }, '')
-  const hashBytes = await sha256.encode(new TextEncoder().encode(entryDetails))
-  return base32.encode(hashBytes)
-}
-
-export class DirIndexHtmlPlugin extends BasePlugin {
-  readonly id = 'dir-index-html-plugin'
-  readonly codes = [dagPbCode]
-  canHandle (context: PluginContext): boolean {
-    const { cid, pathDetails, directoryEntries } = context
-    if (pathDetails == null) {
-      return false
-    }
-    if (pathDetails.terminalElement?.type !== 'directory') {
-      return false
-    }
-
-    if (directoryEntries == null || directoryEntries.length === 0) {
-      return false
-    }
-
-    return cid.code === dagPbCode
-  }
-
-  async handle (context: PluginContext & Required<Pick<PluginContext, 'byteRangeContext' | 'pathDetails' | 'directoryEntries'>>): Promise<Response> {
-    const { resource, pathDetails, directoryEntries } = context
-
-    const { terminalElement, ipfsRoots } = pathDetails
-
-    const gatewayURL = resource
-    const htmlResponse = dirIndexHtml(terminalElement, directoryEntries, { gatewayURL, log: this.log })
-
-    context.byteRangeContext.setBody(htmlResponse)
-
-    const etagPrefix = `DirIndex-${await getAssetHash(directoryEntries)}_CID-`
-
-    const response = okRangeResponse(resource, context.byteRangeContext.getBody('text/html'), { byteRangeContext: context.byteRangeContext, log: this.log }, {
-      headers: {
-        'Content-Type': context.byteRangeContext.getContentType() ?? 'text/html',
-        // see https://github.com/ipfs/gateway-conformance/pull/219
-        'Cache-Control': 'public, max-age=604800, stale-while-revalidate=2678400',
-        'X-Ipfs-Roots': getIpfsRoots(ipfsRoots),
-        // e.g. DirIndex-<asset_hash>_CID-<cid>
-        Etag: getETag({ cid: terminalElement.cid, reqFormat: context.reqFormat, contentPrefix: etagPrefix })
-      }
-    })
-
-    return response
-  }
-}
-
-export const dirIndexHtmlPluginFactory: VerifiedFetchPluginFactory = (opts) => new DirIndexHtmlPlugin(opts)

package/src/plugins/plugin-handle-json.ts

@@ -1,80 +0,0 @@
-import * as ipldDagCbor from '@ipld/dag-cbor'
-import * as ipldDagJson from '@ipld/dag-json'
-import toBuffer from 'it-to-buffer'
-import { code as jsonCode } from 'multiformats/codecs/json'
-import { CODEC_CBOR } from '../constants.ts'
-import { notAcceptableResponse, okRangeResponse } from '../utils/responses.js'
-import { BasePlugin } from './plugin-base.js'
-import type { PluginContext } from './types.js'
-
-/**
- * Handles `dag-json` content, including requests with Accept: `application/vnd.ipld.dag-cbor` and `application/cbor`.
- */
-export class JsonPlugin extends BasePlugin {
-  readonly id = 'json-plugin'
-  readonly codes = [ipldDagJson.code, jsonCode]
-
-  canHandle ({ cid, accept, byteRangeContext }: PluginContext): boolean {
-    if (byteRangeContext == null) {
-      return false
-    }
-
-    if (accept?.mimeType === 'application/vnd.ipld.dag-json' && cid.code !== ipldDagCbor.code && cid.code !== CODEC_CBOR) {
-      // we can handle application/vnd.ipld.dag-json, but if the CID codec is
-      // cbor related, cbor related plugins will handle it
-      // TODO: remove the need for deny-listing cases in plugins
-      return true
-    }
-
-    return ipldDagJson.code === cid.code || jsonCode === cid.code
-  }
-
-  async handle (context: PluginContext & Required<Pick<PluginContext, 'byteRangeContext'>>): Promise<Response> {
-    const { path, resource, cid, accept, options } = context
-    const { getBlockstore } = this.pluginOptions
-    const session = options?.session ?? true
-
-    this.log.trace('fetching %c/%s', cid, path)
-
-    const terminalCid = context.pathDetails?.terminalElement.cid ?? context.cid
-    const blockstore = getBlockstore(terminalCid, resource, session, options)
-    const block = await toBuffer(blockstore.get(terminalCid, options))
-    let body: string | Uint8Array
-
-    if (accept?.mimeType === 'application/vnd.ipld.dag-cbor' || accept?.mimeType === 'application/cbor') {
-      try {
-        // if vnd.ipld.dag-cbor has been specified, convert to the format - note
-        // that this supports more data types than regular JSON, the content-type
-        // response header is set so the user knows to process it differently
-        const obj = ipldDagJson.decode(block)
-        body = ipldDagCbor.encode(obj)
-      } catch (err) {
-        this.log.error('could not transform %c to application/vnd.ipld.dag-cbor - %e', err)
-        return notAcceptableResponse(resource)
-      }
-    } else {
-      // skip decoding
-      body = block
-    }
-
-    let contentType: string
-    if (accept == null) {
-      if (ipldDagJson.code === cid.code) {
-        contentType = 'application/vnd.ipld.dag-json'
-      } else {
-        contentType = 'application/json'
-      }
-    } else {
-      contentType = accept?.mimeType.split(';')[0]
-    }
-
-    context.byteRangeContext.setBody(body)
-
-    const response = okRangeResponse(resource, context.byteRangeContext.getBody(contentType), { byteRangeContext: context.byteRangeContext, log: this.log })
-    response.headers.set('content-type', context.byteRangeContext.getContentType() ?? contentType)
-    if (!context.byteRangeContext.isValidRangeRequest) {
-      response.headers.set('content-length', body.length.toString())
-    }
-    return response
-  }
-}

package/src/plugins/plugin-handle-raw.ts

@@ -1,110 +0,0 @@
-import toBuffer from 'it-to-buffer'
-import { code as rawCode } from 'multiformats/codecs/raw'
-import { identity } from 'multiformats/hashes/identity'
-import { getContentType } from '../utils/get-content-type.js'
-import { notFoundResponse, okRangeResponse } from '../utils/responses.js'
-import { BasePlugin } from './plugin-base.js'
-import type { PluginContext } from './types.js'
-import type { AcceptHeader } from '../utils/select-output-type.ts'
-
-/**
- * These are Accept header values that will cause content type sniffing to be
- * skipped and set to these values.
- */
-const RAW_HEADERS = [
-  'application/vnd.ipld.dag-json',
-  'application/vnd.ipld.raw',
-  'application/octet-stream'
-]
-
-/**
- * if the user has specified an `Accept` header, and it's in our list of
- * allowable "raw" format headers, use that instead of detecting the content
- * type. This avoids the user from receiving something different when they
- * signal that they want to `Accept` a specific mime type.
- */
-function getOverriddenRawContentType ({ headers, accept }: { headers?: HeadersInit, accept?: AcceptHeader }): string | undefined {
-  // accept has already been resolved by getResolvedAcceptHeader, if we have it, use it.
-  const acceptHeader = accept?.mimeType ?? new Headers(headers).get('accept') ?? ''
-
-  // e.g. "Accept: text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8"
-  const acceptHeaders = acceptHeader.split(',')
-    .map(s => s.split(';')[0])
-    .map(s => s.trim())
-
-  for (const mimeType of acceptHeaders) {
-    if (mimeType === '*/*') {
-      return
-    }
-
-    if (RAW_HEADERS.includes(mimeType ?? '')) {
-      return mimeType
-    }
-  }
-}
-
-export class RawPlugin extends BasePlugin {
-  readonly id = 'raw-plugin'
-  codes: number[] = [rawCode, identity.code]
-
-  canHandle ({ cid, accept, query, byteRangeContext }: PluginContext): boolean {
-    if (byteRangeContext == null) {
-      return false
-    }
-
-    return accept?.mimeType === 'application/vnd.ipld.raw' || query.format === 'raw'
-  }
-
-  async handle (context: PluginContext & Required<Pick<PluginContext, 'byteRangeContext'>>): Promise<Response> {
-    const { path, resource, cid, accept, query, options } = context
-    const { getBlockstore, contentTypeParser } = this.pluginOptions
-    const session = options?.session ?? true
-    const log = this.log
-
-    if (accept?.mimeType === 'application/vnd.ipld.raw' || query.format === 'raw') {
-      context.reqFormat = 'raw'
-      context.query.download = true
-      context.query.filename = context.query.filename ?? `${cid.toString()}.bin`
-      log.trace('set content disposition to force download')
-    } else {
-      log.trace('did not set content disposition, raw block will display inline')
-    }
-
-    if (path.length > 0 && cid.code === rawCode) {
-      log.trace('404-ing raw codec request for %c/%s', cid, path)
-      return notFoundResponse(resource)
-    }
-
-    const terminalCid = context.pathDetails?.terminalElement.cid ?? context.cid
-    const blockstore = getBlockstore(terminalCid, resource, session, options)
-    const result = await toBuffer(blockstore.get(terminalCid, options))
-
-    context.byteRangeContext.setBody(result)
-
-    // if the user has specified an `Accept` header that corresponds to a raw
-    // type, honour that header, so for example they don't request
-    // `application/vnd.ipld.raw` but get `application/octet-stream`
-    const contentType = await getContentType({
-      filename: query.filename,
-      bytes: result,
-      path,
-      defaultContentType: getOverriddenRawContentType({ headers: options?.headers, accept }),
-      contentTypeParser,
-      log
-    })
-
-    const response = okRangeResponse(resource, context.byteRangeContext.getBody(contentType), { byteRangeContext: context.byteRangeContext, log }, {
-      redirected: false
-    })
-
-    response.headers.set('content-type', context.byteRangeContext.getContentType() ?? contentType)
-    response.headers.set('x-ipfs-roots', terminalCid.toV1().toString())
-
-    // only set content-length if it is not a range request
-    if (!context.byteRangeContext.isRangeRequest) {
-      response.headers.set('content-length', result.byteLength.toString())
-    }
-
-    return response
-  }
-}

package/src/plugins/plugins.ts

@@ -1,5 +0,0 @@
-/**
- * Export extension (non-default) plugins here
- */
-export { DirIndexHtmlPlugin, dirIndexHtmlPluginFactory } from './plugin-handle-dir-index-html.js'
-export { DagCborHtmlPreviewPlugin, dagCborHtmlPreviewPluginFactory } from './plugin-handle-dag-cbor-html-preview.js'