@helia/verified-fetch 0.0.0-28d62f7

package/src/utils/get-e-tag.ts

@@ -0,0 +1,36 @@
+import type { RequestFormatShorthand } from '../types.js'
+import type { CID } from 'multiformats/cid'
+
+interface GetETagArg {
+  cid: CID
+  reqFormat?: RequestFormatShorthand
+  rangeStart?: number
+  rangeEnd?: number
+  /**
+   * Weak Etag is used when we can't guarantee byte-for-byte-determinism (generated, or mutable content).
+   * Some examples:
+   * - IPNS requests
+   * - CAR streamed with blocks in non-deterministic order
+   * - TAR streamed with files in non-deterministic order
+   */
+  weak?: boolean
+}
+
+/**
+ * etag
+ * you need to wrap cid  with ""
+ * we use strong Etags for immutable responses and weak one (prefixed with W/ ) for mutable/generated ones (ipns and generated HTML).
+ * block and car responses should have different etag than deserialized one, so you can add some prefix like we do in existing gateway
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
+ * @see https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
+ */
+export function getETag ({ cid, reqFormat, weak, rangeStart, rangeEnd }: GetETagArg): string {
+  const prefix = weak === true ? 'W/' : ''
+  let suffix = reqFormat == null ? '' : `.${reqFormat}`
+  if (rangeStart != null || rangeEnd != null) {
+    suffix += `.${rangeStart ?? '0'}-${rangeEnd ?? 'N'}`
+  }
+
+  return `${prefix}"${cid.toString()}${suffix}"`
+}

package/src/utils/get-stream-from-async-iterable.ts

@@ -0,0 +1,45 @@
+import { CustomProgressEvent } from 'progress-events'
+import type { VerifiedFetchInit } from '../index.js'
+import type { ComponentLogger } from '@libp2p/interface'
+
+/**
+ * Converts an async iterator of Uint8Array bytes to a stream and returns the first chunk of bytes
+ */
+export async function getStreamFromAsyncIterable (iterator: AsyncIterable<Uint8Array>, path: string, logger: ComponentLogger, options?: Pick<VerifiedFetchInit, 'onProgress'>): Promise<{ stream: ReadableStream<Uint8Array>, firstChunk: Uint8Array }> {
+  const log = logger.forComponent('helia:verified-fetch:get-stream-from-async-iterable')
+  const reader = iterator[Symbol.asyncIterator]()
+  const { value: firstChunk, done } = await reader.next()
+
+  if (done === true) {
+    log.error('No content found for path', path)
+    throw new Error('No content found')
+  }
+
+  const stream = new ReadableStream({
+    async start (controller) {
+      // the initial value is already available
+      options?.onProgress?.(new CustomProgressEvent<void>('verified-fetch:request:progress:chunk'))
+      controller.enqueue(firstChunk)
+    },
+    async pull (controller) {
+      const { value, done } = await reader.next()
+
+      if (done === true) {
+        if (value != null) {
+          options?.onProgress?.(new CustomProgressEvent<void>('verified-fetch:request:progress:chunk'))
+          controller.enqueue(value)
+        }
+        controller.close()
+        return
+      }
+
+      options?.onProgress?.(new CustomProgressEvent<void>('verified-fetch:request:progress:chunk'))
+      controller.enqueue(value)
+    }
+  })
+
+  return {
+    stream,
+    firstChunk
+  }
+}

package/src/utils/get-tar-stream.ts

@@ -0,0 +1,68 @@
+import { CodeError } from '@libp2p/interface'
+import { exporter, recursive, type UnixFSEntry } from 'ipfs-unixfs-exporter'
+import map from 'it-map'
+import { pipe } from 'it-pipe'
+import { pack, type TarEntryHeader, type TarImportCandidate } from 'it-tar'
+import type { AbortOptions } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+
+const EXPORTABLE = ['file', 'raw', 'directory']
+
+function toHeader (file: UnixFSEntry): Partial<TarEntryHeader> & { name: string } {
+  let mode: number | undefined
+  let mtime: Date | undefined
+
+  if (file.type === 'file' || file.type === 'directory') {
+    mode = file.unixfs.mode
+    mtime = file.unixfs.mtime != null ? new Date(Number(file.unixfs.mtime.secs * 1000n)) : undefined
+  }
+
+  return {
+    name: file.path,
+    mode,
+    mtime,
+    size: Number(file.size),
+    type: file.type === 'directory' ? 'directory' : 'file'
+  }
+}
+
+function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate {
+  if (!EXPORTABLE.includes(entry.type)) {
+    throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS')
+  }
+
+  const candidate: TarImportCandidate = {
+    header: toHeader(entry)
+  }
+
+  if (entry.type === 'file' || entry.type === 'raw') {
+    candidate.body = entry.content()
+  }
+
+  return candidate
+}
+
+export async function * tarStream (ipfsPath: string, blockstore: Blockstore, options?: AbortOptions): AsyncGenerator<Uint8Array> {
+  const file = await exporter(ipfsPath, blockstore, options)
+
+  if (file.type === 'file' || file.type === 'raw') {
+    yield * pipe(
+      [toTarImportCandidate(file)],
+      pack()
+    )
+
+    return
+  }
+
+  if (file.type === 'directory') {
+    yield * pipe(
+      recursive(ipfsPath, blockstore, options),
+      (source) => map(source, (entry) => toTarImportCandidate(entry)),
+      pack()
+    )
+
+    return
+  }
+
+  throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS')
+}

package/src/utils/parse-resource.ts

@@ -0,0 +1,40 @@
+import { CID } from 'multiformats/cid'
+import { parseUrlString } from './parse-url-string.js'
+import type { ParsedUrlStringResults } from './parse-url-string.js'
+import type { Resource } from '../index.js'
+import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns'
+import type { ComponentLogger } from '@libp2p/interface'
+import type { ProgressOptions } from 'progress-events'
+
+export interface ParseResourceComponents {
+  ipns: IPNS
+  logger: ComponentLogger
+}
+
+export interface ParseResourceOptions extends ProgressOptions<ResolveProgressEvents | IPNSRoutingEvents | ResolveDnsLinkProgressEvents> {
+
+}
+/**
+ * Handles the different use cases for the `resource` argument.
+ * The resource can represent an IPFS path, IPNS path, or CID.
+ * If the resource represents an IPNS path, we need to resolve it to a CID.
+ */
+export async function parseResource (resource: Resource, { ipns, logger }: ParseResourceComponents, options?: ParseResourceOptions): Promise<ParsedUrlStringResults> {
+  if (typeof resource === 'string') {
+    return parseUrlString({ urlString: resource, ipns, logger }, { onProgress: options?.onProgress })
+  }
+
+  const cid = CID.asCID(resource)
+
+  if (cid != null) {
+    // an actual CID
+    return {
+      cid,
+      protocol: 'ipfs',
+      path: '',
+      query: {}
+    }
+  }
+
+  throw new TypeError(`Invalid resource. Cannot determine CID from resource: ${resource}`)
+}

package/src/utils/parse-url-string.ts

@@ -0,0 +1,154 @@
+import { peerIdFromString } from '@libp2p/peer-id'
+import { CID } from 'multiformats/cid'
+import { TLRU } from './tlru.js'
+import type { RequestFormatShorthand } from '../types.js'
+import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
+import type { ComponentLogger } from '@libp2p/interface'
+import type { ProgressOptions } from 'progress-events'
+
+const ipnsCache = new TLRU<ResolveResult>(1000)
+
+export interface ParseUrlStringInput {
+  urlString: string
+  ipns: IPNS
+  logger: ComponentLogger
+}
+export interface ParseUrlStringOptions extends ProgressOptions<ResolveProgressEvents | IPNSRoutingEvents | ResolveDnsLinkProgressEvents> {
+
+}
+
+export interface ParsedUrlQuery extends Record<string, string | unknown> {
+  format?: RequestFormatShorthand
+  download?: boolean
+  filename?: string
+}
+
+export interface ParsedUrlStringResults {
+  protocol: string
+  path: string
+  cid: CID
+  query: ParsedUrlQuery
+}
+
+const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/$?]+)\/?(?<path>[^$?]*)\??(?<queryString>.*)$/
+
+/**
+ * A function that parses ipfs:// and ipns:// URLs, returning an object with easily recognizable properties.
+ *
+ * After determining the protocol successfully, we process the cidOrPeerIdOrDnsLink:
+ * * If it's ipfs, it parses the CID or throws an Aggregate error
+ * * If it's ipns, it attempts to resolve the PeerId and then the DNSLink. If both fail, an Aggregate error is thrown.
+ */
+export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStringInput, options?: ParseUrlStringOptions): Promise<ParsedUrlStringResults> {
+  const log = logger.forComponent('helia:verified-fetch:parse-url-string')
+  const match = urlString.match(URL_REGEX)
+
+  if (match == null || match.groups == null) {
+    throw new TypeError(`Invalid URL: ${urlString}, please use ipfs:// or ipns:// URLs only.`)
+  }
+
+  const { protocol, cidOrPeerIdOrDnsLink, path: urlPath, queryString } = match.groups
+
+  let cid: CID | undefined
+  let resolvedPath: string | undefined
+  const errors: Error[] = []
+
+  if (protocol === 'ipfs') {
+    try {
+      cid = CID.parse(cidOrPeerIdOrDnsLink)
+    } catch (err) {
+      log.error(err)
+      errors.push(new TypeError('Invalid CID for ipfs://<cid> URL'))
+    }
+  } else {
+    let resolveResult = ipnsCache.get(cidOrPeerIdOrDnsLink)
+
+    if (resolveResult != null) {
+      cid = resolveResult.cid
+      resolvedPath = resolveResult.path
+      log.trace('resolved %s to %c from cache', cidOrPeerIdOrDnsLink, cid)
+    } else {
+      // protocol is ipns
+      log.trace('Attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink)
+      let peerId = null
+
+      try {
+        peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
+        resolveResult = await ipns.resolve(peerId, { onProgress: options?.onProgress })
+        cid = resolveResult?.cid
+        resolvedPath = resolveResult?.path
+        log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid)
+        ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
+      } catch (err) {
+        if (peerId == null) {
+          log.error('Could not parse PeerId string "%s"', cidOrPeerIdOrDnsLink, err)
+          errors.push(new TypeError(`Could not parse PeerId in ipns url "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`))
+        } else {
+          log.error('Could not resolve PeerId %c', peerId, err)
+          errors.push(new TypeError(`Could not resolve PeerId "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`))
+        }
+      }
+
+      if (cid == null) {
+        log.trace('Attempting to resolve DNSLink for %s', cidOrPeerIdOrDnsLink)
+
+        try {
+          resolveResult = await ipns.resolveDns(cidOrPeerIdOrDnsLink, { onProgress: options?.onProgress })
+          cid = resolveResult?.cid
+          resolvedPath = resolveResult?.path
+          log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid)
+          ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
+        } catch (err) {
+          log.error('Could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err)
+          errors.push(err as Error)
+        }
+      }
+    }
+  }
+
+  if (cid == null) {
+    throw new AggregateError(errors, `Invalid resource. Cannot determine CID from URL "${urlString}"`)
+  }
+
+  // parse query string
+  const query: Record<string, any> = {}
+
+  if (queryString != null && queryString.length > 0) {
+    const queryParts = queryString.split('&')
+    for (const part of queryParts) {
+      const [key, value] = part.split('=')
+      query[key] = decodeURIComponent(value)
+    }
+
+    if (query.download != null) {
+      query.download = query.download === 'true'
+    }
+
+    if (query.filename != null) {
+      query.filename = query.filename.toString()
+    }
+  }
+
+  /**
+   * join the path from resolve result & given path.
+   * e.g. /ipns/<peerId>/ that is resolved to /ipfs/<cid>/<path1>, when requested as /ipns/<peerId>/<path2>, should be
+   * resolved to /ipfs/<cid>/<path1>/<path2>
+   */
+  const pathParts = []
+
+  if (urlPath.length > 0) {
+    pathParts.push(urlPath)
+  }
+
+  if (resolvedPath != null && resolvedPath.length > 0) {
+    pathParts.push(resolvedPath)
+  }
+  const path = pathParts.join('/')
+
+  return {
+    protocol,
+    cid,
+    path,
+    query
+  }
+}

package/src/utils/responses.ts

@@ -0,0 +1,29 @@
+export function okResponse (body?: BodyInit | null): Response {
+  return new Response(body, {
+    status: 200,
+    statusText: 'OK'
+  })
+}
+
+export function notSupportedResponse (body?: BodyInit | null): Response {
+  const response = new Response(body, {
+    status: 501,
+    statusText: 'Not Implemented'
+  })
+  response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
+  return response
+}
+
+export function notAcceptableResponse (body?: BodyInit | null): Response {
+  return new Response(body, {
+    status: 406,
+    statusText: 'Not Acceptable'
+  })
+}
+
+export function badRequestResponse (body?: BodyInit | null): Response {
+  return new Response(body, {
+    status: 400,
+    statusText: 'Bad Request'
+  })
+}

package/src/utils/select-output-type.ts

@@ -0,0 +1,167 @@
+import { code as dagCborCode } from '@ipld/dag-cbor'
+import { code as dagJsonCode } from '@ipld/dag-json'
+import { code as dagPbCode } from '@ipld/dag-pb'
+import { code as jsonCode } from 'multiformats/codecs/json'
+import { code as rawCode } from 'multiformats/codecs/raw'
+import type { RequestFormatShorthand } from '../types.js'
+import type { CID } from 'multiformats/cid'
+
+/**
+ * This maps supported response types for each codec supported by verified-fetch
+ */
+const CID_TYPE_MAP: Record<number, string[]> = {
+  [dagCborCode]: [
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car'
+  ],
+  [dagJsonCode]: [
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car'
+  ],
+  [jsonCode]: [
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car'
+  ],
+  [dagPbCode]: [
+    'application/octet-stream',
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car',
+    'application/x-tar'
+  ],
+  [rawCode]: [
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car',
+    'application/x-tar'
+  ]
+}
+
+/**
+ * Selects an output mime-type based on the CID and a passed `Accept` header
+ */
+export function selectOutputType (cid: CID, accept?: string): string | undefined {
+  const cidMimeTypes = CID_TYPE_MAP[cid.code]
+
+  if (accept != null) {
+    return chooseMimeType(accept, cidMimeTypes)
+  }
+}
+
+function chooseMimeType (accept: string, validMimeTypes: string[]): string | undefined {
+  const requestedMimeTypes = accept
+    .split(',')
+    .map(s => {
+      const parts = s.trim().split(';')
+
+      return {
+        mimeType: `${parts[0]}`.trim(),
+        weight: parseQFactor(parts[1])
+      }
+    })
+    .sort((a, b) => {
+      if (a.weight === b.weight) {
+        return 0
+      }
+
+      if (a.weight > b.weight) {
+        return -1
+      }
+
+      return 1
+    })
+    .map(s => s.mimeType)
+
+  for (const headerFormat of requestedMimeTypes) {
+    for (const mimeType of validMimeTypes) {
+      if (headerFormat.includes(mimeType)) {
+        return mimeType
+      }
+
+      if (headerFormat === '*/*') {
+        return mimeType
+      }
+
+      if (headerFormat.startsWith('*/') && mimeType.split('/')[1] === headerFormat.split('/')[1]) {
+        return mimeType
+      }
+
+      if (headerFormat.endsWith('/*') && mimeType.split('/')[0] === headerFormat.split('/')[0]) {
+        return mimeType
+      }
+    }
+  }
+}
+
+/**
+ * Parses q-factor weighting from the accept header to allow letting some mime
+ * types take precedence over others.
+ *
+ * If the q-factor for an acceptable mime representation is omitted it defaults
+ * to `1`.
+ *
+ * All specified values should be in the range 0-1.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept#q
+ */
+function parseQFactor (str?: string): number {
+  if (str != null) {
+    str = str.trim()
+  }
+
+  if (str == null || !str.startsWith('q=')) {
+    return 1
+  }
+
+  const factor = parseFloat(str.replace('q=', ''))
+
+  if (isNaN(factor)) {
+    return 0
+  }
+
+  return factor
+}
+
+const FORMAT_TO_MIME_TYPE: Record<RequestFormatShorthand, string> = {
+  raw: 'application/vnd.ipld.raw',
+  car: 'application/vnd.ipld.car',
+  'dag-json': 'application/vnd.ipld.dag-json',
+  'dag-cbor': 'application/vnd.ipld.dag-cbor',
+  json: 'application/json',
+  cbor: 'application/cbor',
+  'ipns-record': 'application/vnd.ipfs.ipns-record',
+  tar: 'application/x-tar'
+}
+
+/**
+ * Converts a `format=...` query param to a mime type as would be found in the
+ * `Accept` header, if a valid mapping is available
+ */
+export function queryFormatToAcceptHeader (format?: RequestFormatShorthand): string | undefined {
+  if (format != null) {
+    return FORMAT_TO_MIME_TYPE[format]
+  }
+}

package/src/utils/tlru.ts

@@ -0,0 +1,52 @@
+import hashlru from 'hashlru'
+
+/**
+ * Time Aware Least Recent Used Cache
+ *
+ * @see https://arxiv.org/pdf/1801.00390
+ */
+export class TLRU<T> {
+  private readonly lru: ReturnType<typeof hashlru>
+
+  constructor (maxSize: number) {
+    this.lru = hashlru(maxSize)
+  }
+
+  get (key: string): T | undefined {
+    const value = this.lru.get(key)
+
+    if (value != null) {
+      if (value.expire != null && value.expire < Date.now()) {
+        this.lru.remove(key)
+
+        return undefined
+      }
+
+      return value.value
+    }
+
+    return undefined
+  }
+
+  set (key: string, value: T, ttl: number): void {
+    this.lru.set(key, { value, expire: Date.now() + ttl })
+  }
+
+  has (key: string): boolean {
+    const value = this.get(key)
+
+    if (value != null) {
+      return true
+    }
+
+    return false
+  }
+
+  remove (key: string): void {
+    this.lru.remove(key)
+  }
+
+  clear (): void {
+    this.lru.clear()
+  }
+}

package/src/utils/walk-path.ts

@@ -0,0 +1,34 @@
+import { walkPath as exporterWalk, type ExporterOptions, type ReadableStorage, type UnixFSEntry } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+
+export interface PathWalkerOptions extends ExporterOptions {
+
+}
+export interface PathWalkerResponse {
+  ipfsRoots: CID[]
+  terminalElement: UnixFSEntry
+
+}
+
+export interface PathWalkerFn {
+  (blockstore: ReadableStorage, path: string, options?: PathWalkerOptions): Promise<PathWalkerResponse>
+}
+
+export async function walkPath (blockstore: ReadableStorage, path: string, options?: PathWalkerOptions): Promise<PathWalkerResponse> {
+  const ipfsRoots: CID[] = []
+  let terminalElement: UnixFSEntry | undefined
+
+  for await (const entry of exporterWalk(path, blockstore, options)) {
+    ipfsRoots.push(entry.cid)
+    terminalElement = entry
+  }
+
+  if (terminalElement == null) {
+    throw new Error('No terminal element found')
+  }
+
+  return {
+    ipfsRoots,
+    terminalElement
+  }
+}