@helia/verified-fetch 0.0.0 → 1.0.1

package/src/utils/dag-cbor-to-safe-json.ts

@@ -0,0 +1,27 @@
+import { decode } from 'cborg'
+import { encode } from 'cborg/json'
+
+/**
+ * Take a `DAG-CBOR` encoded `Uint8Array`, deserialize it as an object and
+ * re-serialize it in a form that can be passed to `JSON.serialize` and then
+ * `JSON.parse` without losing any data.
+ */
+export function dagCborToSafeJSON (buf: Uint8Array): string {
+  const obj = decode(buf, {
+    allowIndefinite: false,
+    coerceUndefinedToNull: false,
+    allowNaN: false,
+    allowInfinity: false,
+    strict: true,
+    useMaps: false,
+    rejectDuplicateMapKeys: true,
+
+    // this is different to `DAG-CBOR` - the reason we disallow BigInts is
+    // because we are about to re-encode to `JSON` which does not support
+    // BigInts. Blocks containing large numbers should be deserialized using a
+    // cbor decoder instead
+    allowBigInt: false
+  })
+
+  return new TextDecoder().decode(encode(obj))
+}

package/src/utils/get-content-disposition-filename.ts

@@ -0,0 +1,18 @@
+/**
+ * Takes a filename URL param and returns a string for use in a
+ * `Content-Disposition` header
+ */
+export function getContentDispositionFilename (filename: string): string {
+  const asciiOnly = replaceNonAsciiCharacters(filename)
+
+  if (asciiOnly === filename) {
+    return `filename="${filename}"`
+  }
+
+  return `filename="${asciiOnly}"; filename*=UTF-8''${encodeURIComponent(filename)}`
+}
+
+function replaceNonAsciiCharacters (filename: string): string {
+  // eslint-disable-next-line no-control-regex
+  return filename.replace(/[^\x00-\x7F]/g, '_')
+}

package/src/utils/get-e-tag.ts

@@ -0,0 +1,36 @@
+import type { RequestFormatShorthand } from '../types.js'
+import type { CID } from 'multiformats/cid'
+
+interface GetETagArg {
+  cid: CID
+  reqFormat?: RequestFormatShorthand
+  rangeStart?: number
+  rangeEnd?: number
+  /**
+   * Weak Etag is used when we can't guarantee byte-for-byte-determinism (generated, or mutable content).
+   * Some examples:
+   * - IPNS requests
+   * - CAR streamed with blocks in non-deterministic order
+   * - TAR streamed with files in non-deterministic order
+   */
+  weak?: boolean
+}
+
+/**
+ * etag
+ * you need to wrap cid  with ""
+ * we use strong Etags for immutable responses and weak one (prefixed with W/ ) for mutable/generated ones (ipns and generated HTML).
+ * block and car responses should have different etag than deserialized one, so you can add some prefix like we do in existing gateway
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
+ * @see https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
+ */
+export function getETag ({ cid, reqFormat, weak, rangeStart, rangeEnd }: GetETagArg): string {
+  const prefix = weak === true ? 'W/' : ''
+  let suffix = reqFormat == null ? '' : `.${reqFormat}`
+  if (rangeStart != null || rangeEnd != null) {
+    suffix += `.${rangeStart ?? '0'}-${rangeEnd ?? 'N'}`
+  }
+
+  return `${prefix}"${cid.toString()}${suffix}"`
+}

package/src/utils/{get-stream-and-content-type.ts → get-stream-from-async-iterable.ts}

@@ -1,27 +1,25 @@
 import { CustomProgressEvent } from 'progress-events'
-import { getContentType } from './get-content-type.js'
 import type { VerifiedFetchInit } from '../index.js'
 import type { ComponentLogger } from '@libp2p/interface'
 
 /**
- * Converts an async iterator of Uint8Array bytes to a stream and attempts to determine the content type of those bytes.
+ * Converts an async iterator of Uint8Array bytes to a stream and returns the first chunk of bytes
  */
-export async function getStreamAndContentType (iterator: AsyncIterable<Uint8Array>, path: string, logger: ComponentLogger, options?: Pick<VerifiedFetchInit, 'onProgress'>): Promise<{ contentType: string, stream: ReadableStream<Uint8Array> }> {
-  const log = logger.forComponent('helia:verified-fetch:get-stream-and-content-type')
+export async function getStreamFromAsyncIterable (iterator: AsyncIterable<Uint8Array>, path: string, logger: ComponentLogger, options?: Pick<VerifiedFetchInit, 'onProgress'>): Promise<{ stream: ReadableStream<Uint8Array>, firstChunk: Uint8Array }> {
+  const log = logger.forComponent('helia:verified-fetch:get-stream-from-async-iterable')
   const reader = iterator[Symbol.asyncIterator]()
-  const { value, done } = await reader.next()
-  options?.onProgress?.(new CustomProgressEvent<void>('verified-fetch:request:progress:chunk'))
+  const { value: firstChunk, done } = await reader.next()
 
   if (done === true) {
     log.error('No content found for path', path)
     throw new Error('No content found')
   }
 
-  const contentType = await getContentType({ bytes: value, path })
   const stream = new ReadableStream({
     async start (controller) {
       // the initial value is already available
-      controller.enqueue(value)
+      options?.onProgress?.(new CustomProgressEvent<void>('verified-fetch:request:progress:chunk'))
+      controller.enqueue(firstChunk)
     },
     async pull (controller) {
       const { value, done } = await reader.next()
@@ -40,5 +38,8 @@ export async function getStreamAndContentType (iterator: AsyncIterable<Uint8Arra
     }
   })
 
-  return { contentType, stream }
+  return {
+    stream,
+    firstChunk
+  }
 }

package/src/utils/get-tar-stream.ts

@@ -0,0 +1,68 @@
+import { CodeError } from '@libp2p/interface'
+import { exporter, recursive, type UnixFSEntry } from 'ipfs-unixfs-exporter'
+import map from 'it-map'
+import { pipe } from 'it-pipe'
+import { pack, type TarEntryHeader, type TarImportCandidate } from 'it-tar'
+import type { AbortOptions } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+
+const EXPORTABLE = ['file', 'raw', 'directory']
+
+function toHeader (file: UnixFSEntry): Partial<TarEntryHeader> & { name: string } {
+  let mode: number | undefined
+  let mtime: Date | undefined
+
+  if (file.type === 'file' || file.type === 'directory') {
+    mode = file.unixfs.mode
+    mtime = file.unixfs.mtime != null ? new Date(Number(file.unixfs.mtime.secs * 1000n)) : undefined
+  }
+
+  return {
+    name: file.path,
+    mode,
+    mtime,
+    size: Number(file.size),
+    type: file.type === 'directory' ? 'directory' : 'file'
+  }
+}
+
+function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate {
+  if (!EXPORTABLE.includes(entry.type)) {
+    throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS')
+  }
+
+  const candidate: TarImportCandidate = {
+    header: toHeader(entry)
+  }
+
+  if (entry.type === 'file' || entry.type === 'raw') {
+    candidate.body = entry.content()
+  }
+
+  return candidate
+}
+
+export async function * tarStream (ipfsPath: string, blockstore: Blockstore, options?: AbortOptions): AsyncGenerator<Uint8Array> {
+  const file = await exporter(ipfsPath, blockstore, options)
+
+  if (file.type === 'file' || file.type === 'raw') {
+    yield * pipe(
+      [toTarImportCandidate(file)],
+      pack()
+    )
+
+    return
+  }
+
+  if (file.type === 'directory') {
+    yield * pipe(
+      recursive(ipfsPath, blockstore, options),
+      (source) => map(source, (entry) => toTarImportCandidate(entry)),
+      pack()
+    )
+
+    return
+  }
+
+  throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS')
+}

package/src/utils/parse-url-string.ts

@@ -1,6 +1,7 @@
 import { peerIdFromString } from '@libp2p/peer-id'
 import { CID } from 'multiformats/cid'
 import { TLRU } from './tlru.js'
+import type { RequestFormatShorthand } from '../types.js'
 import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressOptions } from 'progress-events'
@@ -16,11 +17,17 @@ export interface ParseUrlStringOptions extends ProgressOptions<ResolveProgressEv
 
 }
 
+export interface ParsedUrlQuery extends Record<string, string | unknown> {
+  format?: RequestFormatShorthand
+  download?: boolean
+  filename?: string
+}
+
 export interface ParsedUrlStringResults {
   protocol: string
   path: string
   cid: CID
-  query: Record<string, string>
+  query: ParsedUrlQuery
 }
 
 const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/$?]+)\/?(?<path>[^$?]*)\??(?<queryString>.*)$/
@@ -31,7 +38,6 @@ const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/$?]+)\/?
  * After determining the protocol successfully, we process the cidOrPeerIdOrDnsLink:
  * * If it's ipfs, it parses the CID or throws an Aggregate error
  * * If it's ipns, it attempts to resolve the PeerId and then the DNSLink. If both fail, an Aggregate error is thrown.
- *
  */
 export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStringInput, options?: ParseUrlStringOptions): Promise<ParsedUrlStringResults> {
   const log = logger.forComponent('helia:verified-fetch:parse-url-string')
@@ -105,7 +111,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
   }
 
   // parse query string
-  const query: Record<string, string> = {}
+  const query: Record<string, any> = {}
 
   if (queryString != null && queryString.length > 0) {
     const queryParts = queryString.split('&')
@@ -113,6 +119,14 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
       const [key, value] = part.split('=')
       query[key] = decodeURIComponent(value)
     }
+
+    if (query.download != null) {
+      query.download = query.download === 'true'
+    }
+
+    if (query.filename != null) {
+      query.filename = query.filename.toString()
+    }
   }
 
   /**

package/src/utils/responses.ts

@@ -0,0 +1,29 @@
+export function okResponse (body?: BodyInit | null): Response {
+  return new Response(body, {
+    status: 200,
+    statusText: 'OK'
+  })
+}
+
+export function notSupportedResponse (body?: BodyInit | null): Response {
+  const response = new Response(body, {
+    status: 501,
+    statusText: 'Not Implemented'
+  })
+  response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
+  return response
+}
+
+export function notAcceptableResponse (body?: BodyInit | null): Response {
+  return new Response(body, {
+    status: 406,
+    statusText: 'Not Acceptable'
+  })
+}
+
+export function badRequestResponse (body?: BodyInit | null): Response {
+  return new Response(body, {
+    status: 400,
+    statusText: 'Bad Request'
+  })
+}

package/src/utils/select-output-type.ts

@@ -0,0 +1,167 @@
+import { code as dagCborCode } from '@ipld/dag-cbor'
+import { code as dagJsonCode } from '@ipld/dag-json'
+import { code as dagPbCode } from '@ipld/dag-pb'
+import { code as jsonCode } from 'multiformats/codecs/json'
+import { code as rawCode } from 'multiformats/codecs/raw'
+import type { RequestFormatShorthand } from '../types.js'
+import type { CID } from 'multiformats/cid'
+
+/**
+ * This maps supported response types for each codec supported by verified-fetch
+ */
+const CID_TYPE_MAP: Record<number, string[]> = {
+  [dagCborCode]: [
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car'
+  ],
+  [dagJsonCode]: [
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car'
+  ],
+  [jsonCode]: [
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car'
+  ],
+  [dagPbCode]: [
+    'application/octet-stream',
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car',
+    'application/x-tar'
+  ],
+  [rawCode]: [
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car',
+    'application/x-tar'
+  ]
+}
+
+/**
+ * Selects an output mime-type based on the CID and a passed `Accept` header
+ */
+export function selectOutputType (cid: CID, accept?: string): string | undefined {
+  const cidMimeTypes = CID_TYPE_MAP[cid.code]
+
+  if (accept != null) {
+    return chooseMimeType(accept, cidMimeTypes)
+  }
+}
+
+function chooseMimeType (accept: string, validMimeTypes: string[]): string | undefined {
+  const requestedMimeTypes = accept
+    .split(',')
+    .map(s => {
+      const parts = s.trim().split(';')
+
+      return {
+        mimeType: `${parts[0]}`.trim(),
+        weight: parseQFactor(parts[1])
+      }
+    })
+    .sort((a, b) => {
+      if (a.weight === b.weight) {
+        return 0
+      }
+
+      if (a.weight > b.weight) {
+        return -1
+      }
+
+      return 1
+    })
+    .map(s => s.mimeType)
+
+  for (const headerFormat of requestedMimeTypes) {
+    for (const mimeType of validMimeTypes) {
+      if (headerFormat.includes(mimeType)) {
+        return mimeType
+      }
+
+      if (headerFormat === '*/*') {
+        return mimeType
+      }
+
+      if (headerFormat.startsWith('*/') && mimeType.split('/')[1] === headerFormat.split('/')[1]) {
+        return mimeType
+      }
+
+      if (headerFormat.endsWith('/*') && mimeType.split('/')[0] === headerFormat.split('/')[0]) {
+        return mimeType
+      }
+    }
+  }
+}
+
+/**
+ * Parses q-factor weighting from the accept header to allow letting some mime
+ * types take precedence over others.
+ *
+ * If the q-factor for an acceptable mime representation is omitted it defaults
+ * to `1`.
+ *
+ * All specified values should be in the range 0-1.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept#q
+ */
+function parseQFactor (str?: string): number {
+  if (str != null) {
+    str = str.trim()
+  }
+
+  if (str == null || !str.startsWith('q=')) {
+    return 1
+  }
+
+  const factor = parseFloat(str.replace('q=', ''))
+
+  if (isNaN(factor)) {
+    return 0
+  }
+
+  return factor
+}
+
+const FORMAT_TO_MIME_TYPE: Record<RequestFormatShorthand, string> = {
+  raw: 'application/vnd.ipld.raw',
+  car: 'application/vnd.ipld.car',
+  'dag-json': 'application/vnd.ipld.dag-json',
+  'dag-cbor': 'application/vnd.ipld.dag-cbor',
+  json: 'application/json',
+  cbor: 'application/cbor',
+  'ipns-record': 'application/vnd.ipfs.ipns-record',
+  tar: 'application/x-tar'
+}
+
+/**
+ * Converts a `format=...` query param to a mime type as would be found in the
+ * `Accept` header, if a valid mapping is available
+ */
+export function queryFormatToAcceptHeader (format?: RequestFormatShorthand): string | undefined {
+  if (format != null) {
+    return FORMAT_TO_MIME_TYPE[format]
+  }
+}

package/src/utils/walk-path.ts

@@ -1,10 +1,11 @@
 import { walkPath as exporterWalk, type ExporterOptions, type ReadableStorage, type UnixFSEntry } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
 
 export interface PathWalkerOptions extends ExporterOptions {
 
 }
 export interface PathWalkerResponse {
-  ipfsRoots: string[]
+  ipfsRoots: CID[]
   terminalElement: UnixFSEntry
 
 }
@@ -14,13 +15,11 @@ export interface PathWalkerFn {
 }
 
 export async function walkPath (blockstore: ReadableStorage, path: string, options?: PathWalkerOptions): Promise<PathWalkerResponse> {
-  const entries: UnixFSEntry[] = []
-  const ipfsRoots: string[] = []
+  const ipfsRoots: CID[] = []
   let terminalElement: UnixFSEntry | undefined
 
   for await (const entry of exporterWalk(path, blockstore, options)) {
-    entries.push(entry)
-    ipfsRoots.push(entry.cid.toString())
+    ipfsRoots.push(entry.cid)
     terminalElement = entry
   }