@helia/verified-fetch 0.0.0-9b1ddf8 → 0.0.0-dc2e7a6

package/src/utils/get-e-tag.ts

@@ -0,0 +1,36 @@
+import type { RequestFormatShorthand } from '../types.js'
+import type { CID } from 'multiformats/cid'
+
+interface GetETagArg {
+  cid: CID
+  reqFormat?: RequestFormatShorthand
+  rangeStart?: number
+  rangeEnd?: number
+  /**
+   * Weak Etag is used when we can't guarantee byte-for-byte-determinism (generated, or mutable content).
+   * Some examples:
+   * - IPNS requests
+   * - CAR streamed with blocks in non-deterministic order
+   * - TAR streamed with files in non-deterministic order
+   */
+  weak?: boolean
+}
+
+/**
+ * etag
+ * you need to wrap cid  with ""
+ * we use strong Etags for immutable responses and weak one (prefixed with W/ ) for mutable/generated ones (ipns and generated HTML).
+ * block and car responses should have different etag than deserialized one, so you can add some prefix like we do in existing gateway
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
+ * @see https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
+ */
+export function getETag ({ cid, reqFormat, weak, rangeStart, rangeEnd }: GetETagArg): string {
+  const prefix = weak === true ? 'W/' : ''
+  let suffix = reqFormat == null ? '' : `.${reqFormat}`
+  if (rangeStart != null || rangeEnd != null) {
+    suffix += `.${rangeStart ?? '0'}-${rangeEnd ?? 'N'}`
+  }
+
+  return `${prefix}"${cid.toString()}${suffix}"`
+}

package/src/utils/parse-url-string.ts

@@ -1,6 +1,7 @@
 import { peerIdFromString } from '@libp2p/peer-id'
 import { CID } from 'multiformats/cid'
 import { TLRU } from './tlru.js'
+import type { RequestFormatShorthand } from '../types.js'
 import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressOptions } from 'progress-events'
@@ -16,11 +17,15 @@ export interface ParseUrlStringOptions extends ProgressOptions<ResolveProgressEv
 
 }
 
+export interface ParsedUrlQuery extends Record<string, string | unknown> {
+  format?: RequestFormatShorthand
+}
+
 export interface ParsedUrlStringResults {
   protocol: string
   path: string
   cid: CID
-  query: Record<string, string>
+  query: ParsedUrlQuery
 }
 
 const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/$?]+)\/?(?<path>[^$?]*)\??(?<queryString>.*)$/

package/src/verified-fetch.ts

@@ -1,19 +1,20 @@
-import { dagCbor as heliaDagCbor, type DAGCBOR } from '@helia/dag-cbor'
-import { dagJson as heliaDagJson, type DAGJSON } from '@helia/dag-json'
 import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
 import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers'
-import { json as heliaJson, type JSON } from '@helia/json'
 import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs'
 import { code as dagCborCode } from '@ipld/dag-cbor'
 import { code as dagJsonCode } from '@ipld/dag-json'
 import { code as dagPbCode } from '@ipld/dag-pb'
 import { code as jsonCode } from 'multiformats/codecs/json'
-import { decode, code as rawCode } from 'multiformats/codecs/raw'
+import { code as rawCode } from 'multiformats/codecs/raw'
+import { identity } from 'multiformats/hashes/identity'
 import { CustomProgressEvent } from 'progress-events'
+import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js'
+import { getETag } from './utils/get-e-tag.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { parseResource } from './utils/parse-resource.js'
 import { walkPath, type PathWalkerFn } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
+import type { RequestFormatShorthand } from './types.js'
 import type { Helia } from '@helia/interface'
 import type { AbortOptions, Logger } from '@libp2p/interface'
 import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
@@ -23,9 +24,6 @@ interface VerifiedFetchComponents {
   helia: Helia
   ipns?: IPNS
   unixfs?: HeliaUnixFs
-  dagJson?: DAGJSON
-  json?: JSON
-  dagCbor?: DAGCBOR
   pathWalker?: PathWalkerFn
 }
 
@@ -62,18 +60,29 @@ function convertOptions (options?: VerifiedFetchOptions): (Omit<VerifiedFetchOpt
   }
 }
 
+function okResponse (body?: BodyInit | null): Response {
+  return new Response(body, {
+    status: 200,
+    statusText: 'OK'
+  })
+}
+
+function notSupportedResponse (body?: BodyInit | null): Response {
+  return new Response(body, {
+    status: 501,
+    statusText: 'Not Implemented'
+  })
+}
+
 export class VerifiedFetch {
   private readonly helia: Helia
   private readonly ipns: IPNS
   private readonly unixfs: HeliaUnixFs
-  private readonly dagJson: DAGJSON
-  private readonly dagCbor: DAGCBOR
-  private readonly json: JSON
   private readonly pathWalker: PathWalkerFn
   private readonly log: Logger
   private readonly contentTypeParser: ContentTypeParser | undefined
 
-  constructor ({ helia, ipns, unixfs, dagJson, json, dagCbor, pathWalker }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
+  constructor ({ helia, ipns, unixfs, pathWalker }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
     this.helia = helia
     this.log = helia.logger.forComponent('helia:verified-fetch')
     this.ipns = ipns ?? heliaIpns(helia, {
@@ -83,9 +92,6 @@ export class VerifiedFetch {
       ]
     })
     this.unixfs = unixfs ?? heliaUnixFs(helia)
-    this.dagJson = dagJson ?? heliaDagJson(helia)
-    this.json = json ?? heliaJson(helia)
-    this.dagCbor = dagCbor ?? heliaDagCbor(helia)
     this.pathWalker = pathWalker ?? walkPath
     this.contentTypeParser = init?.contentTypeParser
     this.log.trace('created VerifiedFetch instance')
@@ -93,54 +99,48 @@ export class VerifiedFetch {
 
   // handle vnd.ipfs.ipns-record
   private async handleIPNSRecord ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
-    const response = new Response('vnd.ipfs.ipns-record support is not implemented', { status: 501 })
+    const response = notSupportedResponse('vnd.ipfs.ipns-record support is not implemented')
     response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
     return response
   }
 
   // handle vnd.ipld.car
   private async handleIPLDCar ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
-    const response = new Response('vnd.ipld.car support is not implemented', { status: 501 })
+    const response = notSupportedResponse('vnd.ipld.car support is not implemented')
     response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
     return response
   }
 
-  private async handleDagJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
-    this.log.trace('fetching %c/%s', cid, path)
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid, path }))
-    const result = await this.dagJson.get(cid, {
-      signal: options?.signal,
-      onProgress: options?.onProgress
-    })
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid, path }))
-    const response = new Response(JSON.stringify(result), { status: 200 })
-    response.headers.set('content-type', 'application/json')
-    return response
-  }
-
   private async handleJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
     this.log.trace('fetching %c/%s', cid, path)
     options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid, path }))
-    const result: Record<any, any> = await this.json.get(cid, {
+    const result = await this.helia.blockstore.get(cid, {
       signal: options?.signal,
       onProgress: options?.onProgress
     })
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid, path }))
-    const response = new Response(JSON.stringify(result), { status: 200 })
+    const response = okResponse(result)
     response.headers.set('content-type', 'application/json')
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid, path }))
     return response
   }
 
   private async handleDagCbor ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
     this.log.trace('fetching %c/%s', cid, path)
     options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid, path }))
-    const result = await this.dagCbor.get<Uint8Array>(cid, {
-      signal: options?.signal,
-      onProgress: options?.onProgress
-    })
+    // return body as binary
+    const block = await this.helia.blockstore.get(cid)
+    let body: string | Uint8Array
+
+    try {
+      body = dagCborToSafeJSON(block)
+    } catch (err) {
+      this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err)
+      body = block
+    }
+
+    const response = okResponse(body)
+    response.headers.set('content-type', body instanceof Uint8Array ? 'application/octet-stream' : 'application/json')
     options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid, path }))
-    const response = new Response(result, { status: 200 })
-    await this.setContentType(result, path, response)
     return response
   }
 
@@ -166,7 +166,7 @@ export class VerifiedFetch {
         // terminalElement = stat
       } catch (err: any) {
         this.log('error loading path %c/%s', dirCid, rootFilePath, err)
-        return new Response('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented', { status: 501 })
+        return notSupportedResponse('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented')
       } finally {
         options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: dirCid, path: rootFilePath }))
       }
@@ -177,15 +177,16 @@ export class VerifiedFetch {
       signal: options?.signal,
       onProgress: options?.onProgress
     })
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: resolvedCID, path: '' }))
     this.log('got async iterator for %c/%s', cid, path)
 
     const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
       onProgress: options?.onProgress
     })
-    const response = new Response(stream, { status: 200 })
+    const response = okResponse(stream)
     await this.setContentType(firstChunk, path, response)
 
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: resolvedCID, path: '' }))
+
     return response
   }
 
@@ -193,9 +194,10 @@ export class VerifiedFetch {
     this.log.trace('fetching %c/%s', cid, path)
     options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:start', { cid, path }))
     const result = await this.helia.blockstore.get(cid)
-    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid, path }))
-    const response = new Response(decode(result), { status: 200 })
+    const response = okResponse(result)
     await this.setContentType(result, path, response)
+
+    options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid, path }))
     return response
   }
 
@@ -231,8 +233,8 @@ export class VerifiedFetch {
    * @see https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter
    * @default 'raw'
    */
-  private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: string | null }): string | null {
-    const formatMap: Record<string, string> = {
+  private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: RequestFormatShorthand | null }): RequestFormatShorthand | null {
+    const formatMap: Record<string, RequestFormatShorthand> = {
       'vnd.ipld.raw': 'raw',
       'vnd.ipld.car': 'car',
       'application/x-tar': 'tar',
@@ -261,22 +263,23 @@ export class VerifiedFetch {
    * These format handlers should adjust the response headers as specified in https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers
    */
   private readonly formatHandlers: Record<string, FetchHandlerFunction> = {
-    raw: async () => new Response('application/vnd.ipld.raw support is not implemented', { status: 501 }),
+    raw: async () => notSupportedResponse('application/vnd.ipld.raw support is not implemented'),
     car: this.handleIPLDCar,
     'ipns-record': this.handleIPNSRecord,
-    tar: async () => new Response('application/x-tar support is not implemented', { status: 501 }),
-    'dag-json': async () => new Response('application/vnd.ipld.dag-json support is not implemented', { status: 501 }),
-    'dag-cbor': async () => new Response('application/vnd.ipld.dag-cbor support is not implemented', { status: 501 }),
-    json: async () => new Response('application/json support is not implemented', { status: 501 }),
-    cbor: async () => new Response('application/cbor support is not implemented', { status: 501 })
+    tar: async () => notSupportedResponse('application/x-tar support is not implemented'),
+    'dag-json': async () => notSupportedResponse('application/vnd.ipld.dag-json support is not implemented'),
+    'dag-cbor': async () => notSupportedResponse('application/vnd.ipld.dag-cbor support is not implemented'),
+    json: async () => notSupportedResponse('application/json support is not implemented'),
+    cbor: async () => notSupportedResponse('application/cbor support is not implemented')
   }
 
   private readonly codecHandlers: Record<number, FetchHandlerFunction> = {
-    [dagJsonCode]: this.handleDagJson,
     [dagPbCode]: this.handleDagPb,
+    [dagJsonCode]: this.handleJson,
     [jsonCode]: this.handleJson,
     [dagCborCode]: this.handleDagCbor,
-    [rawCode]: this.handleRaw
+    [rawCode]: this.handleRaw,
+    [identity.code]: this.handleRaw
   }
 
   async fetch (resource: Resource, opts?: VerifiedFetchOptions): Promise<Response> {
@@ -318,12 +321,12 @@ export class VerifiedFetch {
       if (codecHandler != null) {
         response = await codecHandler.call(this, { cid, path, options, terminalElement })
       } else {
-        return new Response(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`, { status: 501 })
+        return notSupportedResponse(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`)
       }
     }
 
-    response.headers.set('etag', cid.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
-    response.headers.set('cache-cotrol', 'public, max-age=29030400, immutable')
+    response.headers.set('etag', getETag({ cid, reqFormat: format ?? undefined, weak: false }))
+    response.headers.set('cache-control', 'public, max-age=29030400, immutable')
     response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header
 
     if (ipfsRoots != null) {