@helia/verified-fetch 0.0.0-6c88ee1 → 0.0.0-754c7af

package/src/index.ts

@@ -3,7 +3,7 @@
  *
  * `@helia/verified-fetch` provides a [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)-like API for retrieving content from the [IPFS](https://ipfs.tech/) network.
  *
- * All content is retrieved in a [trustless manner](https://www.techopedia.com/definition/trustless), and the integrity of all bytes are verified by comparing hashes of the data.
+ * All content is retrieved in a [trustless manner](https://www.techopedia.com/definition/trustless), and the integrity of all bytes are verified by comparing hashes of the data. By default, CIDs are retrieved over HTTP from [trustless gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/).
  *
  * This is a marked improvement over `fetch` which offers no such protections and is vulnerable to all sorts of attacks like [Content Spoofing](https://owasp.org/www-community/attacks/Content_Spoofing), [DNS Hijacking](https://en.wikipedia.org/wiki/DNS_hijacking), etc.
  *
@@ -33,7 +33,7 @@
  * import { verifiedFetch } from '@helia/verified-fetch'
  * import { CID } from 'multiformats/cid'
  *
- * const cid = CID.parse('bafyFoo') // some image file
+ * const cid = CID.parse('bafyFoo') // some json file
  * const response = await verifiedFetch(cid)
  * const json = await response.json()
  * ```
@@ -75,7 +75,7 @@
  * const fetch = await createVerifiedFetch({
  *  gateways: ['https://trustless-gateway.link'],
  *  routers: ['http://delegated-ipfs.dev']
- *})
+ * })
  *
  * const resp = await fetch('ipfs://bafy...')
  *
@@ -112,6 +112,214 @@
  * const json = await resp.json()
  * ```
  *
+ * ### Custom content-type parsing
+ *
+ * By default, if the response can be parsed as JSON, `@helia/verified-fetch` sets the `Content-Type` header as `application/json`, otherwise it sets it as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type.
+ *
+ * If you require an accurate content-type you can provide a `contentTypeParser` function as an option to `createVerifiedFetch` to handle parsing the content type.
+ *
+ * The function you provide will be called with the first chunk of bytes from the file and should return a string or a promise of a string.
+ *
+ * @example Customizing content-type parsing
+ *
+ * ```typescript
+ * import { createVerifiedFetch } from '@helia/verified-fetch'
+ * import { fileTypeFromBuffer } from '@sgtpooki/file-type'
+ *
+ * const fetch = await createVerifiedFetch({
+ *  gateways: ['https://trustless-gateway.link'],
+ *  routers: ['http://delegated-ipfs.dev']
+ * }, {
+ *  contentTypeParser: async (bytes) => {
+ *    // call to some magic-byte recognition library like magic-bytes, file-type, or your own custom byte recognition
+ *    const result = await fileTypeFromBuffer(bytes)
+ *    return result?.mime
+ *  }
+ * })
+ * ```
+ *
+ * ### IPLD codec handling
+ *
+ * IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption.
+ *
+ * #### DAG-PB
+ *
+ * [DAG-PB](https://ipld.io/docs/codecs/known/dag-pb/) is the codec we are most likely to encounter, it is what [UnixFS](https://github.com/ipfs/specs/blob/main/UNIXFS.md) uses under the hood.
+ *
+ * ##### Using the DAG-PB codec as a Blob
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://Qmfoo')
+ * const blob = await res.blob()
+ *
+ * console.info(blob) // Blob { size: x, type: 'application/octet-stream' }
+ * ```
+ *
+ * ##### Using the DAG-PB codec as an ArrayBuffer
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://Qmfoo')
+ * const buf = await res.arrayBuffer()
+ *
+ * console.info(buf) // ArrayBuffer { [Uint8Contents]: < ... >, byteLength: x }
+ * ```
+ *
+ * ##### Using the DAG-PB codec as a stream
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://Qmfoo')
+ * const reader = res.body?.getReader()
+ *
+ * while (true) {
+ *   const next = await reader.read()
+ *
+ *   if (next?.done === true) {
+ *     break
+ *   }
+ *
+ *   if (next?.value != null) {
+ *     console.info(next.value) // Uint8Array(x) [ ... ]
+ *   }
+ * }
+ * ```
+ *
+ * ##### Content-Type
+ *
+ * When fetching `DAG-PB` data, the content type will be set to `application/octet-stream` unless a custom content-type parser is configured.
+ *
+ * #### JSON
+ *
+ * The JSON codec is a very simple codec, a block parseable with this codec is a JSON string encoded into a `Uint8Array`.
+ *
+ * ##### Using the JSON codec
+ *
+ * ```typescript
+ * import * as json from 'multiformats/codecs/json'
+ *
+ * const block = new TextEncoder().encode('{ "hello": "world" }')
+ * const obj = json.decode(block)
+ *
+ * console.info(obj) // { hello: 'world' }
+ * ```
+ *
+ * ##### Content-Type
+ *
+ * When the `JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`.
+ *
+ * ### DAG-JSON
+ *
+ * [DAG-JSON](https://ipld.io/docs/codecs/known/dag-json/) expands on the `JSON` codec, adding the ability to contain [CID](https://docs.ipfs.tech/concepts/content-addressing/)s which act as links to other blocks, and byte arrays.
+ *
+ * `CID`s and byte arrays are represented using special object structures with a single `"/"` property.
+ *
+ * Using `DAG-JSON` has two important caveats:
+ *
+ * 1. Your `JSON` structure cannot contain an object with only a `"/"` property, as it will be interpreted as a special type.
+ * 2. Since `JSON` has no technical limit on number sizes, `DAG-JSON` also allows numbers larger than `Number.MAX_SAFE_INTEGER`. JavaScript requires use of `BigInt`s to represent numbers larger than this, and `JSON.parse` does not support them, so precision will be lost.
+ *
+ * Otherwise this codec follows the same rules as the `JSON` codec.
+ *
+ * ##### Using the DAG-JSON codec
+ *
+ * ```typescript
+ * import * as dagJson from '@ipld/dag-json'
+ *
+ * const block = new TextEncoder().encode(`{
+ *   "hello": "world",
+ *   "cid": {
+ *     "/": "baeaaac3imvwgy3zao5xxe3de"
+ *   },
+ *   "buf": {
+ *     "/": {
+ *       "bytes": "AAECAwQ"
+ *     }
+ *   }
+ * }`)
+ *
+ * const obj = dagJson.decode(block)
+ *
+ * console.info(obj)
+ * // {
+ * // hello: 'world',
+ * // cid: CID(baeaaac3imvwgy3zao5xxe3de),
+ * // buf: Uint8Array(5) [ 0, 1, 2, 3, 4 ]
+ * // }
+ * ```
+ *
+ * ##### Content-Type
+ *
+ * When the `DAG-JSON` codec is encountered in the requested CID, the `Content-Type` header of the response will be set to `application/json`.
+ *
+ * `DAG-JSON` data can be parsed from the response by using the `.json()` function, which will return `CID`s/byte arrays as plain `{ "/": ... }` objects:
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ * import * as dagJson from '@ipld/dag-json'
+ *
+ * const res = await verifiedFetch('ipfs://bafyDAGJSON')
+ *
+ * // either:
+ * const obj = await res.json()
+ * console.info(obj.cid) // { "/": "baeaaac3imvwgy3zao5xxe3de" }
+ * console.info(obj.buf) // { "/": { "bytes": "AAECAwQ" } }
+ * ```
+ *
+ * Alternatively, it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get `CID` objects and `Uint8Array`s:
+ *
+ *```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ * import * as dagJson from '@ipld/dag-json'
+ *
+ * const res = await verifiedFetch('ipfs://bafyDAGJSON')
+ *
+ * // or:
+ * const obj = dagJson.decode(await res.arrayBuffer())
+ * console.info(obj.cid) // CID(baeaaac3imvwgy3zao5xxe3de)
+ * console.info(obj.buf) // Uint8Array(5) [ 0, 1, 2, 3, 4 ]
+ * ```
+ *
+ * #### DAG-CBOR
+ *
+ * [DAG-CBOR](https://ipld.io/docs/codecs/known/dag-cbor/) uses the [Concise Binary Object Representation](https://cbor.io/) format for serialization instead of JSON.
+ *
+ * This supports more datatypes in a safer way than JSON and is smaller on the wire to boot so is usually preferable to JSON or DAG-JSON.
+ *
+ * ##### Content-Type
+ *
+ * Not all data types supported by `DAG-CBOR` can be successfully turned into JSON and back into the same binary form.
+ *
+ * When a decoded block can be round-tripped to JSON, the `Content-Type` will be set to `application/json`. In this case the `.json()` method on the `Response` object can be used to obtain an object representation of the response.
+ *
+ * When it cannot, the `Content-Type` will be `application/octet-stream` - in this case the `@ipld/dag-json` module must be used to deserialize the return value from `.arrayBuffer()`.
+ *
+ * ##### Detecting JSON-safe DAG-CBOR
+ *
+ * If the `Content-Type` header of the response is `application/json`, the `.json()` method may be used to access the response body in object form, otherwise the `.arrayBuffer()` method must be used to decode the raw bytes using the `@ipld/dag-cbor` module.
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ * import * as dagCbor from '@ipld/dag-cbor'
+ *
+ * const res = await verifiedFetch('ipfs://bafyDagCborCID')
+ * let obj
+ *
+ * if (res.headers.get('Content-Type') === 'application/json') {
+ *   // DAG-CBOR data can be safely decoded as JSON
+ *   obj = await res.json()
+ * } else {
+ *   // response contains non-JSON friendly data types
+ *   obj = dagCbor.decode(await res.arrayBuffer())
+ * }
+ *
+ * console.info(obj) // ...
+ * ```
+ *
  * ## Comparison to fetch
  *
  * This module attempts to act as similarly to the `fetch()` API as possible.
@@ -129,7 +337,7 @@
  * 2. IPNS protocol: `ipns://<peerId>` & `ipns://<publicKey>` & `ipns://<hostUri_Supporting_DnsLink_TxtRecords>`
  * 3. CID instances: An actual CID instance `CID.parse('bafy...')`
  *
- * As well as support for pathing & params for item 1 & 2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453).
+ * As well as support for pathing & params for items 1 & 2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453).
  *
  * If you pass a CID instance, it assumes you want the content for that specific CID only, and does not support pathing or params for that CID.
  *
@@ -242,7 +450,7 @@ import type { ProgressEvent, ProgressOptions } from 'progress-events'
 export type Resource = string | CID
 
 export interface CIDDetail {
-  cid: string
+  cid: CID
   path: string
 }
 
@@ -257,13 +465,38 @@ export interface VerifiedFetch {
 }
 
 /**
- * Instead of passing a Helia instance, you can pass a list of gateways and routers, and a HeliaHTTP instance will be created for you.
+ * Instead of passing a Helia instance, you can pass a list of gateways and
+ * routers, and a HeliaHTTP instance will be created for you.
  */
-export interface CreateVerifiedFetchWithOptions {
+export interface CreateVerifiedFetchInit {
   gateways: string[]
   routers?: string[]
 }
 
+export interface CreateVerifiedFetchOptions {
+  /**
+   * A function to handle parsing content type from bytes. The function you
+   * provide will be passed the first set of bytes we receive from the network,
+   * and should return a string that will be used as the value for the
+   * `Content-Type` header in the response.
+   */
+  contentTypeParser?: ContentTypeParser
+}
+
+/**
+ * A ContentTypeParser attempts to return the mime type of a given file. It
+ * receives the first chunk of the file data and the file name, if it is
+ * available.  The function can be sync or async and if it returns/resolves to
+ * `undefined`, `application/octet-stream` will be used.
+ */
+export interface ContentTypeParser {
+  /**
+   * Attempt to determine a mime type, either via of the passed bytes or the
+   * filename if it is available.
+   */
+  (bytes: Uint8Array, fileName?: string): Promise<string | undefined> | string | undefined
+}
+
 export type BubbledProgressEvents =
   // unixfs
   GetEvents |
@@ -280,8 +513,9 @@ export type VerifiedFetchProgressEvents =
 /**
  * Options for the `fetch` function returned by `createVerifiedFetch`.
  *
- * This method accepts all the same options as the `fetch` function in the browser, plus an `onProgress` option to
- * listen for progress events.
+ * This interface contains all the same fields as the [options object](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options)
+ * passed to `fetch` in browsers, plus an `onProgress` option to listen for
+ * progress events.
  */
 export interface VerifiedFetchInit extends RequestInit, ProgressOptions<BubbledProgressEvents | VerifiedFetchProgressEvents> {
 }
@@ -289,7 +523,9 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions<BubbledP
 /**
  * Create and return a Helia node
  */
-export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchWithOptions): Promise<VerifiedFetch> {
+export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchInit, options?: CreateVerifiedFetchOptions): Promise<VerifiedFetch> {
+  const contentTypeParser: ContentTypeParser | undefined = options?.contentTypeParser
+
   if (!isHelia(init)) {
     init = await createHeliaHTTP({
       blockBrokers: [
@@ -301,7 +537,7 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchWit
     })
   }
 
-  const verifiedFetchInstance = new VerifiedFetchClass({ helia: init })
+  const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, { contentTypeParser })
   async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise<Response> {
     return verifiedFetchInstance.fetch(resource, options)
   }

package/src/utils/dag-cbor-to-safe-json.ts

@@ -0,0 +1,44 @@
+import { decode } from 'cborg'
+import { encode } from 'cborg/json'
+import { CID } from 'multiformats/cid'
+import type { TagDecoder } from 'cborg'
+
+// https://github.com/ipfs/go-ipfs/issues/3570#issuecomment-273931692
+const CID_CBOR_TAG = 0x2A
+
+function cidDecoder (bytes: Uint8Array): CID {
+  if (bytes[0] !== 0) {
+    throw new Error('Invalid CID for CBOR tag 42; expected leading 0x00')
+  }
+
+  return CID.decode(bytes.subarray(1)) // ignore leading 0x00
+}
+
+/**
+ * Take a `DAG-CBOR` encoded `Uint8Array`, deserialize it as an object and
+ * re-serialize it in a form that can be passed to `JSON.serialize` and then
+ * `JSON.parse` without losing any data.
+ */
+export function dagCborToSafeJSON (buf: Uint8Array): string {
+  const tags: TagDecoder[] = []
+  tags[CID_CBOR_TAG] = cidDecoder
+
+  const obj = decode(buf, {
+    allowIndefinite: false,
+    coerceUndefinedToNull: true,
+    allowNaN: false,
+    allowInfinity: false,
+    strict: true,
+    useMaps: false,
+    rejectDuplicateMapKeys: true,
+    tags,
+
+    // this is different to `DAG-CBOR` - the reason we disallow BigInts is
+    // because we are about to re-encode to `JSON` which does not support
+    // BigInts. Blocks containing large numbers should be deserialized using a
+    // cbor decoder instead
+    allowBigInt: false
+  })
+
+  return new TextDecoder().decode(encode(obj))
+}

package/src/utils/{get-stream-and-content-type.ts → get-stream-from-async-iterable.ts}

@@ -1,27 +1,25 @@
 import { CustomProgressEvent } from 'progress-events'
-import { getContentType } from './get-content-type.js'
 import type { VerifiedFetchInit } from '../index.js'
 import type { ComponentLogger } from '@libp2p/interface'
 
 /**
- * Converts an async iterator of Uint8Array bytes to a stream and attempts to determine the content type of those bytes.
+ * Converts an async iterator of Uint8Array bytes to a stream and returns the first chunk of bytes
  */
-export async function getStreamAndContentType (iterator: AsyncIterable<Uint8Array>, path: string, logger: ComponentLogger, options?: Pick<VerifiedFetchInit, 'onProgress'>): Promise<{ contentType: string, stream: ReadableStream<Uint8Array> }> {
-  const log = logger.forComponent('helia:verified-fetch:get-stream-and-content-type')
+export async function getStreamFromAsyncIterable (iterator: AsyncIterable<Uint8Array>, path: string, logger: ComponentLogger, options?: Pick<VerifiedFetchInit, 'onProgress'>): Promise<{ stream: ReadableStream<Uint8Array>, firstChunk: Uint8Array }> {
+  const log = logger.forComponent('helia:verified-fetch:get-stream-from-async-iterable')
   const reader = iterator[Symbol.asyncIterator]()
-  const { value, done } = await reader.next()
+  const { value: firstChunk, done } = await reader.next()
 
   if (done === true) {
     log.error('No content found for path', path)
     throw new Error('No content found')
   }
 
-  const contentType = await getContentType({ bytes: value, path })
   const stream = new ReadableStream({
     async start (controller) {
       // the initial value is already available
       options?.onProgress?.(new CustomProgressEvent<void>('verified-fetch:request:progress:chunk'))
-      controller.enqueue(value)
+      controller.enqueue(firstChunk)
     },
     async pull (controller) {
       const { value, done } = await reader.next()
@@ -40,5 +38,8 @@ export async function getStreamAndContentType (iterator: AsyncIterable<Uint8Arra
     }
   })
 
-  return { contentType, stream }
+  return {
+    stream,
+    firstChunk
+  }
 }