@helia/verified-fetch 4.1.0 → 5.0.0

package/src/utils/content-types.ts

@@ -0,0 +1,159 @@
+import { code as dagCborCode } from '@ipld/dag-cbor'
+import { code as dagJsonCode } from '@ipld/dag-json'
+import { code as dagPbCode } from '@ipld/dag-pb'
+import { code as jsonCode } from 'multiformats/codecs/json'
+import { code as rawCode } from 'multiformats/codecs/raw'
+import { CODEC_CBOR } from '../constants.ts'
+import type { ContentType } from '../index.ts'
+import type { CID } from 'multiformats/cid'
+
+export const MEDIA_TYPE_DAG_CBOR = 'application/vnd.ipld.dag-cbor'
+export const MEDIA_TYPE_CBOR = 'application/cbor'
+export const MEDIA_TYPE_DAG_JSON = 'application/vnd.ipld.dag-json'
+export const MEDIA_TYPE_JSON = 'application/json'
+export const MEDIA_TYPE_RAW = 'application/vnd.ipld.raw'
+export const MEDIA_TYPE_OCTET_STREAM = 'application/octet-stream'
+export const MEDIA_TYPE_IPNS_RECORD = 'application/vnd.ipfs.ipns-record'
+export const MEDIA_TYPE_CAR = 'application/vnd.ipld.car'
+export const MEDIA_TYPE_TAR = 'application/x-tar'
+export const MEDIA_TYPE_DAG_PB = 'application/vnd.ipld.dag-pb'
+
+export const CONTENT_TYPE_OCTET_STREAM: ContentType = {
+  mediaType: MEDIA_TYPE_OCTET_STREAM,
+  codecs: [rawCode, jsonCode, dagJsonCode, CODEC_CBOR, dagCborCode, dagPbCode],
+  immutable: true,
+  extension: '.bin',
+  etag: '',
+  disposition: 'attachment'
+}
+
+export const CONTENT_TYPE_DAG_CBOR: ContentType = {
+  mediaType: MEDIA_TYPE_DAG_CBOR,
+  codecs: [dagCborCode, CODEC_CBOR, dagJsonCode, jsonCode, dagPbCode, rawCode],
+  immutable: true,
+  extension: '.cbor',
+  etag: '.dag-cbor',
+  disposition: 'attachment'
+}
+
+export const CONTENT_TYPE_CBOR: ContentType = {
+  mediaType: MEDIA_TYPE_CBOR,
+  codecs: [CODEC_CBOR, dagCborCode, dagJsonCode, jsonCode, dagPbCode, rawCode],
+  immutable: true,
+  extension: '.cbor',
+  etag: '.cbor',
+  disposition: 'attachment'
+}
+
+export const CONTENT_TYPE_DAG_JSON: ContentType = {
+  mediaType: MEDIA_TYPE_DAG_JSON,
+  codecs: [dagJsonCode, jsonCode, dagCborCode, CODEC_CBOR, dagPbCode, rawCode],
+  immutable: true,
+  extension: '.json',
+  etag: '.dag-json',
+  disposition: 'inline'
+}
+
+export const CONTENT_TYPE_JSON: ContentType = {
+  mediaType: MEDIA_TYPE_JSON,
+  codecs: [jsonCode, dagJsonCode, dagCborCode, CODEC_CBOR, dagPbCode, rawCode],
+  immutable: true,
+  extension: '.json',
+  etag: '.json',
+  disposition: 'inline'
+}
+
+export const CONTENT_TYPE_RAW: ContentType = {
+  mediaType: MEDIA_TYPE_RAW,
+  codecs: [rawCode, jsonCode, dagJsonCode, CODEC_CBOR, dagCborCode, dagPbCode],
+  immutable: true,
+  extension: '.raw',
+  etag: '.raw',
+  disposition: 'attachment'
+}
+
+export const CONTENT_TYPE_IPNS: ContentType = {
+  mediaType: MEDIA_TYPE_IPNS_RECORD,
+  codecs: [],
+  immutable: false,
+  extension: '.bin',
+  etag: '.bin',
+  disposition: 'attachment'
+}
+
+export const CONTENT_TYPE_CAR: ContentType = {
+  mediaType: MEDIA_TYPE_CAR,
+  codecs: [rawCode, jsonCode, dagJsonCode, CODEC_CBOR, dagCborCode, dagPbCode],
+  immutable: false,
+  extension: '.car',
+  etag: '.car',
+  disposition: 'attachment'
+}
+
+export const CONTENT_TYPE_TAR: ContentType = {
+  mediaType: MEDIA_TYPE_TAR,
+  codecs: [rawCode, dagPbCode],
+  immutable: false,
+  extension: '.x-tar',
+  etag: '.x-tar',
+  disposition: 'attachment'
+}
+
+export const CONTENT_TYPES: ContentType[] = [
+  CONTENT_TYPE_DAG_CBOR,
+  CONTENT_TYPE_CBOR,
+  CONTENT_TYPE_DAG_JSON,
+  CONTENT_TYPE_JSON,
+  CONTENT_TYPE_RAW,
+  CONTENT_TYPE_IPNS,
+  CONTENT_TYPE_CAR,
+  CONTENT_TYPE_TAR,
+  CONTENT_TYPE_OCTET_STREAM
+]
+
+export function getContentType (mediaType?: string | null): ContentType | undefined {
+  return CONTENT_TYPES
+    .find(m => m.mediaType === mediaType)
+}
+
+export function getContentTypesForCid (cid?: CID): ContentType[] {
+  if (cid == null) {
+    return []
+  }
+
+  return CONTENT_TYPES
+    .filter(m => m.codecs.includes(cid.code))
+    .sort((a, b) => {
+      // prefer content types where the CID code is earlier in the list of
+      // supported codecs
+      const posA = a.codecs.indexOf(cid.code)
+      const posB = b.codecs.indexOf(cid.code)
+
+      if (posA < posB) {
+        return -1
+      }
+
+      if (posB < posA) {
+        return 1
+      }
+
+      return 0
+    })
+}
+
+export function getSupportedContentTypes (protocol: string = 'ipfs:', cid?: CID): ContentType[] {
+  let contentTypes = getContentTypesForCid(cid)
+
+  if (protocol === 'ipfs:' && contentTypes.length === 0) {
+    contentTypes = [
+      CONTENT_TYPE_RAW,
+      CONTENT_TYPE_OCTET_STREAM
+    ]
+  }
+
+  if (protocol === 'ipns:') {
+    contentTypes.push(CONTENT_TYPE_IPNS)
+  }
+
+  return contentTypes
+}

package/src/utils/convert-output.ts

@@ -0,0 +1,187 @@
+import * as dagCbor from '@ipld/dag-cbor'
+import * as dagJson from '@ipld/dag-json'
+import * as dagPb from '@ipld/dag-pb'
+import * as json from 'multiformats/codecs/json'
+import * as raw from 'multiformats/codecs/raw'
+import { identity } from 'multiformats/hashes/identity'
+import { CODEC_CBOR } from '../constants.ts'
+import { getContentTypesForCid, MEDIA_TYPE_CBOR, MEDIA_TYPE_DAG_CBOR, MEDIA_TYPE_DAG_JSON, MEDIA_TYPE_JSON, MEDIA_TYPE_OCTET_STREAM, MEDIA_TYPE_RAW } from './content-types.ts'
+import type { AcceptHeader, ContentType } from '../index.ts'
+import type { CID } from 'multiformats/cid'
+
+const CONVERSIONS: Record<number, Record<string, (buf: Uint8Array) => Uint8Array>> = {
+  [dagCbor.code]: {
+    [MEDIA_TYPE_JSON]: (buf) => {
+      return dagJson.encode(dagCbor.decode(buf))
+    },
+    [MEDIA_TYPE_DAG_JSON]: (buf) => {
+      return dagJson.encode(dagCbor.decode(buf))
+    },
+    [MEDIA_TYPE_DAG_CBOR]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_CBOR]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_RAW]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_OCTET_STREAM]: (buf) => {
+      return buf
+    }
+  },
+  [CODEC_CBOR]: {
+    [MEDIA_TYPE_JSON]: (buf) => {
+      return dagJson.encode(dagCbor.decode(buf))
+    },
+    [MEDIA_TYPE_DAG_JSON]: (buf) => {
+      return dagJson.encode(dagCbor.decode(buf))
+    },
+    [MEDIA_TYPE_DAG_CBOR]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_CBOR]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_RAW]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_OCTET_STREAM]: (buf) => {
+      return buf
+    }
+  },
+  [dagJson.code]: {
+    [MEDIA_TYPE_CBOR]: (buf) => {
+      return dagCbor.encode(dagJson.decode(buf))
+    },
+    [MEDIA_TYPE_DAG_CBOR]: (buf) => {
+      return dagCbor.encode(dagJson.decode(buf))
+    },
+    [MEDIA_TYPE_DAG_JSON]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_JSON]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_RAW]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_OCTET_STREAM]: (buf) => {
+      return buf
+    }
+  },
+  [json.code]: {
+    [MEDIA_TYPE_CBOR]: (buf) => {
+      return dagCbor.encode(json.decode(buf))
+    },
+    [MEDIA_TYPE_DAG_CBOR]: (buf) => {
+      return dagCbor.encode(json.decode(buf))
+    },
+    [MEDIA_TYPE_DAG_JSON]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_JSON]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_RAW]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_OCTET_STREAM]: (buf) => {
+      return buf
+    }
+  },
+  [dagPb.code]: {
+    [MEDIA_TYPE_CBOR]: (buf) => {
+      return dagCbor.encode(dagPb.decode(buf))
+    },
+    [MEDIA_TYPE_DAG_CBOR]: (buf) => {
+      return dagCbor.encode(dagPb.decode(buf))
+    },
+    [MEDIA_TYPE_JSON]: (buf) => {
+      return dagJson.encode(dagPb.decode(buf))
+    },
+    [MEDIA_TYPE_DAG_JSON]: (buf) => {
+      return dagJson.encode(dagPb.decode(buf))
+    },
+    [MEDIA_TYPE_RAW]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_OCTET_STREAM]: (buf) => {
+      return buf
+    }
+  },
+  [raw.code]: {
+    [MEDIA_TYPE_CBOR]: (buf) => {
+      return dagCbor.encode(buf)
+    },
+    [MEDIA_TYPE_DAG_CBOR]: (buf) => {
+      return dagCbor.encode(buf)
+    },
+    [MEDIA_TYPE_JSON]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_DAG_JSON]: (buf) => {
+      return dagJson.encode(buf)
+    },
+    [MEDIA_TYPE_RAW]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_OCTET_STREAM]: (buf) => {
+      return buf
+    }
+  },
+  [identity.code]: {
+    [MEDIA_TYPE_CBOR]: (buf) => {
+      return dagCbor.encode(buf)
+    },
+    [MEDIA_TYPE_DAG_CBOR]: (buf) => {
+      return dagCbor.encode(buf)
+    },
+    [MEDIA_TYPE_JSON]: (buf) => {
+      return dagJson.encode(buf)
+    },
+    [MEDIA_TYPE_DAG_JSON]: (buf) => {
+      return dagJson.encode(buf)
+    },
+    [MEDIA_TYPE_RAW]: (buf) => {
+      return buf
+    },
+    [MEDIA_TYPE_OCTET_STREAM]: (buf) => {
+      return buf
+    }
+  }
+}
+
+export interface ConvertedOutput {
+  block: Uint8Array
+  contentType: string
+}
+
+/**
+ * Where supported, deserialize the passed block using the codec appropriate for
+ * the CID, then loop over the acceptable content types and attempt to convert
+ * the deserialized value to that format and serialize it back to a block -
+ * return the first successful result.
+ */
+export function convertOutput (cid: CID, block: Uint8Array, accept: AcceptHeader[]): { contentType: ContentType, output: Uint8Array } {
+  if (accept.length === 0) {
+    // return current format
+    return {
+      contentType: getContentTypesForCid(cid)[0],
+      output: block
+    }
+  }
+
+  for (const format of accept) {
+    const converter = CONVERSIONS[cid.code]?.[format.contentType.mediaType]
+
+    if (converter != null) {
+      return {
+        contentType: format.contentType,
+        output: converter(block)
+      }
+    }
+  }
+
+  throw new Error(`Could not convert ${cid} to any of ${accept.map(a => a.contentType.mediaType).join(', ')}`)
+}

package/src/utils/error-to-response.ts

@@ -0,0 +1,49 @@
+import { CONTENT_TYPE_OCTET_STREAM, CONTENT_TYPE_RAW } from './content-types.ts'
+import { badGatewayResponse, gatewayTimeoutResponse, internalServerErrorResponse, notAcceptableResponse, notFoundResponse, preconditionFailedResponse } from './responses.js'
+import type { Resource } from '../index.js'
+
+export function errorToResponse (resource: Resource | string, err: any): Response {
+  // if a signal abort caused the error, throw the error
+  if (err.name === 'AbortError') {
+    throw err
+  }
+
+  // could not reach an upstream server, bad connection or offline
+  if (err.code === 'ECONNREFUSED' || err.code === 'ECANCELLED' || err.name === 'DNSQueryFailedError') {
+    return badGatewayResponse(resource.toString(), err)
+  }
+
+  // data was not parseable, user may be able to request raw block
+  if (['NotUnixFSError'].includes(err.name)) {
+    return notAcceptableResponse(resource.toString(), [
+      CONTENT_TYPE_RAW,
+      CONTENT_TYPE_OCTET_STREAM
+    ])
+  }
+
+  // an upstream server didn't respond in time but inside the signal timeout
+  if (err.code === 'ETIMEOUT' || err.name === 'TimeoutError') {
+    return gatewayTimeoutResponse(resource.toString(), err)
+  }
+
+  // path was not under DAG root
+  if (['ERR_NO_PROP', 'ERR_NO_TERMINAL_ELEMENT', 'ERR_NOT_FOUND'].includes(err.code)) {
+    return notFoundResponse(resource.toString())
+  }
+
+  // path was not under DAG root
+  if (['DoesNotExistError'].includes(err.name)) {
+    return notFoundResponse(resource.toString())
+  }
+
+  if (['BlockNotFoundWhileOfflineError'].includes(err.name)) {
+    return preconditionFailedResponse(resource.toString())
+  }
+
+  if (['RecordNotFoundError'].includes('err.name')) {
+    return badGatewayResponse(resource.toString(), err)
+  }
+
+  // can't tell what went wrong, return a generic error
+  return internalServerErrorResponse(resource.toString(), err)
+}

package/src/utils/get-content-disposition-filename.ts

@@ -1,8 +1,14 @@
+import { InvalidParametersError } from '@libp2p/interface'
+
 /**
  * Takes a filename URL param and returns a string for use in a
  * `Content-Disposition` header
  */
-export function getContentDispositionFilename (filename: string): string {
+export function getContentDispositionFilename (filename: string | null): string {
+  if (filename == null) {
+    throw new InvalidParametersError('Cannot get filename for Content-Disposition header - filename argument was missing')
+  }
+
   const asciiOnly = replaceNonAsciiCharacters(filename)
 
   if (asciiOnly === filename) {

package/src/utils/get-e-tag.ts

@@ -1,56 +1,47 @@
-import type { RequestFormatShorthand } from '../index.js'
+import type { ContentType } from '../index.ts'
 import type { CID } from 'multiformats/cid'
 
 interface GetETagArg {
-  cid: CID
-  reqFormat?: RequestFormatShorthand
-  rangeStart?: number
-  rangeEnd?: number
   /**
-   * Weak Etag is used when we can't guarantee byte-for-byte-determinism (generated, or mutable content).
-   * Some examples:
-   * - IPNS requests
-   * - CAR streamed with blocks in non-deterministic order
-   * - TAR streamed with files in non-deterministic order
+   * The CID that was resolved to the content in the response
    */
-  weak?: boolean
+  cid: CID
 
   /**
-   * A custom prefix to use for the content of the etag. This is needed for some cases (like dir-index-html) where we need to use a custom prefix for the etag.
+   * The content type of the response
    */
-  contentPrefix?: string
-}
+  contentType: ContentType
 
-const getPrefix = ({ weak, reqFormat }: Partial<GetETagArg>): string => {
-  if (reqFormat === 'tar' || reqFormat === 'car' || reqFormat === 'ipns-record' || weak === true) {
-    return 'W/'
-  }
-  return ''
-}
+  /**
+   * Where the byte range starts, if specified
+   */
+  rangeStart?: number
 
-const getFormatSuffix = ({ reqFormat }: Partial<GetETagArg>): string => {
-  if (reqFormat == null) {
-    return ''
-  }
-  if (reqFormat === 'tar') {
-    return '.x-tar'
-  }
+  /**
+   * Where the byte range ends, if specified
+   */
+  rangeEnd?: number
 
-  return `.${reqFormat}`
+  /**
+   * A custom prefix to use for the content of the etag
+   */
+  contentPrefix?: string
 }
 
 /**
- * etag
- * you need to wrap cid  with ""
- * we use strong Etags for immutable responses and weak one (prefixed with W/ ) for mutable/generated ones (ipns, car, tar, and generated HTML).
- * block and car responses should have different etag than deserialized one, so you can add some prefix like we do in existing gateway
+ * We use strong Etags for immutable responses and weak ones (prefixed with W/ )
+ * for mutable/generated ones (ipns, car, tar, and generated HTML).
+ *
+ * Block and car responses should have different etags to deserialized ones, so
+ * you can add a prefix like we do in the existing gateway.
  *
  * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
  * @see https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
  */
-export function getETag ({ cid, reqFormat, weak, rangeStart, rangeEnd, contentPrefix }: GetETagArg): string {
-  const prefix = getPrefix({ weak, reqFormat })
-  let suffix = getFormatSuffix({ reqFormat })
+export function getETag ({ cid, contentType, rangeStart, rangeEnd, contentPrefix }: GetETagArg): string {
+  const prefix = contentType.immutable ? '' : 'W/'
+  let suffix = contentType.etag
+
   if (rangeStart != null || rangeEnd != null) {
     suffix += `.${rangeStart ?? '0'}-${rangeEnd ?? 'N'}`
   }

package/src/utils/get-offset-and-length.ts

@@ -1,25 +1,79 @@
-import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
+import { InvalidRangeError } from '../errors.ts'
 
-export function getOffsetAndLength (entry: UnixFSEntry, entityBytes?: string): { offset: number, length: number } {
-  if (entityBytes == null) {
-    return {
-      offset: 0,
-      length: Infinity
+interface OffsetLength<T = number> {
+  offset: T
+  length: T
+}
+
+function downCast ({ offset, length }: OffsetLength<bigint>): OffsetLength {
+  return {
+    offset: Number(offset),
+    length: Number(length)
+  }
+}
+
+/**
+ * N.b. Range start (zero indexed, inclusive) and end (inclusive) - this is
+ * different to `Uint8Array.subarray` which is end exclusive.
+ */
+export function rangeToOffsetAndLength (size: bigint | number, start?: number | bigint, end?: number | bigint): { offset: number, length: number } {
+  size = BigInt(size)
+
+  if (start != null) {
+    start = BigInt(start)
+
+    if (start < 0n) {
+      throw new InvalidRangeError('Range start cannot be negative')
+    }
+
+    if (start >= size) {
+      throw new InvalidRangeError('Range start cannot be larger than total bytes')
     }
   }
 
-  const parts = entityBytes.split(':')
-  const start = parseInt(parts[0], 10)
-  const end = parts[1] === '*' ? Infinity : parseInt(parts[1], 10)
+  if (end != null) {
+    end = BigInt(end)
 
-  if (isNaN(start) || isNaN(end)) {
-    throw new Error('Could not parse entity-bytes')
+    const abs = end > 0n ? end : -end
+
+    if (abs >= size) {
+      throw new InvalidRangeError('Range end cannot be larger than total bytes')
+    }
+
+    if (end < 0n) {
+      if (start == null) {
+        start = size + end
+        end = size - 1n
+      } else {
+        throw new InvalidRangeError('Range end cannot be negative')
+      }
+    }
+
+    // make inclusive end exclusive
+    end += 1n
   }
 
-  const entrySize = Number(entry.size)
+  return downCast(toOffsetAndLength(size, start, end))
+}
+
+/**
+ * Translate `from:to` zero-indexed inclusive bytes to offset/length
+ */
+export function entityBytesToOffsetAndLength (size: number | bigint, entityBytes?: string | null): { offset: number, length: number } {
+  size = BigInt(size)
+  const parts = (entityBytes ?? '0:*').split(':')
+  const start = BigInt(parts[0])
+  const end = parts[1] === '*' ? size : BigInt(parts[1])
+
+  return downCast(toOffsetAndLength(size, start, end))
+}
 
-  if (start >= 0) {
-    if (end >= 0) {
+/**
+ * Translate `from:to` zero-indexed inclusive bytes to offset/length
+ */
+function toOffsetAndLength (size: bigint, start: bigint = 0n, end: bigint = size): { offset: bigint, length: bigint } {
+  if (start >= 0n) {
+    if (end >= 0n) {
       return {
         offset: start,
         length: end - start
@@ -27,28 +81,28 @@ export function getOffsetAndLength (entry: UnixFSEntry, entityBytes?: string): {
     } else {
       return {
         offset: start,
-        length: (entrySize - start) + end
+        length: (size - start) + end
       }
     }
   }
 
   // start < 0
-  let offset = entrySize + start
+  let offset = size + start
 
-  if (Math.abs(start) > entrySize) {
-    offset = 0
+  if (Math.abs(Number(start)) > Number(size)) {
+    offset = 0n
   }
 
-  if (end >= 0) {
+  if (end >= 0n) {
     return {
       offset,
-      length: (entrySize - offset) + end
+      length: end - offset
     }
   }
 
   // end < 0
   return {
     offset,
-    length: (entrySize - offset) + end
+    length: (size - offset) + end
   }
 }