@atproto-labs/fetch 0.0.1 → 0.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto-labs/fetch",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "license": "MIT",
   "description": "Isomorphic wrapper utilities for fetch API",
   "keywords": [
@@ -11,7 +11,7 @@
   "repository": {
     "type": "git",
     "url": "https://github.com/bluesky-social/atproto",
-    "directory": "packages/fetch"
+    "directory": "packages/internal/fetch"
   },
   "type": "commonjs",
   "main": "dist/index.js",
@@ -23,13 +23,14 @@
     }
   },
   "dependencies": {
-    "tslib": "^2.6.2",
-    "zod": "^3.22.4",
-    "@atproto-labs/transformer": "0.0.1"
+    "@atproto-labs/pipe": "0.1.0"
   },
   "devDependencies": {
     "typescript": "^5.3.3"
   },
+  "optionalDependencies": {
+    "zod": "^3.23.8"
+  },
   "scripts": {
     "build": "tsc --build tsconfig.json"
   }

package/src/fetch-error.ts

@@ -1,44 +1,26 @@
-import { Transformer } from '@atproto-labs/transformer'
-
-export type FetchErrorOptions = {
-  cause?: unknown
-  request?: Request
-  response?: Response
-}
-
 export class FetchError extends Error {
-  public readonly request?: Request
-  public readonly response?: Response
+  public readonly statusCode: number
 
-  constructor(
-    public readonly statusCode: number,
-    message?: string,
-    { cause, request, response }: FetchErrorOptions = {},
-  ) {
-    super(message, { cause })
-    this.request = request
-    this.response = response
-  }
+  constructor(statusCode?: number, message?: string, options?: ErrorOptions) {
+    if (statusCode == null || !message) {
+      const info = extractInfo(extractRootCause(options?.cause))
+      statusCode = statusCode ?? info[0]
+      message = message || info[1]
+    }
 
-  static async from(err: unknown) {
-    const cause = extractCause(err)
-    return new FetchError(...extractInfo(cause), { cause })
-  }
-}
+    super(message, options)
 
-export const fetchFailureHandler: Transformer<unknown, never> = async (
-  err: unknown,
-) => {
-  throw await FetchError.from(err)
+    this.statusCode = statusCode
+  }
 }
 
-function extractCause(err: unknown): unknown {
+function extractRootCause(err: unknown): unknown {
   // Unwrap the Network error from undici (i.e. Node's internal fetch() implementation)
   // https://github.com/nodejs/undici/blob/3274c975947ce11a08508743df026f73598bfead/lib/web/fetch/index.js#L223-L228
   if (
     err instanceof TypeError &&
     err.message === 'fetch failed' &&
-    err.cause instanceof Error
+    err.cause !== undefined
   ) {
     return err.cause
   }
@@ -46,32 +28,32 @@ function extractCause(err: unknown): unknown {
   return err
 }
 
-export function extractInfo(
-  err: unknown,
-): [statusCode: number, message: string] {
+function extractInfo(err: unknown): [statusCode: number, message: string] {
   if (typeof err === 'string' && err.length > 0) {
-    return [502, err]
+    return [500, err]
   }
 
   if (!(err instanceof Error)) {
-    return [502, 'Unable to fetch']
+    return [500, 'Failed to fetch']
   }
 
-  if ('code' in err && typeof err.code === 'string') {
+  const code = err['code']
+  if (typeof code === 'string') {
     switch (true) {
-      case err.code === 'ENOTFOUND':
-        return [404, 'Invalid hostname']
-      case err.code === 'ECONNREFUSED':
+      case code === 'ENOTFOUND':
+        return [400, 'Invalid hostname']
+      case code === 'ECONNREFUSED':
         return [502, 'Connection refused']
-      case err.code === 'DEPTH_ZERO_SELF_SIGNED_CERT':
+      case code === 'DEPTH_ZERO_SELF_SIGNED_CERT':
         return [502, 'Self-signed certificate']
-      case err.code.startsWith('ERR_TLS'):
+      case code.startsWith('ERR_TLS'):
         return [502, 'TLS error']
-      case err.code.startsWith('ECONN'):
+      case code.startsWith('ECONN'):
         return [502, 'Connection error']
+      default:
+        return [500, `${code} error`]
     }
   }
 
-  // Let's assume that other errors are "bad gateway" errors
-  return [502, err.message]
+  return [500, err.message]
 }

package/src/fetch-request.ts

@@ -1,40 +1,70 @@
-import { Transformer } from '@atproto-labs/transformer'
-
 import { FetchError } from './fetch-error.js'
+import { asRequest } from './fetch.js'
 import { isIp } from './util.js'
 
-export type RequestTranformer = Transformer<Request>
+export class FetchRequestError extends FetchError {
+  constructor(
+    public readonly request: Request,
+    statusCode?: number,
+    message?: string,
+    options?: ErrorOptions,
+  ) {
+    super(statusCode, message, options)
+  }
+
+  static from(request: Request, cause: unknown): FetchRequestError {
+    if (cause instanceof FetchRequestError) return cause
+    return new FetchRequestError(request, undefined, undefined, { cause })
+  }
+}
 
-export function protocolCheckRequestTransform(
-  protocols: Iterable<string>,
-): RequestTranformer {
+const extractUrl = (input: Request | string | URL) =>
+  typeof input === 'string'
+    ? new URL(input)
+    : input instanceof URL
+      ? input
+      : new URL(input.url)
+
+export function protocolCheckRequestTransform(protocols: Iterable<string>) {
   const allowedProtocols = new Set<string>(protocols)
 
-  return async (request) => {
-    const { protocol } = new URL(request.url)
+  return (input: Request | string | URL, init?: RequestInit) => {
+    const { protocol } = extractUrl(input)
+
+    const request = asRequest(input, init)
 
     if (!allowedProtocols.has(protocol)) {
-      throw new FetchError(400, `${protocol} is not allowed`, { request })
+      throw new FetchRequestError(
+        request,
+        400,
+        `"${protocol}" protocol is not allowed`,
+      )
     }
 
     return request
   }
 }
 
-export function requireHostHeaderTranform(): RequestTranformer {
-  return async (request) => {
+export function requireHostHeaderTranform() {
+  return (input: Request | string | URL, init?: RequestInit) => {
     // Note that fetch() will automatically add the Host header from the URL and
     // discard any Host header manually set in the request.
 
-    const { protocol, hostname } = new URL(request.url)
+    const { protocol, hostname } = extractUrl(input)
+
+    const request = asRequest(input, init)
 
     // "Host" header only makes sense in the context of an HTTP request
-    if (protocol !== 'http:' && protocol !== 'https') {
-      throw new FetchError(400, `Forbidden protocol ${protocol}`, { request })
+    if (protocol !== 'http:' && protocol !== 'https:') {
+      throw new FetchRequestError(
+        request,
+        400,
+        `"${protocol}" requests are not allowed`,
+      )
     }
 
     if (!hostname || isIp(hostname)) {
-      throw new FetchError(400, 'Invalid hostname', { request })
+      throw new FetchRequestError(request, 400, 'Invalid hostname')
     }
 
     return request
@@ -54,7 +84,7 @@ export const DEFAULT_FORBIDDEN_DOMAIN_NAMES = [
 
 export function forbiddenDomainNameRequestTransform(
   denyList: Iterable<string> = DEFAULT_FORBIDDEN_DOMAIN_NAMES,
-): RequestTranformer {
+) {
   const denySet = new Set<string>(denyList)
 
   // Optimization: if no forbidden domain names are provided, we can skip the
@@ -63,12 +93,14 @@ export function forbiddenDomainNameRequestTransform(
     return async (request) => request
   }
 
-  return async (request) => {
-    const { hostname } = new URL(request.url)
+  return async (input: Request | string | URL, init?: RequestInit) => {
+    const { hostname } = extractUrl(input)
+
+    const request = asRequest(input, init)
 
     // Full domain name check
     if (denySet.has(hostname)) {
-      throw new FetchError(403, 'Forbidden hostname', { request })
+      throw new FetchRequestError(request, 403, 'Forbidden hostname')
     }
 
     // Sub domain name check
@@ -76,7 +108,7 @@ export function forbiddenDomainNameRequestTransform(
     while (curDot !== -1) {
       const subdomain = hostname.slice(curDot + 1)
       if (denySet.has(`*.${subdomain}`)) {
-        throw new FetchError(403, 'Forbidden hostname', { request })
+        throw new FetchRequestError(request, 403, 'Forbidden hostname')
       }
       curDot = hostname.indexOf('.', curDot + 1)
     }

package/src/fetch-response.ts

@@ -1,27 +1,58 @@
-import { Transformer, compose } from '@atproto-labs/transformer'
-import { z } from 'zod'
+import { Transformer, pipe } from '@atproto-labs/pipe'
 
-import { FetchError, FetchErrorOptions } from './fetch-error.js'
-import { Json, ifObject, ifString } from './util.js'
+// optional dependency for typing purposes
+import type { ZodTypeAny, ParseParams, TypeOf } from 'zod'
+
+import { FetchError } from './fetch-error.js'
 import { TransformedResponse } from './transformed-response.js'
+import {
+  Json,
+  MaxBytesTransformStream,
+  cancelBody,
+  ifObject,
+  ifString,
+  logCancellationError,
+} from './util.js'
 
 export type ResponseTranformer = Transformer<Response>
 export type ResponseMessageGetter = Transformer<Response, string | undefined>
 
-const extractResponseMessage: ResponseMessageGetter = async (response) => {
-  if (!response.body) return undefined
+export class FetchResponseError extends FetchError {
+  constructor(
+    public readonly response: Response,
+    statusCode: number = response.status,
+    message: string = response.statusText,
+    options?: ErrorOptions,
+  ) {
+    super(statusCode, message, options)
+  }
 
-  const contentType = response.headers.get('content-type')
-  if (!contentType) return undefined
+  static async from(
+    response: Response,
+    customMessage: string | ResponseMessageGetter = extractResponseMessage,
+    statusCode = response.status,
+    options?: ErrorOptions,
+  ) {
+    const message =
+      typeof customMessage === 'string'
+        ? customMessage
+        : typeof customMessage === 'function'
+          ? await customMessage(response)
+          : undefined
+
+    return new FetchResponseError(response, statusCode, message, options)
+  }
+}
 
-  const mimeType = contentType.split(';')[0].trim()
+const extractResponseMessage: ResponseMessageGetter = async (response) => {
+  const mimeType = extractMime(response)
   if (!mimeType) return undefined
 
   try {
     if (mimeType === 'text/plain') {
       return await response.text()
     } else if (/^application\/(?:[^+]+\+)?json$/i.test(mimeType)) {
-      const json = await response.json()
+      const json: unknown = await response.json()
 
       if (typeof json === 'string') return json
 
@@ -41,127 +72,170 @@ const extractResponseMessage: ResponseMessageGetter = async (response) => {
   return undefined
 }
 
-export class FetchResponseError extends FetchError {
-  constructor(
-    response: Response,
-    statusCode: number = response.status,
-    message: string = response.statusText,
-    options?: Omit<FetchErrorOptions, 'response'>,
-  ) {
-    super(statusCode, message, { response, ...options })
+export async function peekJson(
+  response: Response,
+  maxSize = Infinity,
+): Promise<undefined | Json> {
+  const type = extractMime(response)
+  if (type !== 'application/json') return undefined
+  checkLength(response, maxSize)
+
+  // 1) Clone the request so we can consume the body
+  const clonedResponse = response.clone()
+
+  // 2) Make sure the request's body is not too large
+  const limitedResponse =
+    response.body && maxSize < Infinity
+      ? new TransformedResponse(
+          clonedResponse,
+          new MaxBytesTransformStream(maxSize),
+        )
+      : // Note: some runtimes (e.g. react-native) don't expose a body property
+        clonedResponse
+
+  // 3) Parse the JSON
+  return limitedResponse.json()
+}
+
+export function checkLength(response: Response, maxBytes: number) {
+  // Note: negation accounts for invalid value types (NaN, non numbers)
+  if (!(maxBytes >= 0)) {
+    throw new TypeError('maxBytes must be a non-negative number')
+  }
+  const length = extractLength(response)
+  if (length != null && length > maxBytes) {
+    throw new FetchResponseError(response, 502, 'Response too large')
   }
+  return length
+}
 
-  static async from(
-    response: Response,
-    statusCode = response.status,
-    customMessage: string | ResponseMessageGetter = extractResponseMessage,
-    options?: Omit<FetchErrorOptions, 'response'>,
-  ) {
-    const message =
-      typeof customMessage === 'string'
-        ? customMessage
-        : typeof customMessage === 'function'
-          ? await customMessage(response)
-          : undefined
+export function extractLength(response: Response) {
+  const contentLength = response.headers.get('Content-Length')
+  if (contentLength == null) return undefined
+  if (!/^\d+$/.test(contentLength)) {
+    throw new FetchResponseError(response, 502, 'Invalid Content-Length')
+  }
+  const length = Number(contentLength)
+  if (!Number.isSafeInteger(length)) {
+    throw new FetchResponseError(response, 502, 'Content-Length too large')
+  }
+  return length
+}
 
-    // Make sure the body gets consumed as, in some environments (Node 👀), the
-    // response will not automatically be GC'd.
-    if (!response.bodyUsed) await response.body?.cancel()
+export function extractMime(response: Response) {
+  const contentType = response.headers.get('Content-Type')
+  if (contentType == null) return undefined
 
-    return new FetchResponseError(response, statusCode, message, options)
+  return contentType.split(';', 1)[0]!.trim()
+}
+
+/**
+ * If the transformer results in an error, ensure that the response body is
+ * consumed as, in some environments (Node 👀), the response will not
+ * automatically be GC'd.
+ *
+ * @see {@link https://undici.nodejs.org/#/?id=garbage-collection}
+ * @param [onCancellationError] - Callback to handle any async body cancelling
+ * error. Defaults to logging the error. Do not use `null` if the request is
+ * cloned.
+ */
+export function cancelBodyOnError<T>(
+  transformer: Transformer<Response, T>,
+  onCancellationError: null | ((err: unknown) => void) = logCancellationError,
+): (response: Response) => Promise<T> {
+  return async (response) => {
+    try {
+      return await transformer(response)
+    } catch (err) {
+      await cancelBody(response, onCancellationError ?? undefined)
+      throw err
+    }
   }
 }
 
 export function fetchOkProcessor(
   customMessage?: string | ResponseMessageGetter,
 ): ResponseTranformer {
-  return async (response) => {
-    if (response.ok) return response
-    throw await FetchResponseError.from(response, undefined, customMessage)
-  }
+  return cancelBodyOnError((response) => {
+    return fetchOkTransformer(response, customMessage)
+  })
+}
+
+export async function fetchOkTransformer(
+  response: Response,
+  customMessage?: string | ResponseMessageGetter,
+) {
+  if (response.ok) return response
+  throw await FetchResponseError.from(response, customMessage)
 }
 
 export function fetchMaxSizeProcessor(maxBytes: number): ResponseTranformer {
   if (maxBytes === Infinity) return (response) => response
   if (!Number.isFinite(maxBytes) || maxBytes < 0) {
-    throw new TypeError('maxBytes must be a non-negative number')
+    throw new TypeError('maxBytes must be a 0, Infinity or a positive number')
   }
-  return async (response) => fetchResponseMaxSize(response, maxBytes)
+  return cancelBodyOnError((response) => {
+    return fetchResponseMaxSizeChecker(response, maxBytes)
+  })
 }
 
-export async function fetchResponseMaxSize(
+export function fetchResponseMaxSizeChecker(
   response: Response,
   maxBytes: number,
-): Promise<Response> {
+): Response {
   if (maxBytes === Infinity) return response
-  if (!response.body) return response
+  checkLength(response, maxBytes)
 
-  const contentLength = response.headers.get('content-length')
-  if (contentLength) {
-    const length = Number(contentLength)
-    if (!(length < maxBytes)) {
-      const err = new FetchResponseError(response, 502, 'Response too large')
-      await response.body.cancel(err)
-      throw err
-    }
-  }
-
-  let bytesRead = 0
-
-  const transform = new TransformStream<Uint8Array, Uint8Array>({
-    transform: (
-      chunk: Uint8Array,
-      ctrl: TransformStreamDefaultController<Uint8Array>,
-    ) => {
-      if ((bytesRead += chunk.length) <= maxBytes) {
-        ctrl.enqueue(chunk)
-      } else {
-        ctrl.error(new FetchResponseError(response, 502, 'Response too large'))
-      }
-    },
-  })
+  // Some engines (react-native 👀) don't expose a body property. In that case,
+  // we will only rely on the Content-Length header.
+  if (!response.body) return response
 
+  const transform = new MaxBytesTransformStream(maxBytes)
   return new TransformedResponse(response, transform)
 }
 
-export type ContentTypeCheckFn = (contentType: string) => boolean
-export type ContentTypeCheck = string | RegExp | ContentTypeCheckFn
+export type MimeTypeCheckFn = (mimeType: string) => boolean
+export type MimeTypeCheck = string | RegExp | MimeTypeCheckFn
 
 export function fetchTypeProcessor(
-  expectedType: ContentTypeCheck,
+  expectedMime: MimeTypeCheck,
   contentTypeRequired = true,
 ): ResponseTranformer {
-  const isExpected: ContentTypeCheckFn =
-    typeof expectedType === 'string'
-      ? (ct) => ct === expectedType
-      : expectedType instanceof RegExp
-        ? (ct) => expectedType.test(ct)
-        : expectedType
+  const isExpected: MimeTypeCheckFn =
+    typeof expectedMime === 'string'
+      ? (mimeType) => mimeType === expectedMime
+      : expectedMime instanceof RegExp
+        ? (mimeType) => expectedMime.test(mimeType)
+        : expectedMime
+
+  return cancelBodyOnError((response) => {
+    return fetchResponseTypeChecker(response, isExpected, contentTypeRequired)
+  })
+}
 
-  return async (response) => {
-    const contentType = response.headers
-      .get('content-type')
-      ?.split(';')[0]!
-      .trim()
-
-    if (contentType) {
-      if (!isExpected(contentType)) {
-        throw await FetchResponseError.from(
-          response,
-          502,
-          `Unexpected response Content-Type (${contentType})`,
-        )
-      }
-    } else if (contentTypeRequired) {
+export async function fetchResponseTypeChecker(
+  response: Response,
+  isExpectedMime: MimeTypeCheckFn,
+  contentTypeRequired = true,
+): Promise<Response> {
+  const mimeType = extractMime(response)
+  if (mimeType) {
+    if (!isExpectedMime(mimeType)) {
       throw await FetchResponseError.from(
         response,
+        `Unexpected response Content-Type (${mimeType})`,
         502,
-        'Missing response Content-Type header',
       )
     }
-
-    return response
+  } else if (contentTypeRequired) {
+    throw await FetchResponseError.from(
+      response,
+      'Missing response Content-Type header',
+      502,
+    )
   }
+
+  return response
 }
 
 export type ParsedJsonResponse<T = Json> = {
@@ -169,17 +243,9 @@ export type ParsedJsonResponse<T = Json> = {
   json: T
 }
 
-export async function jsonTranformer<T = Json>(
+export async function fetchResponseJsonTranformer<T = Json>(
   response: Response,
 ): Promise<ParsedJsonResponse<T>> {
-  if (response.body === null) {
-    throw new FetchResponseError(response, 502, 'No response body')
-  }
-
-  if (response.bodyUsed) {
-    throw new FetchResponseError(response, 500, 'Response body already used')
-  }
-
   try {
     const json = (await response.json()) as T
     return { response, json }
@@ -194,19 +260,19 @@ export async function jsonTranformer<T = Json>(
 }
 
 export function fetchJsonProcessor<T = Json>(
-  contentType: ContentTypeCheck = /^application\/(?:[^+]+\+)?json$/,
+  expectedMime: MimeTypeCheck = /^application\/(?:[^+]+\+)?json$/,
   contentTypeRequired = true,
 ): Transformer<Response, ParsedJsonResponse<T>> {
-  return compose(
-    fetchTypeProcessor(contentType, contentTypeRequired),
-    jsonTranformer<T>,
+  return pipe(
+    fetchTypeProcessor(expectedMime, contentTypeRequired),
+    cancelBodyOnError(fetchResponseJsonTranformer<T>),
   )
 }
 
-export function fetchJsonZodProcessor<S extends z.ZodTypeAny>(
+export function fetchJsonZodProcessor<S extends ZodTypeAny>(
   schema: S,
-  params?: Partial<z.ParseParams>,
-): Transformer<ParsedJsonResponse, z.infer<S>> {
-  return async (jsonResponse: ParsedJsonResponse): Promise<z.infer<S>> =>
+  params?: Partial<ParseParams>,
+): Transformer<ParsedJsonResponse, TypeOf<S>> {
+  return async (jsonResponse: ParsedJsonResponse): Promise<TypeOf<S>> =>
     schema.parseAsync(jsonResponse.json, params)
 }