@startsimpli/integrations 0.2.34 → 0.2.36

package/src/safe-fetch/index.ts

@@ -0,0 +1,387 @@
+/**
+ * safeFetch — server-side fetch of a USER-SUPPLIED url with SSRF guards
+ * (crochet_patterns-llmj: "paste a link to a pattern").
+ *
+ * SERVER-ONLY. Exported from the `@startsimpli/integrations/safe-fetch`
+ * subpath, never from the package root (client components import the root, and
+ * the default resolver is `node:dns`).
+ *
+ * Contract:
+ * - only http/https urls, no `user:pass@`;
+ * - every hop's host is resolved and EVERY address is checked
+ *   (`isBlockedAddress`); redirects are followed by hand (`redirect: 'manual'`)
+ *   so each hop is re-checked; at most `maxRedirects` (default 3);
+ * - only `allowTypes` content types (default text/html, application/pdf);
+ * - aborts over `maxBytes` (header or streamed) and after `timeoutMs`.
+ *
+ * DNS pinning (crochet_patterns-yfj8 #1, design D5): when `fetch` is not
+ * injected, requests go through npm `undici` with an Agent whose
+ * `connect.lookup` is `pinnedLookup(lookup)`. The check runs inside the lookup
+ * the socket uses, so the address that is checked is the address that is
+ * connected to — a record that flips to a private IP between the pre-flight
+ * check and connect is refused at connect.
+ *
+ * `lookup` and `fetch` are injectable so tests never touch the network.
+ */
+import { lookup as dnsLookup } from 'node:dns/promises';
+import { isIP, type LookupFunction } from 'node:net';
+import { Agent, fetch as undiciFetch } from 'undici';
+
+export type SafeFetchErrorCode =
+  | 'BAD_URL'
+  | 'BLOCKED_ADDRESS'
+  | 'TOO_MANY_REDIRECTS'
+  | 'UNSUPPORTED_TYPE'
+  | 'TOO_LARGE'
+  | 'TIMEOUT'
+  | 'HTTP_ERROR';
+
+export class SafeFetchError extends Error {
+  constructor(
+    message: string,
+    public readonly code: SafeFetchErrorCode,
+    /** The hop that was refused, when there is one. */
+    public readonly url?: string,
+  ) {
+    super(message);
+    this.name = 'SafeFetchError';
+  }
+}
+
+export interface ResolvedAddress {
+  address: string;
+  family: 4 | 6;
+}
+
+export interface SafeFetchOptions {
+  /** Allowed content types (parameters like `; charset=` are ignored). Default text/html + application/pdf. */
+  allowTypes?: string[];
+  /** Byte cap on the body. Default 5 MB. */
+  maxBytes?: number;
+  /** Redirect hops allowed. Default 3. */
+  maxRedirects?: number;
+  /** Whole-request time cap. Default 10 s. */
+  timeoutMs?: number;
+  /** Resolves a hostname to ALL its addresses. Default: node:dns lookup({all:true}). */
+  lookup?: (hostname: string) => Promise<ResolvedAddress[]>;
+  /** Default: npm undici fetch with a DNS-pinning Agent (see `pinnedLookup`). */
+  fetch?: typeof fetch;
+}
+
+export interface SafeFetchResult {
+  /** The url that finally answered (after redirects). */
+  finalUrl: string;
+  status: number;
+  /** Bare media type, e.g. `text/html`. */
+  contentType: string;
+  body: Uint8Array;
+}
+
+export const SAFE_FETCH_DEFAULTS = {
+  allowTypes: ['text/html', 'application/pdf'],
+  maxBytes: 5 * 1024 * 1024,
+  maxRedirects: 3,
+  timeoutMs: 10_000,
+} as const;
+
+// ---------------------------------------------------------------------------
+// Address blocklist
+// ---------------------------------------------------------------------------
+
+function parseV4(addr: string): number[] | null {
+  const parts = addr.split('.');
+  if (parts.length !== 4) return null;
+  const bytes = parts.map((p) => (/^\d{1,3}$/.test(p) ? Number(p) : NaN));
+  return bytes.every((b) => b >= 0 && b <= 255) ? bytes : null;
+}
+
+function parseV6(addr: string): number[] | null {
+  let s = addr.toLowerCase();
+  const zone = s.indexOf('%');
+  if (zone !== -1) s = s.slice(0, zone);
+  // A trailing dotted quad (::ffff:1.2.3.4) becomes two hextets.
+  const lastColon = s.lastIndexOf(':');
+  const tail = s.slice(lastColon + 1);
+  if (tail.includes('.')) {
+    const v4 = parseV4(tail);
+    if (!v4) return null;
+    s = `${s.slice(0, lastColon + 1)}${((v4[0]! << 8) | v4[1]!).toString(16)}:${((v4[2]! << 8) | v4[3]!).toString(16)}`;
+  }
+  const halves = s.split('::');
+  if (halves.length > 2) return null;
+  const head = halves[0] ? halves[0].split(':') : [];
+  const rest = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
+  const missing = 8 - head.length - rest.length;
+  if (halves.length === 1 ? missing !== 0 : missing < 1) return null;
+  const hextets = [...head, ...Array<string>(halves.length === 2 ? missing : 0).fill('0'), ...rest];
+  const bytes: number[] = [];
+  for (const h of hextets) {
+    if (!/^[0-9a-f]{1,4}$/.test(h)) return null;
+    const n = parseInt(h, 16);
+    bytes.push(n >> 8, n & 0xff);
+  }
+  return bytes;
+}
+
+function isBlockedV4([a, b, c]: number[]): boolean {
+  return (
+    a === 0 || // 0/8 "this network", incl. 0.0.0.0
+    a === 10 || // 10/8
+    a === 127 || // loopback
+    (a === 100 && b! >= 64 && b! <= 127) || // 100.64/10 CGNAT
+    (a === 169 && b === 254) || // link-local, incl. cloud metadata 169.254.169.254
+    (a === 172 && b! >= 16 && b! <= 31) || // 172.16/12
+    (a === 192 && b === 0 && c === 0) || // 192.0.0/24 IETF protocol assignments
+    (a === 192 && b === 168) || // 192.168/16
+    (a === 198 && (b === 18 || b === 19)) || // 198.18/15 benchmarking
+    a! >= 224 // 224/4 multicast, 240/4 reserved, 255.255.255.255
+  );
+}
+
+function startsWith(bytes: number[], prefix: number[]): boolean {
+  return prefix.every((b, i) => bytes[i] === b);
+}
+
+function isBlockedV6(bytes: number[]): boolean {
+  const first = bytes[0]!;
+  if ((first & 0xfe) === 0xfc) return true; // fc00::/7 unique local
+  if (first === 0xfe && (bytes[1]! & 0xc0) === 0x80) return true; // fe80::/10 link-local
+  if (first === 0xfe && (bytes[1]! & 0xc0) === 0xc0) return true; // fec0::/10 site-local (deprecated)
+  if (first === 0xff) return true; // ff00::/8 multicast
+  const zero = (n: number) => Array<number>(n).fill(0);
+  // ::/96 — :: and ::1 and the deprecated v4-compatible form; checked as the embedded v4
+  // (::  -> 0.0.0.0, ::1 -> 0.0.0.1, both in 0/8).
+  if (startsWith(bytes, zero(12))) return isBlockedV4(bytes.slice(12));
+  // ::ffff:0:0/96 v4-mapped
+  if (startsWith(bytes, [...zero(10), 0xff, 0xff])) return isBlockedV4(bytes.slice(12));
+  // 64:ff9b::/96 NAT64 well-known prefix
+  if (startsWith(bytes, [0x00, 0x64, 0xff, 0x9b, ...zero(8)])) return isBlockedV4(bytes.slice(12));
+  // 64:ff9b:1::/48 local-use NAT64 — translates to whatever the local network chooses
+  if (startsWith(bytes, [0x00, 0x64, 0xff, 0x9b, 0x00, 0x01])) return true;
+  // 2002::/16 6to4 — the v4 is bytes 2..5
+  if (bytes[0] === 0x20 && bytes[1] === 0x02) return isBlockedV4(bytes.slice(2, 6));
+  return false;
+}
+
+/**
+ * True when a connection to `addr` must be refused: loopback, private,
+ * link-local (cloud metadata), CGNAT, benchmarking, multicast, reserved, the
+ * unspecified address, IPv6 loopback/ULA/link-local/multicast, and any IPv6
+ * form that embeds a blocked IPv4 (mapped, compatible, NAT64, 6to4).
+ * Anything that does not parse as an IP is refused (fail closed).
+ */
+export function isBlockedAddress(addr: string): boolean {
+  const s = addr.startsWith('[') && addr.endsWith(']') ? addr.slice(1, -1) : addr;
+  const family = isIP(s.split('%')[0]!);
+  if (family === 4) {
+    const v4 = parseV4(s);
+    return !v4 || isBlockedV4(v4);
+  }
+  if (family === 6) {
+    const v6 = parseV6(s);
+    return !v6 || isBlockedV6(v6);
+  }
+  return true;
+}
+
+// ---------------------------------------------------------------------------
+// DNS
+// ---------------------------------------------------------------------------
+
+async function dnsLookupAll(hostname: string): Promise<ResolvedAddress[]> {
+  const addrs = await dnsLookup(hostname, { all: true, verbatim: true });
+  return addrs.map((a) => ({ address: a.address, family: a.family === 6 ? 6 : 4 }));
+}
+
+/** Resolves `hostname` and throws BLOCKED_ADDRESS if ANY answer is blocked. */
+async function resolveChecked(
+  resolve: (hostname: string) => Promise<ResolvedAddress[]>,
+  hostname: string,
+  url?: string,
+): Promise<ResolvedAddress[]> {
+  let addrs: ResolvedAddress[];
+  try {
+    addrs = await resolve(hostname);
+  } catch (err) {
+    throw new SafeFetchError(`Could not resolve ${hostname}: ${(err as Error)?.message ?? err}`, 'BAD_URL', url);
+  }
+  if (!addrs.length) throw new SafeFetchError(`Could not resolve ${hostname}`, 'BAD_URL', url);
+  const blocked = addrs.find((a) => isBlockedAddress(a.address));
+  if (blocked) {
+    throw new SafeFetchError(`${hostname} resolves to a blocked address (${blocked.address})`, 'BLOCKED_ADDRESS', url);
+  }
+  return addrs;
+}
+
+/**
+ * A `dns.lookup`-shaped function for `net.connect` / undici `connect.lookup`.
+ * It resolves with `resolve`, refuses the whole answer if any address is
+ * blocked, and hands the socket only the addresses it validated — so the
+ * connection goes to an address that was checked, not to a second resolution.
+ * Handles both callback forms (`{all:true}` → array; otherwise address+family).
+ */
+export function pinnedLookup(resolve: SafeFetchOptions['lookup'] = dnsLookupAll): LookupFunction {
+  return (hostname, options, callback) => {
+    const wanted = options?.family === 4 || (options?.family as unknown) === 'IPv4' ? 4
+      : options?.family === 6 || (options?.family as unknown) === 'IPv6' ? 6 : 0;
+    resolveChecked(resolve, hostname).then(
+      (addrs) => {
+        const list = wanted ? addrs.filter((a) => a.family === wanted) : addrs;
+        if (!list.length) {
+          const err = Object.assign(new Error(`No IPv${wanted} address for ${hostname}`), { code: 'ENOTFOUND' });
+          callback(err, '', 0);
+          return;
+        }
+        if (options?.all) callback(null, list.map((a) => ({ address: a.address, family: a.family })));
+        else callback(null, list[0]!.address, list[0]!.family);
+      },
+      (err: Error) => callback(err as NodeJS.ErrnoException, '', 0),
+    );
+  };
+}
+
+// ---------------------------------------------------------------------------
+// safeFetch
+// ---------------------------------------------------------------------------
+
+function parseHopUrl(raw: string | URL, base?: URL): URL {
+  let url: URL;
+  try {
+    url = new URL(raw, base);
+  } catch {
+    throw new SafeFetchError(`Not a valid url: ${String(raw)}`, 'BAD_URL', String(raw));
+  }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+    throw new SafeFetchError(`Only http and https urls are allowed (got ${url.protocol})`, 'BAD_URL', url.href);
+  }
+  if (url.username || url.password) {
+    throw new SafeFetchError('Urls with credentials are not allowed', 'BAD_URL', url.href);
+  }
+  return url;
+}
+
+function bareHost(url: URL): string {
+  const h = url.hostname;
+  return h.startsWith('[') && h.endsWith(']') ? h.slice(1, -1) : h;
+}
+
+/** Rejects as soon as `signal` aborts, so a hung resolver cannot outlive the time cap. */
+function untilAborted<T>(p: Promise<T>, signal: AbortSignal): Promise<T> {
+  if (signal.aborted) return Promise.reject(new SafeFetchError('Timed out', 'TIMEOUT'));
+  return new Promise<T>((resolve, reject) => {
+    const onAbort = () => reject(new SafeFetchError('Timed out', 'TIMEOUT'));
+    signal.addEventListener('abort', onAbort, { once: true });
+    p.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort));
+  });
+}
+
+/** undici wraps connect failures as `TypeError('fetch failed', { cause })`. */
+function findSafeFetchError(err: unknown): SafeFetchError | undefined {
+  for (let e: unknown = err, depth = 0; e && depth < 5; e = (e as { cause?: unknown }).cause, depth++) {
+    if (e instanceof SafeFetchError) return e;
+  }
+  return undefined;
+}
+
+async function readCapped(res: Response, maxBytes: number, url: string): Promise<Uint8Array> {
+  if (!res.body) return new Uint8Array(0);
+  const reader = res.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    total += value.byteLength;
+    if (total > maxBytes) {
+      await reader.cancel().catch(() => undefined);
+      throw new SafeFetchError(`Response is larger than ${maxBytes} bytes`, 'TOO_LARGE', url);
+    }
+    chunks.push(value);
+  }
+  const body = new Uint8Array(total);
+  let offset = 0;
+  for (const c of chunks) {
+    body.set(c, offset);
+    offset += c.byteLength;
+  }
+  return body;
+}
+
+export async function safeFetch(url: string, options: SafeFetchOptions = {}): Promise<SafeFetchResult> {
+  const allowTypes = (options.allowTypes ?? SAFE_FETCH_DEFAULTS.allowTypes).map((t) => t.toLowerCase());
+  const maxBytes = options.maxBytes ?? SAFE_FETCH_DEFAULTS.maxBytes;
+  const maxRedirects = options.maxRedirects ?? SAFE_FETCH_DEFAULTS.maxRedirects;
+  const timeoutMs = options.timeoutMs ?? SAFE_FETCH_DEFAULTS.timeoutMs;
+  const resolve = options.lookup ?? dnsLookupAll;
+
+  let current = parseHopUrl(url);
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  let agent: Agent | undefined;
+  let fetchImpl: (input: string, init: RequestInit) => Promise<Response>;
+  if (options.fetch) {
+    fetchImpl = options.fetch;
+  } else {
+    // npm undici's own fetch: a npm-undici dispatcher must not be handed to Node's global fetch.
+    const pinned = new Agent({ connect: { lookup: pinnedLookup(resolve) } });
+    agent = pinned;
+    fetchImpl = (input, init) =>
+      undiciFetch(input, { ...(init as Parameters<typeof undiciFetch>[1]), dispatcher: pinned }) as unknown as Promise<Response>;
+  }
+
+  try {
+    for (let hop = 0; ; hop++) {
+      const host = bareHost(current);
+      // Pre-flight: an IP literal is checked as-is (never resolved); a name is resolved and every answer checked.
+      if (isIP(host)) {
+        if (isBlockedAddress(host)) {
+          throw new SafeFetchError(`Blocked address ${host}`, 'BLOCKED_ADDRESS', current.href);
+        }
+      } else {
+        await untilAborted(resolveChecked(resolve, host, current.href), controller.signal);
+      }
+
+      const res = await fetchImpl(current.href, { redirect: 'manual', signal: controller.signal });
+
+      if (res.status >= 300 && res.status < 400) {
+        const location = res.headers.get('location');
+        await res.body?.cancel().catch(() => undefined);
+        if (!location) throw new SafeFetchError(`HTTP ${res.status} without a Location`, 'HTTP_ERROR', current.href);
+        if (hop >= maxRedirects) {
+          throw new SafeFetchError(`More than ${maxRedirects} redirects`, 'TOO_MANY_REDIRECTS', current.href);
+        }
+        current = parseHopUrl(location, current);
+        continue;
+      }
+
+      if (!res.ok) {
+        await res.body?.cancel().catch(() => undefined);
+        throw new SafeFetchError(`HTTP ${res.status}`, 'HTTP_ERROR', current.href);
+      }
+
+      const contentType = (res.headers.get('content-type') ?? '').split(';')[0]!.trim().toLowerCase();
+      if (!contentType || !allowTypes.includes(contentType)) {
+        await res.body?.cancel().catch(() => undefined);
+        throw new SafeFetchError(`Unsupported content type: ${contentType || '(none)'}`, 'UNSUPPORTED_TYPE', current.href);
+      }
+
+      const declared = Number(res.headers.get('content-length'));
+      if (Number.isFinite(declared) && declared > maxBytes) {
+        await res.body?.cancel().catch(() => undefined);
+        throw new SafeFetchError(`Response is larger than ${maxBytes} bytes`, 'TOO_LARGE', current.href);
+      }
+
+      const body = await readCapped(res, maxBytes, current.href);
+      return { finalUrl: current.href, status: res.status, contentType, body };
+    }
+  } catch (err) {
+    const own = findSafeFetchError(err);
+    if (own) throw own;
+    if (controller.signal.aborted) throw new SafeFetchError(`Timed out after ${timeoutMs} ms`, 'TIMEOUT', current.href);
+    throw new SafeFetchError(`Fetch failed: ${(err as Error)?.message ?? err}`, 'HTTP_ERROR', current.href);
+  } finally {
+    clearTimeout(timer);
+    if (agent) void agent.destroy().catch(() => undefined);
+  }
+}