@huynle/hub 0.8.6 → 0.9.0

package/src/central/fetch-budget.ts

@@ -0,0 +1,59 @@
+// The per-user budget of the hub's web fetches (design §4): 30 fetches a minute and 4 at once per user, ONE budget
+// for GET /api/media/remote and GET /api/media/unfurl together (http.ts hands both handlers the same one), so the two
+// routes cannot add up to more than either allows. Cache hits and requests refused before fetching never take from
+// it. In memory, per pod in cluster mode (like the routes' caches).
+
+const WINDOW_MS = 60_000
+export const FETCH_BUDGET_DEFAULTS = { perMinute: 30, concurrent: 4 } as const
+
+export type BudgetTake =
+  | { readonly ok: true; readonly release: () => void }
+  /** `retryAfter`: seconds until a fetch may start again. */
+  | { readonly ok: false; readonly reason: "minute" | "concurrent"; readonly limit: number; readonly retryAfter: number }
+
+export interface FetchBudget {
+  /** One fetch for `userID` now, or why not; release() (idempotent) when the fetch ends, failures included. */
+  take(userID: string): BudgetTake
+}
+
+export interface FetchBudgetOptions {
+  readonly now?: () => number
+  readonly perMinute?: number
+  readonly concurrent?: number
+}
+
+export function createFetchBudget(options: FetchBudgetOptions = {}): FetchBudget {
+  const now = options.now ?? Date.now
+  const perMinute = options.perMinute ?? FETCH_BUDGET_DEFAULTS.perMinute
+  const concurrent = options.concurrent ?? FETCH_BUDGET_DEFAULTS.concurrent
+  const users = new Map<string, { starts: number[]; active: number }>()
+
+  function take(userID: string): BudgetTake {
+    const t = now()
+    const user = users.get(userID) ?? { starts: [], active: 0 }
+    users.set(userID, user)
+    user.starts = user.starts.filter((start) => start > t - WINDOW_MS)
+    const idle = () => {
+      if (user.active === 0 && user.starts.length === 0) users.delete(userID)
+    }
+    if (user.starts.length >= perMinute) {
+      idle()
+      return { ok: false, reason: "minute", limit: perMinute, retryAfter: Math.max(1, Math.ceil((user.starts[0]! + WINDOW_MS - t) / 1000)) }
+    }
+    if (user.active >= concurrent) return { ok: false, reason: "concurrent", limit: concurrent, retryAfter: 1 }
+    user.starts.push(t)
+    user.active++
+    let released = false
+    return {
+      ok: true,
+      release: () => {
+        if (released) return
+        released = true
+        user.active--
+        idle()
+      },
+    }
+  }
+
+  return { take }
+}

package/src/central/fetcher.ts

@@ -0,0 +1,372 @@
+// The safe outbound fetcher (design §4): the hub fetches user-chosen web URLs (remote images, link previews) for
+// signed-in users only. Without care that is SSRF: the hub would reach into its own network (loopback, the pod and
+// service networks, the cloud metadata IP). Rules:
+// - http/https on their default ports only, no userinfo; numeric host tricks (2130706433, 0177.0.0.1, 0x7f.1, …)
+//   are normalised by the WHATWG URL parser and then checked as addresses.
+// - The host is resolved once per hop with the injected resolver, and refused when ANY answer is in a denied range
+//   (DENIED below, plus HUB_FETCH_DENY_CIDRS). IPv4-mapped, IPv4-compatible and NAT64 IPv6 are refused outright.
+// - The connection goes to the exact address that was checked (DNS rebinding): Bun fetch on an IP-literal URL with
+//   Host = the hostname. Bun takes SNI and certificate verification from Host (quirk w8towhww), so Host never
+//   carries a port (only default ports are allowed anyway), and keepalive is off: a pooled socket for that IP would
+//   otherwise skip certificate verification for a different hostname.
+// - Redirects are followed manually, at most MAX_REDIRECTS, each hop fully re-checked. No cookies or credentials;
+//   User-Agent hub-preview/1. Timeouts: connect (until the response headers) and total (DNS, hops and body).
+// - The body is streamed and capped: over the cap the request is aborted (or truncated, when asked). It is never
+//   decompressed (decompress: false, Accept-Encoding: identity, any other Content-Encoding refused): Bun inflates
+//   before the cap would see the bytes, so a 1 MiB gzip bomb could grow to gigabytes.
+// Errors carry the host, never the path or query (a URL may hold leaked data).
+import { isIPv4, isIPv6 } from "node:net"
+
+export type ResolveHost = (host: string) => Promise<readonly string[]>
+
+export type FetchErrorKind = "bad_url" | "refused" | "too_large" | "upstream" | "timeout"
+
+const STATUS: Record<FetchErrorKind, number> = { bad_url: 400, refused: 403, too_large: 413, upstream: 502, timeout: 504 }
+
+/** Why a fetch did not happen or failed; `status` is the HTTP status a route answers with. */
+export class FetchError extends Error {
+  readonly status: number
+  constructor(
+    readonly kind: FetchErrorKind,
+    message: string,
+    readonly host?: string,
+    readonly upstreamStatus?: number,
+  ) {
+    super(message)
+    this.name = "FetchError"
+    this.status = STATUS[kind]
+  }
+}
+
+/** An address block: family, network (masked) and prefix length. */
+export interface Cidr {
+  readonly v6: boolean
+  readonly net: bigint
+  readonly bits: number
+}
+
+export const USER_AGENT = "hub-preview/1"
+export const MAX_REDIRECTS = 3
+export const MAX_URL_CHARS = 2048
+const DEFAULT_TIMEOUTS = { connectMs: 3000, totalMs: 8000 } as const
+
+// ---------- addresses ----------
+
+const V4_BITS = 32
+const V6_BITS = 128
+
+function v4Value(ip: string): bigint {
+  return ip.split(".").reduce((n, part) => (n << 8n) | BigInt(Number(part)), 0n)
+}
+
+/** The 128-bit value of a valid IPv6 address (no zone), embedded dotted IPv4 tails included. */
+function v6Value(ip: string): bigint {
+  let text = ip.toLowerCase()
+  const tail = /(\d+\.\d+\.\d+\.\d+)$/.exec(text)
+  if (tail) {
+    const n = v4Value(tail[1]!)
+    text = `${text.slice(0, -tail[1]!.length)}${(n >> 16n).toString(16)}:${(n & 0xffffn).toString(16)}`
+  }
+  const [head, rest] = text.split("::") as [string, string | undefined]
+  const left = head ? head.split(":") : []
+  const right = rest === undefined ? [] : rest ? rest.split(":") : []
+  const groups = rest === undefined ? left : [...left, ...Array<string>(8 - left.length - right.length).fill("0"), ...right]
+  return groups.reduce((n, g) => (n << 16n) | BigInt(parseInt(g, 16)), 0n)
+}
+
+/** An IP address as {v6, value}, or undefined when the text is not one (zones are not addresses here). */
+function parseIP(text: string): { v6: boolean; value: bigint } | undefined {
+  if (text.includes("%")) return undefined
+  if (isIPv4(text)) return { v6: false, value: v4Value(text) }
+  if (isIPv6(text)) return { v6: true, value: v6Value(text) }
+  return undefined
+}
+
+function cidr(net: string, bits: number): Cidr {
+  const ip = parseIP(net)!
+  const size = ip.v6 ? V6_BITS : V4_BITS
+  return { v6: ip.v6, net: mask(ip.value, bits, size), bits }
+}
+
+const mask = (value: bigint, bits: number, size: number): bigint => (bits === 0 ? 0n : (value >> BigInt(size - bits)) << BigInt(size - bits))
+
+const contains = (block: Cidr, ip: { v6: boolean; value: bigint }) =>
+  block.v6 === ip.v6 && mask(ip.value, block.bits, block.v6 ? V6_BITS : V4_BITS) === block.net
+
+/** Never a public service. */
+const DENIED: readonly Cidr[] = [
+  cidr("0.0.0.0", 8), // "this network"
+  cidr("10.0.0.0", 8), // private
+  cidr("100.64.0.0", 10), // CGNAT
+  cidr("127.0.0.0", 8), // loopback
+  cidr("169.254.0.0", 16), // link-local, cloud metadata
+  cidr("172.16.0.0", 12), // private
+  cidr("192.0.0.0", 24), // IETF protocol assignments
+  cidr("192.0.2.0", 24), // TEST-NET-1
+  cidr("192.88.99.0", 24), // 6to4 relay anycast (deprecated)
+  cidr("192.168.0.0", 16), // private
+  cidr("198.18.0.0", 15), // benchmarking
+  cidr("198.51.100.0", 24), // TEST-NET-2
+  cidr("203.0.113.0", 24), // TEST-NET-3
+  cidr("224.0.0.0", 4), // multicast
+  cidr("240.0.0.0", 4), // reserved, broadcast (255.255.255.255 included)
+  cidr("::", 96), // unspecified, loopback, IPv4-compatible (deprecated)
+  cidr("::ffff:0:0", 96), // IPv4-mapped: refused whatever the embedded address
+  cidr("::ffff:0:0:0", 96), // IPv4-translated (SIIT)
+  cidr("64:ff9b::", 96), // NAT64 well-known prefix
+  cidr("64:ff9b:1::", 48), // NAT64 local-use prefix
+  cidr("100::", 64), // discard-only
+  cidr("2001::", 32), // Teredo: tunnels to an obfuscated IPv4 address
+  cidr("2001:db8::", 32), // documentation
+  cidr("fc00::", 7), // unique local
+  cidr("fe80::", 10), // link-local
+  cidr("fec0::", 10), // site-local (deprecated)
+  cidr("ff00::", 8), // multicast
+]
+
+/** 6to4 (2002::/16) tunnels to the IPv4 address in bits 16-48: refused when that address is. */
+const SIX_TO_FOUR = cidr("2002::", 16)
+
+/** True when the hub must not connect to `address` (anything that is not a plain IP address counts as denied). */
+export function isDeniedAddress(address: string, extra: readonly Cidr[] = []): boolean {
+  const ip = parseIP(address.trim().replace(/^\[(.*)\]$/, "$1"))
+  if (!ip) return true
+  if (DENIED.some((block) => contains(block, ip)) || extra.some((block) => contains(block, ip))) return true
+  if (contains(SIX_TO_FOUR, ip)) {
+    const v4 = { v6: false, value: (ip.value >> 80n) & 0xffffffffn }
+    return DENIED.some((block) => contains(block, v4)) || extra.some((block) => contains(block, v4))
+  }
+  return false
+}
+
+/** HUB_FETCH_DENY_CIDRS: a comma list of CIDRs or bare addresses (v4 or v6). Throws on any invalid entry. */
+export function parseDenyCidrs(text: string | undefined): Cidr[] {
+  const out: Cidr[] = []
+  for (const raw of (text ?? "").split(",")) {
+    const entry = raw.trim()
+    if (!entry) continue
+    const invalid = () => new Error(`HUB_FETCH_DENY_CIDRS: ${JSON.stringify(entry)} is not an IPv4 or IPv6 CIDR (e.g. 10.42.0.0/16, fd00::/8)`)
+    const match = /^([^/\s]+)(?:\/(\d{1,3}))?$/.exec(entry)
+    if (!match) throw invalid()
+    const ip = parseIP(match[1]!)
+    if (!ip) throw invalid()
+    const size = ip.v6 ? V6_BITS : V4_BITS
+    const bits = match[2] === undefined ? size : Number(match[2])
+    if (bits > size) throw invalid()
+    out.push({ v6: ip.v6, net: mask(ip.value, bits, size), bits })
+  }
+  return out
+}
+
+// ---------- URLs ----------
+
+/** A URL the fetcher may request: http(s) on the default port, no userinfo, no fragment. Throws FetchError bad_url. */
+export function checkURL(raw: string | URL): URL {
+  const text = String(raw)
+  if (text.length > MAX_URL_CHARS) throw new FetchError("bad_url", `The URL is longer than ${MAX_URL_CHARS} characters`)
+  let url: URL
+  try {
+    url = new URL(text)
+  } catch {
+    throw new FetchError("bad_url", "Not a valid absolute URL")
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") throw new FetchError("bad_url", "Only http and https URLs can be fetched")
+  if (url.username || url.password) throw new FetchError("bad_url", "URLs with credentials are refused", url.hostname)
+  // WHATWG drops the scheme's default port, so any port left is a non-default one (https on 80 included).
+  if (url.port !== "") throw new FetchError("bad_url", "Only the default ports (80 for http, 443 for https) are allowed", url.hostname)
+  if (!url.hostname) throw new FetchError("bad_url", "The URL has no host")
+  url.hash = ""
+  return url
+}
+
+// ---------- the fetcher ----------
+
+export interface FetcherOptions {
+  /** DNS: every address of a host (production: node:dns lookup with all:true; tests: a fake). */
+  readonly resolve: ResolveHost
+  /** Denied on top of the built-in ranges (HUB_FETCH_DENY_CIDRS, see parseDenyCidrs). Never removes any. */
+  readonly denyCidrs?: readonly Cidr[]
+  /** The HTTP client (default Bun's fetch). It receives the pinned IP-literal URL. */
+  readonly fetchImpl?: typeof fetch
+  readonly timeouts?: { readonly connectMs?: number; readonly totalMs?: number }
+}
+
+export interface FetchOptions {
+  /** Bytes of the (decoded) body read at most. */
+  readonly maxBytes: number
+  /** Over maxBytes: keep the first maxBytes (truncated: true) instead of failing too_large. */
+  readonly truncate?: boolean
+  /** The Accept header (default *\/*). */
+  readonly accept?: string
+}
+
+export interface Fetched {
+  /** The final URL (after redirects). */
+  readonly url: URL
+  /** The address that was checked and connected to for the final hop. */
+  readonly address: string
+  readonly status: number
+  readonly headers: Headers
+  readonly body: Uint8Array<ArrayBuffer>
+  readonly truncated: boolean
+}
+
+export interface Fetcher {
+  readonly timeouts: { readonly connectMs: number; readonly totalMs: number }
+  /** GET `raw` safely; throws FetchError. */
+  fetch(raw: string | URL, options: FetchOptions): Promise<Fetched>
+}
+
+const isRedirect = (status: number) => status === 301 || status === 302 || status === 303 || status === 307 || status === 308
+
+export function createFetcher(options: FetcherOptions): Fetcher {
+  const extra = options.denyCidrs ?? []
+  const fetchImpl = options.fetchImpl ?? globalThis.fetch
+  const timeouts = { connectMs: options.timeouts?.connectMs ?? DEFAULT_TIMEOUTS.connectMs, totalMs: options.timeouts?.totalMs ?? DEFAULT_TIMEOUTS.totalMs }
+
+  /** The checked address to connect to for `url`'s host. */
+  async function pin(url: URL, signal: AbortSignal): Promise<string> {
+    const host = url.hostname
+    const literal = host.startsWith("[") ? host.slice(1, -1) : isIPv4(host) ? host : undefined
+    if (literal !== undefined) {
+      if (isDeniedAddress(literal, extra)) throw new FetchError("refused", `${host} is a refused address`, host)
+      return literal
+    }
+    const name = host.toLowerCase().replace(/\.$/, "")
+    if (name === "localhost" || name.endsWith(".localhost")) throw new FetchError("refused", `${host} is a local name`, host)
+    let addresses: readonly string[]
+    try {
+      addresses = await abortable(options.resolve(name), signal)
+    } catch (cause) {
+      if (signal.aborted) throw cause
+      throw new FetchError("upstream", `${host} does not resolve`, host)
+    }
+    if (addresses.length === 0) throw new FetchError("upstream", `${host} does not resolve`, host)
+    if (addresses.some((a) => isDeniedAddress(a, extra))) throw new FetchError("refused", `${host} resolves to a refused address`, host)
+    return addresses[0]!.replace(/^\[(.*)\]$/, "$1")
+  }
+
+  async function run(raw: string | URL, request: FetchOptions): Promise<Fetched> {
+    const controller = new AbortController()
+    let timedOut = false
+    const expire = () => {
+      timedOut = true
+      controller.abort()
+    }
+    const total = setTimeout(expire, timeouts.totalMs)
+    let connect: ReturnType<typeof setTimeout> | undefined
+    let host: string | undefined
+    try {
+      let url = checkURL(raw)
+      for (let hop = 0; ; hop++) {
+        host = url.hostname
+        const address = await pin(url, controller.signal)
+        const target = new URL(url)
+        target.hostname = isIPv6(address) ? `[${address}]` : address
+        connect = setTimeout(expire, timeouts.connectMs)
+        // Raced with the signal too, so the timeouts hold even for a client that ignores it.
+        // TLS: Bun verifies the certificate against Host (the SNI name). Known Bun bug, not fixable here: it accepts
+        // a wildcard certificate across several labels (*.w.test matches y.x.w.test), for every fetch in the hub.
+        const init: RequestInit & { decompress: boolean } = {
+          method: "GET",
+          // Only default ports reach here, so Host has no port, and a single trailing dot is dropped (the checked
+          // name and address stay as they are): Host is also the SNI name (never "name:443" or "name.").
+          headers: { host: url.host.replace(/\.$/, ""), "user-agent": USER_AGENT, accept: request.accept ?? "*/*", "accept-encoding": "identity" },
+          redirect: "manual",
+          keepalive: false,
+          credentials: "omit",
+          decompress: false,
+          signal: controller.signal,
+        }
+        const response = await abortable(fetchImpl(target.href, init), controller.signal)
+        clearTimeout(connect)
+        if (isRedirect(response.status)) {
+          await response.body?.cancel().catch(() => {})
+          const location = response.headers.get("location")
+          if (!location) throw new FetchError("upstream", `${host} answered a redirect without a location`, host, response.status)
+          if (hop >= MAX_REDIRECTS) throw new FetchError("upstream", `More than ${MAX_REDIRECTS} redirects`, host, response.status)
+          let next: URL
+          try {
+            next = new URL(location, url)
+          } catch {
+            throw new FetchError("upstream", `${host} answered an invalid redirect`, host, response.status)
+          }
+          url = checkURL(next)
+          continue
+        }
+        if (response.status < 200 || response.status > 299) {
+          await response.body?.cancel().catch(() => {})
+          throw new FetchError("upstream", `${host} answered HTTP ${response.status}`, host, response.status)
+        }
+        const encoding = response.headers.get("content-encoding")?.trim().toLowerCase()
+        if (encoding && encoding !== "identity") {
+          await response.body?.cancel().catch(() => {})
+          throw new FetchError("upstream", `${host} sent a compressed response (content-encoding ${encoding}), which is refused`, host)
+        }
+        const declared = Number(response.headers.get("content-length") ?? NaN)
+        if (!request.truncate && Number.isFinite(declared) && declared > request.maxBytes) {
+          await response.body?.cancel().catch(() => {})
+          throw new FetchError("too_large", `The response from ${host} is larger than ${request.maxBytes} bytes`, host)
+        }
+        const { body, truncated } = await readCapped(response, request, host, controller)
+        return { url, address, status: response.status, headers: response.headers, body, truncated }
+      }
+    } catch (cause) {
+      if (timedOut) throw new FetchError("timeout", `${host ?? "The site"} did not answer in time`, host)
+      if (cause instanceof FetchError) throw cause
+      throw new FetchError("upstream", `Could not fetch from ${host ?? "the site"}`, host)
+    } finally {
+      clearTimeout(total)
+      clearTimeout(connect)
+    }
+  }
+
+  return { timeouts, fetch: run }
+}
+
+async function readCapped(response: Response, request: FetchOptions, host: string, controller: AbortController): Promise<{ body: Uint8Array<ArrayBuffer>; truncated: boolean }> {
+  if (!response.body) return { body: new Uint8Array(0), truncated: false }
+  const reader = response.body.getReader()
+  const chunks: Uint8Array[] = []
+  let size = 0
+  while (true) {
+    const { done, value } = await abortable(reader.read(), controller.signal).catch(async (cause: unknown) => {
+      await reader.cancel().catch(() => {})
+      throw cause
+    })
+    if (done) break
+    if (size + value.byteLength > request.maxBytes) {
+      await reader.cancel().catch(() => {})
+      controller.abort()
+      if (!request.truncate) throw new FetchError("too_large", `The response from ${host} is larger than ${request.maxBytes} bytes`, host)
+      chunks.push(value.subarray(0, request.maxBytes - size))
+      return { body: concat(chunks, request.maxBytes), truncated: true }
+    }
+    chunks.push(value)
+    size += value.byteLength
+  }
+  return { body: concat(chunks, size), truncated: false }
+}
+
+function concat(chunks: readonly Uint8Array[], size: number): Uint8Array<ArrayBuffer> {
+  const out = new Uint8Array(size)
+  let at = 0
+  for (const chunk of chunks) {
+    out.set(chunk, at)
+    at += chunk.byteLength
+  }
+  return out
+}
+
+/** `promise`, rejected early when `signal` aborts (a slow resolver counts against the total timeout). */
+function abortable<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
+  if (signal.aborted) return Promise.reject(new Error("aborted"))
+  return new Promise<T>((resolve, reject) => {
+    const onAbort = () => reject(new Error("aborted"))
+    signal.addEventListener("abort", onAbort, { once: true })
+    promise.then(
+      (value) => (signal.removeEventListener("abort", onAbort), resolve(value)),
+      (cause) => (signal.removeEventListener("abort", onAbort), reject(cause)),
+    )
+  })
+}