@bygghammar/cms-platform 0.95.0 → 0.96.0

package/src/server/outbound-http.ts

@@ -0,0 +1,565 @@
+/**
+ * The one way server code fetches a URL that staff or configuration chose.
+ *
+ * Media imports, form webhooks and popup webhooks all fetch addresses a person
+ * typed in. Each of those is a server-side request forgery primitive unless
+ * the request is held to the public internet, so every such request goes
+ * through `safeRequest`, which:
+ *
+ * - accepts only the schemes its policy names (webhooks: https only), and no
+ *   credentials, local or single-label host names, or privileged ports;
+ * - resolves the host name and refuses it unless *every* address it resolves
+ *   to is public (`outbound-address`): a name that also answers 127.0.0.1 is
+ *   refused, not tried;
+ * - connects to the address it validated — the connection is pinned, so a DNS
+ *   answer that changes between the check and the connection (rebinding)
+ *   cannot redirect it, while TLS is still verified against the host name;
+ * - follows redirects itself, validating and pinning every hop, up to a cap.
+ *   A webhook never follows one to another origin (it carries a signed body
+ *   and its signature); and when a hop changes origin or turns into a GET,
+ *   only plain request headers go with it, never entity or signature ones;
+ * - runs under one deadline, and reads at most `maxResponseBytes`, streaming,
+ *   whatever `Content-Length` claims. The deadline holds even against a
+ *   resolver or transport that ignores cancellation (Node's `dns.lookup`
+ *   cannot be cancelled at all);
+ * - reports failures by category with an address-free message, never a query
+ *   string, a resolved address or a response body.
+ *
+ * DNS resolution and pinned connections need Node (`node:dns`, `node:http`,
+ * `node:https`), which is what the self-hosted server runs. Where those are
+ * missing the request fails closed (`dns`), rather than going out unchecked.
+ *
+ * Server-only. Never import from client code.
+ */
+import { addressFamily, isPublicAddress, type AddressFamily } from "./outbound-address";
+import { isBlockedHostname } from "./webhook-url-guard";
+
+export { redactUrl } from "../util/redact-url";
+
+export type OutboundPolicy = {
+  readonly schemes: ReadonlyArray<"https:" | "http:">;
+  readonly maxRedirects: number;
+  /** One deadline for the whole request: every hop, DNS, connect and body. */
+  readonly timeoutMs: number;
+  readonly maxResponseBytes: number;
+  /** Whether a redirect may lead to another origin (scheme, host or port). */
+  readonly crossOriginRedirects: boolean;
+};
+
+/** Configured webhooks: https only, a short deadline, a small reply. */
+export const WEBHOOK_POLICY: OutboundPolicy = Object.freeze({
+  schemes: ["https:"] as const,
+  maxRedirects: 3,
+  timeoutMs: 10_000,
+  maxResponseBytes: 64 * 1024,
+  // A webhook goes to the receiver an administrator named, and to no one else:
+  // the signed body and its signature must not follow a redirect elsewhere.
+  crossOriginRedirects: false,
+});
+
+/** Staff media imports: the web as it is (http too), a longer deadline, a media-sized cap. */
+export const MEDIA_IMPORT_POLICY: OutboundPolicy = Object.freeze({
+  schemes: ["https:", "http:"] as const,
+  maxRedirects: 5,
+  timeoutMs: 30_000,
+  maxResponseBytes: 50 * 1024 * 1024,
+  // A download carries nothing secret, and files live on CDNs.
+  crossOriginRedirects: true,
+});
+
+export type OutboundFailure =
+  | "blocked-url"
+  | "blocked-address"
+  | "dns"
+  | "timeout"
+  | "network"
+  | "too-large"
+  | "too-many-redirects"
+  | "cross-origin-redirect";
+
+const MESSAGES: Record<OutboundFailure, string> = {
+  "blocked-url": "That address is not allowed: it must be a public http(s) address.",
+  "blocked-address": "That address leads inside this server's own network.",
+  dns: "That address could not be looked up.",
+  timeout: "The address did not answer in time.",
+  network: "The address could not be reached.",
+  "too-large": "The reply was too large.",
+  "too-many-redirects": "The address redirected too many times.",
+  "cross-origin-redirect": "The address redirected to another site.",
+};
+
+export type ResolvedAddress = { readonly address: string; readonly family: AddressFamily };
+
+/** Looks a host name up. Injected in tests; Node's resolver otherwise. */
+export type Resolver = (hostname: string, signal: AbortSignal) => Promise<readonly ResolvedAddress[]>;
+
+export type TransportRequest = {
+  readonly url: URL;
+  /** The validated address to connect to. */
+  readonly address: ResolvedAddress;
+  readonly method: string;
+  readonly headers: Readonly<Record<string, string>>;
+  /** Sent as given: a signed body must reach the receiver byte for byte. */
+  readonly body?: string | Uint8Array;
+  readonly signal: AbortSignal;
+};
+
+export type TransportResponse = {
+  readonly status: number;
+  readonly headers: Headers;
+  readonly body: AsyncIterable<Uint8Array> | null;
+  /** Stops reading and releases the connection. */
+  close(): void;
+};
+
+/** Makes one request to one pinned address. Injected in tests; Node's http(s) otherwise. */
+export type Transport = (request: TransportRequest) => Promise<TransportResponse>;
+
+export type SafeRequestInput = {
+  readonly url: string;
+  readonly method?: "GET" | "POST";
+  readonly headers?: Readonly<Record<string, string>>;
+  readonly body?: string | Uint8Array;
+  readonly policy: OutboundPolicy;
+  readonly resolver?: Resolver;
+  readonly transport?: Transport;
+  readonly signal?: AbortSignal;
+};
+
+export type SafeResponse = {
+  readonly ok: true;
+  readonly status: number;
+  readonly headers: Headers;
+  readonly body: Uint8Array;
+  /** The address the final hop was made to. */
+  readonly url: string;
+};
+
+export type SafeFailure = {
+  readonly ok: false;
+  readonly failure: OutboundFailure;
+  /** Safe to show and to log: no address, no query string, no body. */
+  readonly message: string;
+};
+
+const MAX_URL_LENGTH = 2048;
+
+const fail = (failure: OutboundFailure): SafeFailure => ({
+  ok: false,
+  failure,
+  message: MESSAGES[failure],
+});
+
+/** The host as a bare address when it is an IP literal, else null. */
+function literalAddress(url: URL): ResolvedAddress | null {
+  const host = url.hostname.replace(/^\[|\]$/g, "");
+  const family = addressFamily(host);
+  return family === null ? null : { address: host, family };
+}
+
+/**
+ * Whether a URL may be requested at all, before any lookup.
+ *
+ * The WHATWG URL parser has already turned decimal, hex, octal and shorthand
+ * IPv4 forms (`2130706433`, `0x7f.1`) into dotted quads, so the address checks
+ * see the address the connection would use.
+ */
+export function checkOutboundUrl(
+  raw: string | URL,
+  policy: Pick<OutboundPolicy, "schemes">,
+): { ok: true; url: URL } | SafeFailure {
+  let url: URL;
+  try {
+    const text = typeof raw === "string" ? raw.trim() : raw.href;
+    if (text.length === 0 || text.length > MAX_URL_LENGTH) return fail("blocked-url");
+    url = new URL(text);
+  } catch {
+    return fail("blocked-url");
+  }
+  if (!(policy.schemes as readonly string[]).includes(url.protocol)) return fail("blocked-url");
+  if (url.username !== "" || url.password !== "") return fail("blocked-url");
+  const host = url.hostname.toLowerCase().replace(/\.$/, "");
+  if (host === "") return fail("blocked-url");
+  const literal = literalAddress(url);
+  if (literal) {
+    if (!isPublicAddress(literal.address)) return fail("blocked-address");
+  } else {
+    // A name with no dot is a name on the local network (a container, a host
+    // in /etc/hosts), never a public site.
+    if (!host.includes(".") || isBlockedHostname(host)) return fail("blocked-url");
+  }
+  // Privileged ports other than the web's own belong to other protocols.
+  if (url.port !== "") {
+    const port = Number(url.port);
+    if (port < 1024 && port !== 80 && port !== 443) return fail("blocked-url");
+  }
+  return { ok: true, url };
+}
+
+/** Every address the host resolves to, or why the request may not be made. */
+export async function resolvePublicAddresses(
+  url: URL,
+  resolver: Resolver,
+  signal: AbortSignal,
+): Promise<{ ok: true; addresses: readonly ResolvedAddress[] } | SafeFailure> {
+  const literal = literalAddress(url);
+  if (literal) {
+    return isPublicAddress(literal.address) ? { ok: true, addresses: [literal] } : fail("blocked-address");
+  }
+  let addresses: readonly ResolvedAddress[];
+  try {
+    addresses = await untilAborted(
+      Promise.resolve().then(() => resolver(url.hostname, signal)),
+      signal,
+    );
+  } catch {
+    return signal.aborted ? fail("timeout") : fail("dns");
+  }
+  if (addresses.length === 0) return fail("dns");
+  // All or nothing: one private answer among public ones is the rebinding setup.
+  if (!addresses.every((entry) => isPublicAddress(entry.address))) return fail("blocked-address");
+  return { ok: true, addresses };
+}
+
+async function readCapped(
+  body: AsyncIterable<Uint8Array> | null,
+  limit: number,
+): Promise<Uint8Array | null> {
+  if (body === null) return new Uint8Array(0);
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  for await (const chunk of body) {
+    total += chunk.byteLength;
+    if (total > limit) return null;
+    chunks.push(chunk);
+  }
+  const out = new Uint8Array(total);
+  let at = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, at);
+    at += chunk.byteLength;
+  }
+  return out;
+}
+
+/**
+ * `work`, or a rejection the moment `signal` aborts — whichever comes first.
+ *
+ * For work that does not listen to the signal (Node's `dns.lookup`, an
+ * injected transport, a stalled body): the deadline holds regardless. What the
+ * work does after losing is absorbed — a late rejection is not an unhandled
+ * one, and a late value is handed to `onLate` (to close a late connection).
+ */
+function untilAborted<T>(
+  work: Promise<T>,
+  signal: AbortSignal,
+  onLate?: (value: T) => void,
+): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    let settled = false;
+    const onAbort = () => {
+      if (settled) return;
+      settled = true;
+      reject(abortError());
+    };
+    work.then(
+      (value) => {
+        if (settled) {
+          onLate?.(value);
+          return;
+        }
+        settled = true;
+        signal.removeEventListener("abort", onAbort);
+        resolve(value);
+      },
+      (error: unknown) => {
+        if (settled) return;
+        settled = true;
+        signal.removeEventListener("abort", onAbort);
+        reject(error);
+      },
+    );
+    if (signal.aborted) onAbort();
+    else signal.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+function abortError(): Error {
+  const error = new Error("The operation was aborted.");
+  error.name = "AbortError";
+  return error;
+}
+
+/**
+ * The request headers that may go with a hop to another origin, or with a
+ * request a redirect turned into a GET. Nothing that describes a body
+ * (`content-*`) or vouches for one (signatures, timestamps, event ids) does.
+ */
+const PLAIN_HEADERS = new Set(["user-agent", "accept", "accept-language"]);
+
+function plainHeadersOnly(headers: Readonly<Record<string, string>>): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(headers).filter(([name]) => PLAIN_HEADERS.has(name.toLowerCase())),
+  );
+}
+
+/** What the next hop of a redirect is: method and body per the Fetch standard. */
+function redirectedRequest(
+  status: number,
+  method: string,
+  body: string | Uint8Array | undefined,
+): { method: string; body: string | Uint8Array | undefined } {
+  if (status === 307 || status === 308) return { method, body };
+  if (status === 303 || ((status === 301 || status === 302) && method === "POST")) {
+    return { method: method === "HEAD" ? "HEAD" : "GET", body: undefined };
+  }
+  return { method, body };
+}
+
+/** Make one request under the policy. Never throws. */
+export async function safeRequest(input: SafeRequestInput): Promise<SafeResponse | SafeFailure> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), input.policy.timeoutMs);
+  const onAbort = () => controller.abort();
+  input.signal?.addEventListener("abort", onAbort, { once: true });
+  const signal = controller.signal;
+
+  let method: string = input.method ?? "GET";
+  let body = input.body;
+  let headers: Readonly<Record<string, string>> = input.headers ?? {};
+  let target: string | URL = input.url;
+
+  try {
+    const resolver = input.resolver ?? (await nodeResolver());
+    const transport = input.transport ?? (await nodeTransport());
+    if (!resolver || !transport) return fail("dns");
+
+    for (let hop = 0; hop <= input.policy.maxRedirects; hop += 1) {
+      const checked = checkOutboundUrl(target, input.policy);
+      if (!checked.ok) return checked;
+      const url = checked.url;
+      const resolved = await resolvePublicAddresses(url, resolver, signal);
+      if (!resolved.ok) return resolved;
+      if (signal.aborted) return fail("timeout");
+
+      let response: TransportResponse;
+      try {
+        response = await untilAborted(
+          Promise.resolve().then(() =>
+            transport({ url, address: resolved.addresses[0]!, method, headers, body, signal }),
+          ),
+          signal,
+          (late) => late.close(),
+        );
+      } catch (caught) {
+        return signal.aborted || isAbortError(caught) ? fail("timeout") : fail("network");
+      }
+
+      if (response.status >= 300 && response.status < 400 && response.status !== 304) {
+        const location = response.headers.get("location");
+        response.close();
+        if (location === null) {
+          return { ok: true, status: response.status, headers: response.headers, body: new Uint8Array(0), url: url.href };
+        }
+        let next: URL;
+        try {
+          next = new URL(location, url);
+        } catch {
+          return fail("blocked-url");
+        }
+        // Refused before the next hop is even looked up.
+        const crossOrigin = next.origin !== url.origin;
+        if (crossOrigin && !input.policy.crossOriginRedirects) return fail("cross-origin-redirect");
+        const redirected = redirectedRequest(response.status, method, body);
+        if (crossOrigin || redirected.method !== method) headers = plainHeadersOnly(headers);
+        ({ method, body } = redirected);
+        target = next;
+        continue;
+      }
+
+      const declared = Number(response.headers.get("content-length"));
+      if (Number.isFinite(declared) && declared > input.policy.maxResponseBytes) {
+        response.close();
+        return fail("too-large");
+      }
+      let bytes: Uint8Array | null;
+      try {
+        bytes = await untilAborted(readCapped(response.body, input.policy.maxResponseBytes), signal);
+      } catch {
+        response.close();
+        return signal.aborted ? fail("timeout") : fail("network");
+      }
+      if (bytes === null) {
+        response.close();
+        return fail("too-large");
+      }
+      return { ok: true, status: response.status, headers: response.headers, body: bytes, url: url.href };
+    }
+    return fail("too-many-redirects");
+  } catch {
+    return signal.aborted ? fail("timeout") : fail("network");
+  } finally {
+    clearTimeout(timer);
+    input.signal?.removeEventListener("abort", onAbort);
+  }
+}
+
+const isAbortError = (caught: unknown) =>
+  caught instanceof Error && (caught.name === "AbortError" || caught.name === "TimeoutError");
+
+// ---------------------------------------------------------------------------
+// Injected fetch (test seam)
+// ---------------------------------------------------------------------------
+
+/**
+ * A transport over an injected `fetch`, for tests that stand in for the
+ * network with one. Every hop is still checked and followed by `safeRequest`
+ * (the fetch is asked never to follow a redirect itself), but the connection
+ * is the fetch's own: it is not pinned. Production code passes no fetch.
+ */
+export function fetchTransport(fetchImpl: typeof fetch): Transport {
+  return async (request) => {
+    const response = await fetchImpl(request.url.href, {
+      method: request.method,
+      headers: { ...request.headers },
+      body: request.body as BodyInit | undefined,
+      redirect: "manual",
+      signal: request.signal,
+    });
+    const stream = response.body;
+    return {
+      status: response.status,
+      headers: response.headers,
+      body: stream ? streamChunks(stream) : null,
+      close: () => void stream?.cancel().catch(() => undefined),
+    };
+  };
+}
+
+/** With an injected fetch there is no real lookup: names stand for a public host. */
+export const injectedFetchResolver: Resolver = async () => [{ address: "192.88.98.1", family: 4 }];
+
+async function* streamChunks(stream: ReadableStream<Uint8Array>): AsyncIterable<Uint8Array> {
+  const reader = stream.getReader();
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) return;
+      if (value) yield value;
+    }
+  } finally {
+    await reader.cancel().catch(() => undefined);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Node implementations
+// ---------------------------------------------------------------------------
+
+/** Node's resolver, or null where there is none (fail closed). */
+async function nodeResolver(): Promise<Resolver | null> {
+  try {
+    const dns = await import("node:dns");
+    return async (hostname) => {
+      const answers = await dns.promises.lookup(hostname, { all: true, verbatim: true });
+      return answers.map((a) => ({ address: a.address, family: a.family === 6 ? 6 : 4 }));
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** Node's http(s) client, connecting only to the pinned address. Null where there is none. */
+async function nodeTransport(): Promise<Transport | null> {
+  let http: typeof import("node:http");
+  let https: typeof import("node:https");
+  try {
+    http = await import("node:http");
+    https = await import("node:https");
+  } catch {
+    return null;
+  }
+  return (request) =>
+    new Promise<TransportResponse>((resolve, reject) => {
+      const { url, address } = request;
+      const secure = url.protocol === "https:";
+      const hostname = url.hostname.replace(/^\[|\]$/g, "");
+      const client = secure ? https : http;
+      const req = client.request(
+        {
+          protocol: url.protocol,
+          hostname,
+          port: url.port || (secure ? 443 : 80),
+          path: `${url.pathname}${url.search}`,
+          method: request.method,
+          headers: { ...request.headers, host: url.host },
+          // Pinned: whatever the name resolves to now, connect to the address
+          // that was checked. TLS still verifies the certificate for `hostname`.
+          lookup: (_name, options, callback) => {
+            if ((options as { all?: boolean }).all) {
+              (callback as (e: null, a: Array<{ address: string; family: number }>) => void)(null, [
+                { address: address.address, family: address.family },
+              ]);
+            } else {
+              (callback as (e: null, a: string, f: number) => void)(null, address.address, address.family);
+            }
+          },
+          agent: false,
+          signal: request.signal,
+        },
+        (res) => {
+          const headers = new Headers();
+          for (const [name, value] of Object.entries(res.headers)) {
+            if (value === undefined) continue;
+            for (const v of Array.isArray(value) ? value : [value]) headers.append(name, v);
+          }
+          resolve({
+            status: res.statusCode ?? 0,
+            headers,
+            body: res as AsyncIterable<Uint8Array>,
+            close: () => res.destroy(),
+          });
+        },
+      );
+      req.on("error", reject);
+      if (request.body !== undefined) req.end(Buffer.from(request.body));
+      else req.end();
+    });
+}
+
+// ---------------------------------------------------------------------------
+// Media
+// ---------------------------------------------------------------------------
+
+export type MediaKind = { readonly mime: string; readonly ext: string };
+
+/**
+ * What a downloaded file actually is, by its first bytes — never by what the
+ * server said or what the address ends in. SVG and HTML are not media here:
+ * served from the media bucket they would run script.
+ */
+export function sniffMedia(bytes: Uint8Array): MediaKind | null {
+  const at = (offset: number, ...values: number[]) =>
+    values.every((value, i) => bytes[offset + i] === value);
+  const ascii = (offset: number, text: string) =>
+    at(offset, ...Array.from(text, (c) => c.charCodeAt(0)));
+  if (at(0, 0xff, 0xd8, 0xff)) return { mime: "image/jpeg", ext: ".jpg" };
+  if (at(0, 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)) return { mime: "image/png", ext: ".png" };
+  if (ascii(0, "GIF87a") || ascii(0, "GIF89a")) return { mime: "image/gif", ext: ".gif" };
+  if (ascii(0, "RIFF") && ascii(8, "WEBP")) return { mime: "image/webp", ext: ".webp" };
+  if (ascii(0, "RIFF") && ascii(8, "WAVE")) return { mime: "audio/wav", ext: ".wav" };
+  if (ascii(4, "ftyp")) {
+    const brand = String.fromCharCode(...bytes.slice(8, 12));
+    if (brand === "avif" || brand === "avis") return { mime: "image/avif", ext: ".avif" };
+    if (brand === "qt  ") return { mime: "video/quicktime", ext: ".mov" };
+    if (brand.startsWith("M4V")) return { mime: "video/x-m4v", ext: ".m4v" };
+    return { mime: "video/mp4", ext: ".mp4" };
+  }
+  if (at(0, 0x1a, 0x45, 0xdf, 0xa3)) return { mime: "video/webm", ext: ".webm" };
+  if (ascii(0, "OggS")) return { mime: "audio/ogg", ext: ".ogg" };
+  if (ascii(0, "ID3") || at(0, 0xff, 0xfb) || at(0, 0xff, 0xf3) || at(0, 0xff, 0xf2)) {
+    return { mime: "audio/mpeg", ext: ".mp3" };
+  }
+  if (ascii(0, "%PDF-")) return { mime: "application/pdf", ext: ".pdf" };
+  return null;
+}