@fedify/vocab-runtime 2.5.0-dev.2446 → 2.5.0-dev.2492

package/src/public-fetch.ts

@@ -0,0 +1,338 @@
+import { getLogger } from "@logtape/logtape";
+import { issuePinGrant, type PinGrant, revokePinGrant } from "./pin-grant.ts";
+import {
+  type ConnectionPinner,
+  detectRuntime,
+  getRuntimePinner,
+} from "./pinners/mod.ts";
+import {
+  type LookupAllAddresses,
+  lookupAllAddresses,
+  lookupPublicAddresses,
+  parsePublicUrl,
+  type ResolvedAddress,
+  UrlError,
+} from "./url.ts";
+
+const logger = getLogger(["fedify", "runtime", "fetch"]);
+
+/**
+ * Options for {@link openPublicFetch} and {@link withPublicFetch}.
+ */
+export interface PublicFetchOptions {
+  /**
+   * An `AbortSignal` for cancelling the DNS lookup.
+   */
+  readonly signal?: AbortSignal;
+
+  /**
+   * A function that resolves all addresses of a hostname, used instead of
+   * `node:dns.lookup()`.  The addresses it returns are validated in the same
+   * way as the default lookup's.
+   */
+  readonly lookup?: (hostname: string) => Promise<readonly ResolvedAddress[]>;
+}
+
+/**
+ * A session for fetching resources from a single origin.  Its `fetch()`
+ * connects only to the addresses validated when the session was opened, on
+ * runtimes that support it.
+ */
+export interface PublicFetchSession {
+  /**
+   * A copy of the URL the session was opened for.
+   */
+  readonly url: URL;
+
+  /**
+   * Fetches a resource from the session's origin.  Redirects are never
+   * followed; the redirect mode is always `"manual"`.
+   * @throws {TypeError} If the input's origin differs from the session's, or
+   *                     if the session is closed.
+   */
+  fetch(input: Request | URL | string, init?: RequestInit): Promise<Response>;
+
+  /**
+   * Closes the session.  The cancellation of response bodies that have been
+   * neither consumed nor cancelled is started (without waiting for it), and
+   * the session can no longer fetch.  Calling this more than once has no
+   * effect.
+   */
+  close(): Promise<void>;
+}
+
+/**
+ * The functions created by {@link createPublicFetch}.
+ */
+export interface PublicFetch {
+  /**
+   * Validates the URL, resolves its hostname, checks that every resolved
+   * address is public, and opens a session that connects only to those
+   * addresses.
+   * @param url The URL to open a session for.
+   * @param options The options.
+   * @returns The session.  It must be closed.
+   * @throws {UrlError} If the URL is not allowed, or its hostname cannot be
+   *                    resolved (with `reason: "dns"`).
+   */
+  openPublicFetch(
+    url: URL | string,
+    options?: PublicFetchOptions,
+  ): Promise<PublicFetchSession>;
+
+  /**
+   * Opens a session with {@link PublicFetch.openPublicFetch}, runs the given
+   * function with its `fetch()`, and closes the session afterwards.
+   * @param url The URL to open a session for.
+   * @param fn The function to run.  It must not return a `Response`, since
+   *           the response bodies are cancelled when the session is closed.
+   * @param options The options.
+   * @returns The value returned by `fn`.
+   * @throws {UrlError} If the URL is not allowed.
+   * @throws {TypeError} If `fn` returns a `Response`.
+   */
+  withPublicFetch<T>(
+    url: URL | string,
+    fn: (fetch: PublicFetchSession["fetch"]) => Promise<T>,
+    options?: PublicFetchOptions,
+  ): Promise<T>;
+}
+
+/**
+ * The dependencies of {@link createPublicFetch}.
+ */
+export interface PublicFetchDependencies {
+  /**
+   * Gets the pinner to connect with, or `null` to use the global `fetch()`.
+   */
+  readonly pinner: () => Promise<ConnectionPinner | null>;
+
+  /**
+   * The default lookup function.
+   */
+  readonly lookup: LookupAllAddresses;
+}
+
+/**
+ * Creates {@link PublicFetch.openPublicFetch} and
+ * {@link PublicFetch.withPublicFetch} bound to the given pinner and lookup.
+ * This is exported only for tests; it is not exported from any entry point.
+ * @param dependencies The pinner and the default lookup function.
+ * @returns The functions.
+ */
+export function createPublicFetch(
+  dependencies: PublicFetchDependencies,
+): PublicFetch {
+  async function openPublicFetch(
+    url: URL | string,
+    options: PublicFetchOptions = {},
+  ): Promise<PublicFetchSession> {
+    const { url: parsed, hostname, family } = parsePublicUrl(url);
+    if (parsed.username !== "" || parsed.password !== "") {
+      throw new UrlError("URLs with credentials are not allowed");
+    }
+    const lookup = options.lookup ?? dependencies.lookup;
+    let addresses: readonly ResolvedAddress[];
+    if (family !== 0) {
+      addresses = [{ address: hostname, family }];
+    } else if (lookup === lookupAllAddresses && !await canResolve()) {
+      // Without the `net` permission on Deno, the hostname cannot be resolved,
+      // but fetch() cannot connect to it either:
+      return createSession(parsed, globalFetch);
+    } else {
+      options.signal?.throwIfAborted();
+      addresses = await lookupPublicAddresses(hostname, lookup);
+      options.signal?.throwIfAborted();
+    }
+    const pinner = await dependencies.pinner();
+    if (pinner == null || !pinner.supports(parsed)) {
+      logUnpinned(parsed);
+      return createSession(parsed, globalFetch);
+    }
+    const grant = issuePinGrant(parsed, addresses);
+    return createSession(
+      parsed,
+      (input, init) =>
+        pinner.fetch(grant, new Request(input, init), init.signal ?? undefined),
+      { grant, pinner },
+    );
+  }
+
+  async function withPublicFetch<T>(
+    url: URL | string,
+    fn: (fetch: PublicFetchSession["fetch"]) => Promise<T>,
+    options?: PublicFetchOptions,
+  ): Promise<T> {
+    const session = await openPublicFetch(url, options);
+    try {
+      const result = await fn(session.fetch);
+      if (result instanceof Response) {
+        throw new TypeError(
+          "withPublicFetch() callbacks must not return a Response, since " +
+            "its body is cancelled when the session is closed.",
+        );
+      }
+      return result;
+    } catch (error) {
+      logger.debug(
+        "Failed while fetching {url}; closing the session: {error}",
+        { url: session.url.href, error },
+      );
+      throw error;
+    } finally {
+      await session.close();
+    }
+  }
+
+  return { openPublicFetch, withPublicFetch };
+}
+
+const runtimePublicFetch: PublicFetch = createPublicFetch({
+  pinner: getRuntimePinner,
+  lookup: lookupAllAddresses,
+});
+
+/**
+ * Validates the URL, resolves its hostname, checks that every resolved
+ * address is public, and opens a session that connects only to those
+ * addresses.  Connections are pinned on Node.js and Bun.  On other runtimes,
+ * the session uses the global `fetch()` after the check.
+ * @param url The URL to open a session for.
+ * @param options The options.
+ * @returns The session.  It must be closed.
+ * @throws {UrlError} If the URL is not allowed, or its hostname cannot be
+ *                    resolved (with `reason: "dns"`).
+ */
+export const openPublicFetch: PublicFetch["openPublicFetch"] =
+  runtimePublicFetch.openPublicFetch;
+
+/**
+ * Opens a session with {@link openPublicFetch}, runs the given function with
+ * its `fetch()`, and closes the session afterwards.
+ * @param url The URL to open a session for.
+ * @param fn The function to run.  It must not return a `Response`, since
+ *           the response bodies are cancelled when the session is closed.
+ * @param options The options.
+ * @returns The value returned by `fn`.
+ * @throws {UrlError} If the URL is not allowed.
+ * @throws {TypeError} If `fn` returns a `Response`.
+ */
+export const withPublicFetch: PublicFetch["withPublicFetch"] =
+  runtimePublicFetch.withPublicFetch;
+
+/**
+ * Opens a session that neither validates nor pins addresses, for callers that
+ * allow private addresses.  It has the same rules as {@link openPublicFetch}
+ * sessions otherwise: requests are limited to the URL's origin, redirects are
+ * never followed, and unconsumed response bodies are cancelled on close.
+ * It grants nothing beyond calling the global `fetch()` directly.
+ * @param url The URL to open a session for.
+ * @returns The session.  It must be closed.
+ */
+export function openDirectFetch(
+  url: URL | string,
+): Promise<PublicFetchSession> {
+  return Promise.resolve(createSession(new URL(url), globalFetch));
+}
+
+/**
+ * Starts cancelling the body of a response without waiting for it to settle.
+ * The cancellation of a branch of a tee (e.g., a body cloned by a fetch mock)
+ * settles only after the other branch is cancelled too, per the Streams
+ * standard, so awaiting it could block forever.  Failures are only logged.
+ */
+function cancelResponseBody(response: Response, url: URL): void {
+  response.body?.cancel().catch((error) => {
+    logger.debug(
+      "Failed to cancel a response body from {url}: {error}",
+      { url: url.href, error },
+    );
+  });
+}
+
+function globalFetch(
+  input: Request | URL | string,
+  init: RequestInit,
+): Promise<Response> {
+  return fetch(input, init);
+}
+
+function createSession(
+  url: URL,
+  send: (input: Request | URL | string, init: RequestInit) => Promise<Response>,
+  pin?: { readonly grant: PinGrant; readonly pinner: ConnectionPinner },
+): PublicFetchSession {
+  const origin = url.origin;
+  const tracked: Response[] = [];
+  let closed = false;
+
+  async function sessionFetch(
+    input: Request | URL | string,
+    init?: RequestInit,
+  ): Promise<Response> {
+    if (closed) throw new TypeError("The fetch session is closed.");
+    const target = new URL(input instanceof Request ? input.url : input);
+    if (target.origin !== origin) {
+      throw new TypeError(
+        `A fetch session for ${origin} cannot fetch ${target.origin}.`,
+      );
+    }
+    // Override the default redirect mode ("follow") of Request objects too;
+    // Bun also needs the option on fetch() itself:
+    // https://github.com/oven-sh/bun/issues/10754
+    const response = await send(input, { ...init, redirect: "manual" });
+    if (closed) {
+      cancelResponseBody(response, url);
+      throw new TypeError("The fetch session is closed.");
+    }
+    tracked.push(response);
+    return response;
+  }
+
+  async function close(): Promise<void> {
+    if (closed) return;
+    closed = true;
+    for (const response of tracked) {
+      if (response.body == null || response.bodyUsed) continue;
+      if (response.body.locked) {
+        logger.debug(
+          "Cannot cancel a locked response body from {url}.",
+          { url: url.href },
+        );
+        continue;
+      }
+      cancelResponseBody(response, url);
+    }
+    if (pin == null) return;
+    revokePinGrant(pin.grant);
+    await pin.pinner.release(pin.grant);
+  }
+
+  return {
+    get url() {
+      return new URL(url);
+    },
+    fetch: sessionFetch,
+    close,
+  };
+}
+
+async function canResolve(): Promise<boolean> {
+  if (!("Deno" in globalThis)) return true;
+  const permission = await Deno.permissions.query({ name: "net" });
+  return permission.state === "granted";
+}
+
+const loggedUnpinned = new Set<string>();
+
+function logUnpinned(url: URL): void {
+  const runtime = detectRuntime();
+  const key = `${runtime} ${url.protocol}`;
+  if (loggedUnpinned.has(key)) return;
+  loggedUnpinned.add(key);
+  logger.debug(
+    "Outbound {protocol} connections are not pinned to validated addresses " +
+      "on {runtime}; the hostname is resolved again when connecting.",
+    { runtime, protocol: url.protocol },
+  );
+}

package/src/testdata/public-fetch-deno-worker.ts

@@ -0,0 +1,74 @@
+/**
+ * A worker for the Deno test of `openPublicFetch()` in
+ * *../public-fetch.test.ts*.  The debug log about unpinned connections is
+ * emitted once per runtime and protocol in a module instance, and other tests
+ * in the same process may already have emitted it for Deno, so the test runs
+ * this module in a worker, which has its own module instances.
+ *
+ * It opens three sessions through the global `fetch()`, which it replaces
+ * with a recording function, for `https://deno-host.invalid/1`, `/2`, and
+ * `http://deno-host.invalid/3`, and posts the results as a message.
+ * @module
+ */
+import { configure, type LogRecord } from "@logtape/logtape";
+import { openPublicFetch } from "../public-fetch.ts";
+
+const records: LogRecord[] = [];
+await configure({
+  sinks: { buffer: (record) => records.push(record) },
+  loggers: [
+    {
+      category: ["fedify", "runtime", "fetch"],
+      sinks: ["buffer"],
+      lowestLevel: "debug",
+    },
+    { category: ["logtape", "meta"], sinks: [] },
+  ],
+});
+
+const calls: [string, string | undefined][] = [];
+globalThis.fetch = (input, init) => {
+  const url = input instanceof Request ? input.url : String(input);
+  calls.push([url, init?.redirect]);
+  return Promise.resolve(new Response("ok"));
+};
+
+const hostnames: string[] = [];
+const lookup = (hostname: string) => {
+  hostnames.push(hostname);
+  return Promise.resolve([{ address: "93.184.215.14", family: 4 }]);
+};
+
+const bodies: string[] = [];
+for (
+  const url of [
+    "https://deno-host.invalid/1",
+    "https://deno-host.invalid/2",
+    "http://deno-host.invalid/3",
+  ]
+) {
+  const session = await openPublicFetch(url, { lookup });
+  try {
+    const response = await session.fetch(url, { redirect: "follow" });
+    bodies.push(await response.text());
+  } finally {
+    await session.close();
+  }
+}
+
+(globalThis as unknown as { postMessage(message: unknown): void })
+  .postMessage({
+    calls,
+    hostnames,
+    bodies,
+    logs: records
+      .filter((record) =>
+        typeof record.rawMessage === "string" &&
+        record.rawMessage.startsWith("Outbound {protocol} connections are not")
+      )
+      .map((record) => [
+        record.level,
+        record.properties.runtime,
+        record.properties.protocol,
+      ]),
+  });

package/src/url.test.ts

@@ -1216,6 +1216,17 @@ test("validatePublicUrl()", async () => {
   await validatePublicUrl("https://[64:ff9b::8.8.8.8]");
 });
 
+test("validatePublicUrl() keeps rejecting fedify-test.internal on Bun", {
+  skip: !("Bun" in globalThis),
+}, async () => {
+  // openPublicFetch() has no hostname exceptions, but validatePublicUrl()
+  // keeps its Bun-specific ones for compatibility:
+  await rejects(
+    () => validatePublicUrl("https://fedify-test.internal/"),
+    (error) => error instanceof UrlError && error.reason === "disallowed",
+  );
+});
+
 test("validateLookupAddresses() tolerates Cloudflare Workers CNAME entries", () => {
   validateLookupAddresses([
     { address: "app-host.example.net.", family: 4 },

package/src/url.ts

@@ -1,10 +1,21 @@
-import type { LookupAddress } from "node:dns";
 // FIXME: the default dns import exists since tests can stub `dns.lookup()`.
 // This should be replaced by injecting the lookup function such as an internal
 // option on `validatePublicUrl()`
 import dns from "node:dns/promises";
 import { isIP } from "node:net";
 
+/**
+ * An address returned by a DNS lookup.  It has the same shape as
+ * `ResolvedAddress` of `node:dns`, without depending on its type declarations.
+ * @internal
+ */
+export interface ResolvedAddress {
+  /** An IPv4 or IPv6 address. */
+  readonly address: string;
+  /** The IP family of the address: `4` or `6`. */
+  readonly family: number;
+}
+
 export class UrlError extends Error {
   /**
    * The failure category. Defaults to `"disallowed"`.
@@ -993,25 +1004,16 @@ function replaceGatewayHints(
 
 /**
  * Validates a URL to prevent SSRF attacks.
+ *
+ * This function only checks the addresses that the hostname currently
+ * resolves to; it does not pin the connection to those addresses.  A request
+ * made after this check resolves the hostname again, so this function alone
+ * does not prevent DNS rebinding.
  */
 export async function validatePublicUrl(url: string): Promise<void> {
-  const parsed = new URL(url);
-  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
-    throw new UrlError(`Unsupported protocol: ${parsed.protocol}`);
-  }
-  let hostname = parsed.hostname;
-  if (hostname.startsWith("[") && hostname.endsWith("]")) {
-    hostname = hostname.slice(1, -1);
-  }
-  if (hostname === "localhost") {
-    throw new UrlError("Localhost is not allowed");
-  }
-  const hostnameFamily = isIP(hostname);
-  if (hostnameFamily !== 0) {
-    validatePublicIpAddress(hostname, hostnameFamily);
-    return;
-  }
-  if ("Deno" in globalThis && !isIP(hostname)) {
+  const { hostname, family } = parsePublicUrl(url);
+  if (family !== 0) return;
+  if ("Deno" in globalThis) {
     // If the `net` permission is not granted, we can't resolve the hostname.
     // However, we can safely assume that it cannot gain access to private
     // resources.
@@ -1020,7 +1022,8 @@ export async function validatePublicUrl(url: string): Promise<void> {
   }
   // FIXME: This is a temporary workaround for the `Bun` runtime; for unknown
   // reasons, the Web Crypto API does not work as expected after a DNS lookup.
-  // This workaround purposes to prevent unit tests from hanging up:
+  // This workaround purposes to prevent unit tests from hanging up.
+  // It only applies to this function; openPublicFetch() never skips lookups:
   if ("Bun" in globalThis) {
     if (hostname === "example.com" || hostname.endsWith(".example.com")) {
       return;
@@ -1030,13 +1033,101 @@ export async function validatePublicUrl(url: string): Promise<void> {
   }
   // To prevent SSRF via DNS rebinding, we need to resolve all IP addresses
   // and ensure that they are all public:
-  let addresses: LookupAddress[];
+  await lookupPublicAddresses(hostname, lookupAllAddresses);
+}
+
+/**
+ * A function that resolves all addresses of a hostname, like
+ * `node:dns.lookup()` with `{ all: true }`.
+ * @internal
+ */
+export type LookupAllAddresses = (
+  hostname: string,
+) => Promise<readonly ResolvedAddress[]>;
+
+/**
+ * Resolves all addresses of the given hostname through `node:dns/promises`.
+ * The default `dns` object is looked up on every call so that tests can still
+ * stub `dns.lookup()`.
+ * @internal
+ */
+export function lookupAllAddresses(
+  hostname: string,
+): Promise<readonly ResolvedAddress[]> {
+  return dns.lookup(hostname, { all: true });
+}
+
+/**
+ * The result of {@link parsePublicUrl}.
+ * @internal
+ */
+export interface ParsedPublicUrl {
+  readonly url: URL;
+  /** The hostname without brackets around IPv6 literals. */
+  readonly hostname: string;
+  /** The IP family of the hostname, or `0` if it is not an IP literal. */
+  readonly family: number;
+}
+
+/**
+ * Checks the parts of a URL that can be validated without resolving it:
+ * the protocol, the `localhost` hostname, and IP literal hostnames.
+ * @throws {UrlError} If the URL is not allowed.
+ * @internal
+ */
+export function parsePublicUrl(url: string | URL): ParsedPublicUrl {
+  const parsed = new URL(url);
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new UrlError(`Unsupported protocol: ${parsed.protocol}`);
+  }
+  let hostname = parsed.hostname;
+  if (hostname.startsWith("[") && hostname.endsWith("]")) {
+    hostname = hostname.slice(1, -1);
+  }
+  if (hostname === "localhost") {
+    throw new UrlError("Localhost is not allowed");
+  }
+  const family = isIP(hostname);
+  if (family !== 0) validatePublicIpAddress(hostname, family);
+  return { url: parsed, hostname, family };
+}
+
+/**
+ * Resolves the given hostname and checks that every returned IP address is
+ * public.
+ * @returns The IP addresses of the hostname, without non-IP entries.
+ * @throws {UrlError} If the lookup fails, returns no IP address, or returns
+ *                    any non-public address.
+ * @internal
+ */
+export async function lookupPublicAddresses(
+  hostname: string,
+  lookup: LookupAllAddresses,
+): Promise<ResolvedAddress[]> {
+  let addresses: readonly ResolvedAddress[];
   try {
-    addresses = await dns.lookup(hostname, { all: true });
+    addresses = await lookup(hostname);
   } catch (error) {
     throw new UrlError("DNS lookup failed", { cause: error, reason: "dns" });
   }
   validateLookupAddresses(addresses);
+  return addresses
+    .filter(({ address }) => isIP(address) !== 0)
+    .map(({ address }) => ({ address, family: isIP(address) }));
+}
+
+/**
+ * Checks that the given IP address is public.  This is used right before
+ * connecting to an address that passed {@link lookupPublicAddresses}.
+ * @throws {UrlError} If the address is not a public IP address.
+ * @internal
+ */
+export function assertPublicAddress(address: ResolvedAddress): void {
+  const family = isIP(address.address);
+  if (family === 0 || family !== address.family) {
+    throw new UrlError(`Invalid or private address: ${address.address}`);
+  }
+  validatePublicIpAddress(address.address, family);
 }
 
 /**
@@ -1057,7 +1148,7 @@ export async function validatePublicUrl(url: string): Promise<void> {
  * @internal
  */
 export function validateLookupAddresses(
-  addresses: readonly LookupAddress[],
+  addresses: readonly ResolvedAddress[],
 ): void {
   let ipAddressCount = 0;
   for (const { address } of addresses) {

package/tsdown.config.ts

@@ -6,6 +6,7 @@ export default [
   defineConfig({
     entry: [
       "src/mod.ts",
+      "src/internal.ts",
       "src/internal/jsonld-cache.ts",
       "src/internal/portable-dereference.ts",
       "src/internal/signed-representation.ts",
@@ -22,8 +23,12 @@ export default [
   }),
   defineConfig({
     outDir: "dist/tests",
-    entry: (await Array.fromAsync(glob(`src/**/*.test.ts`)))
-      .map((f) => f.replace(sep, "/")),
+    entry: [
+      ...await Array.fromAsync(glob(`src/**/*.test.ts`)),
+      // Run by the Node.js pinner tests in a child process, since Node.js
+      // cannot run TypeScript sources without flags before 22.6.0:
+      "src/pinners/testdata/node-child.ts",
+    ].map((f) => f.replace(sep, "/")),
     format: ["esm", "cjs"],
     platform: "node",
     deps: { neverBundle: [/^node:/, "@fedify/fixture"] },