@fedify/vocab-runtime 2.2.18 → 2.2.20

package/src/docloader.ts

@@ -1,16 +1,21 @@
 import { getLogger } from "@logtape/logtape";
-import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
+import { type Span, SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
 import metadata from "../deno.json" with { type: "json" };
 import { MAX_BODY_SIZE, readBoundedText } from "./body.ts";
 import preloadedContexts from "./contexts.ts";
 import { HttpHeaderLink } from "./link.ts";
+import {
+  openDirectFetch,
+  openPublicFetch,
+  type PublicFetchSession,
+} from "./public-fetch.ts";
 import {
   createActivityPubRequest,
   FetchError,
   type GetUserAgentOptions,
   logRequest,
 } from "./request.ts";
-import { UrlError, validatePublicUrl } from "./url.ts";
+import { UrlError } from "./url.ts";
 
 const logger = getLogger(["fedify", "runtime", "docloader"]);
 const DEFAULT_MAX_REDIRECTION = 20;
@@ -117,7 +122,9 @@ export type AuthenticatedDocumentLoaderFactory = (
  * Gets a {@link RemoteDocument} from the given response.
  * @param url The URL of the document to load.
  * @param response The response to get the document from.
- * @param fetch The function to fetch the document.
+ * @param fetch The function to fetch the document.  It is called with the URL
+ *              of an alternate document, if any, after the response body has
+ *              been either read or its cancellation has been started.
  * @returns The loaded remote document.
  * @throws {FetchError} If the response is not OK.
  * @internal
@@ -130,7 +137,54 @@ export async function getRemoteDocument(
     options?: DocumentLoaderOptions,
   ) => Promise<RemoteDocument>,
 ): Promise<RemoteDocument> {
-  const documentUrl = response.url === "" ? url : response.url;
+  const result = await parseRemoteDocument(url, response);
+  if ("alternate" in result) return await fetch(result.alternate);
+  return result.document;
+}
+
+/**
+ * Starts cancelling the body of a response without waiting for it to settle.
+ * The cancellation of a branch of a tee (e.g., a body cloned by a fetch mock)
+ * settles only after the other branch is cancelled too, per the Streams
+ * standard, so awaiting it could block forever.  Failures are only logged.
+ * @param response The response whose body is cancelled.
+ * @param url The URL of the response, for logging.
+ */
+function cancelResponseBody(response: Response, url: string): void {
+  response.body?.cancel().catch((error) => {
+    logger.debug(
+      "Failed to cancel the response body from {url}: {error}",
+      { url, error },
+    );
+  });
+}
+
+/**
+ * The result of {@link parseRemoteDocument}: either the loaded document, or
+ * the URL of an alternate document to load instead.
+ */
+type ParsedRemoteDocument =
+  | { readonly document: RemoteDocument }
+  | { readonly alternate: string };
+
+/**
+ * Gets a {@link RemoteDocument} from the given response, or the URL of an
+ * alternate document to load instead.  When an alternate document is found,
+ * the response body has been either read or its cancellation has been started
+ * (without waiting for it), so that the alternate document can be loaded after
+ * the response's session is closed.
+ * @param url The URL of the document to load.
+ * @param response The response to get the document from.
+ * @returns The loaded remote document, or the alternate document URL.
+ * @throws {FetchError} If the response is not OK.
+ */
+async function parseRemoteDocument(
+  url: string,
+  response: Response,
+): Promise<ParsedRemoteDocument> {
+  // Since the response URL is an IP address URL on runtimes that pin
+  // connections, the request URL is used instead:
+  const documentUrl = url;
   const docUrl = new URL(documentUrl);
   if (!response.ok) {
     logger.error(
@@ -189,7 +243,8 @@ export async function getRemoteDocument(
             "Found alternate document: {alternateUrl} from {url}",
             { alternateUrl: altUri.href, url: documentUrl },
           );
-          return await fetch(altUri.href);
+          cancelResponseBody(response, documentUrl);
+          return { alternate: altUri.href };
         }
       }
     }
@@ -241,7 +296,7 @@ export async function getRemoteDocument(
           "Found alternate document: {alternateUrl} from {url}",
           { alternateUrl: attribs.href, url: documentUrl },
         );
-        return await fetch(new URL(attribs.href, docUrl).href);
+        return { alternate: new URL(attribs.href, docUrl).href };
       }
     }
     document = JSON.parse(html);
@@ -258,7 +313,7 @@ export async function getRemoteDocument(
       headers: Object.fromEntries(response.headers.entries()),
     },
   );
-  return { contextUrl, document, documentUrl };
+  return { document: { contextUrl, document, documentUrl } };
 }
 
 /**
@@ -297,13 +352,99 @@ export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
  * @since 1.3.0
  */
 export function getDocumentLoader(
+  options: GetDocumentLoaderOptions = {},
+): DocumentLoader {
+  return createDocumentLoader(options);
+}
+
+/**
+ * The dependencies of {@link createDocumentLoader}.
+ */
+export interface DocumentLoaderDependencies {
+  /**
+   * Opens the fetch session for a single hop, instead of
+   * {@link openPublicFetch} (or {@link openDirectFetch} if private addresses
+   * are allowed).
+   */
+  readonly openHop?: (url: string) => Promise<PublicFetchSession>;
+}
+
+/**
+ * The outcome of a single hop of {@link createDocumentLoader}, decided while
+ * its fetch session is open.
+ */
+type HopOutcome = ParsedRemoteDocument | { readonly location: string };
+
+/**
+ * The implementation of {@link getDocumentLoader}, with injectable
+ * dependencies.  This is not exported from any entry point.
+ * @param options Options for the document loader.
+ * @param deps The dependencies.
+ * @returns The document loader.
+ */
+export function createDocumentLoader(
   { allowPrivateAddress, maxRedirection, skipPreloadedContexts, userAgent }:
     GetDocumentLoaderOptions = {},
+  deps: DocumentLoaderDependencies = {},
 ): DocumentLoader {
   const tracerProvider = trace.getTracerProvider();
   const tracer = tracerProvider.getTracer(metadata.name, metadata.version);
   const maximumRedirection = maxRedirection ?? DEFAULT_MAX_REDIRECTION;
 
+  async function openHop(
+    url: string,
+    signal?: AbortSignal,
+  ): Promise<PublicFetchSession> {
+    try {
+      if (deps.openHop != null) return await deps.openHop(url);
+      return allowPrivateAddress
+        ? await openDirectFetch(url)
+        : await openPublicFetch(url, { signal });
+    } catch (error) {
+      if (error instanceof UrlError) {
+        if (error.reason === "dns") {
+          logger.debug("DNS lookup failed for {url}", { url, error });
+        } else {
+          logger.error("Disallowed private URL: {url}", { url, error });
+        }
+      }
+      throw error;
+    }
+  }
+
+  async function fetchHop(
+    session: PublicFetchSession,
+    url: string,
+    span: Span,
+    options?: DocumentLoaderOptions,
+  ): Promise<HopOutcome> {
+    try {
+      const request = createActivityPubRequest(url, { userAgent });
+      logRequest(logger, request);
+      const response = await session.fetch(request, {
+        // Since Bun has a bug that ignores the `Request.redirect` option,
+        // to work around it we specify `redirect: "manual"` here too:
+        // https://github.com/oven-sh/bun/issues/10754
+        redirect: "manual",
+        signal: options?.signal,
+      });
+      span.setAttribute("http.response.status_code", response.status);
+
+      // Follow redirects manually to get the final URL:
+      if (
+        response.status >= 300 && response.status < 400 &&
+        response.headers.has("Location")
+      ) {
+        const location = response.headers.get("Location")!;
+        cancelResponseBody(response, url);
+        return { location };
+      }
+      return await parseRemoteDocument(url, response);
+    } finally {
+      await session.close();
+    }
+  }
+
   async function load(
     url: string,
     options?: DocumentLoaderOptions,
@@ -315,6 +456,11 @@ export function getDocumentLoader(
     if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
       throw new UrlError(`Unsupported protocol: ${parsedUrl.protocol}`);
     }
+    // When merging forward, keep this guard independent of allowPrivateAddress:
+    // private-address opt-in must not permit credentials on any followed URL.
+    if (parsedUrl.username !== "" || parsedUrl.password !== "") {
+      throw new UrlError("URLs with credentials are not allowed");
+    }
     const currentUrl = parsedUrl.href;
     if (!skipPreloadedContexts && currentUrl in preloadedContexts) {
       logger.debug("Using preloaded context: {url}.", { url: currentUrl });
@@ -324,26 +470,9 @@ export function getDocumentLoader(
         documentUrl: currentUrl,
       };
     }
-    if (!allowPrivateAddress) {
-      try {
-        await validatePublicUrl(currentUrl);
-      } catch (error) {
-        if (error instanceof UrlError) {
-          if (error.reason === "dns") {
-            logger.debug("DNS lookup failed for {url}", {
-              url: currentUrl,
-              error,
-            });
-          } else {
-            logger.error("Disallowed private URL: {url}", {
-              url: currentUrl,
-              error,
-            });
-          }
-        }
-        throw error;
-      }
-    }
+    // Opening the session validates the URL; the session is closed by
+    // fetchHop() before the next hop is opened:
+    const session = await openHop(currentUrl, options?.signal);
     visited.add(currentUrl);
 
     return await tracer.startActiveSpan(
@@ -356,22 +485,8 @@ export function getDocumentLoader(
       },
       async (span) => {
         try {
-          const request = createActivityPubRequest(currentUrl, { userAgent });
-          logRequest(logger, request);
-          const response = await fetch(request, {
-            // Since Bun has a bug that ignores the `Request.redirect` option,
-            // to work around it we specify `redirect: "manual"` here too:
-            // https://github.com/oven-sh/bun/issues/10754
-            redirect: "manual",
-            signal: options?.signal,
-          });
-          span.setAttribute("http.response.status_code", response.status);
-
-          // Follow redirects manually to get the final URL:
-          if (
-            response.status >= 300 && response.status < 400 &&
-            response.headers.has("Location")
-          ) {
+          const outcome = await fetchHop(session, currentUrl, span, options);
+          if ("location" in outcome) {
             if (redirected >= maximumRedirection) {
               logger.error(
                 "Too many redirections ({redirections}) while fetching document.",
@@ -382,10 +497,7 @@ export function getDocumentLoader(
                 `Too many redirections (${redirected + 1})`,
               );
             }
-            const redirectUrl = new URL(
-              response.headers.get("Location")!,
-              response.url === "" ? currentUrl : response.url,
-            ).href;
+            const redirectUrl = new URL(outcome.location, currentUrl).href;
             span.setAttribute("http.redirect.url", redirectUrl);
             if (visited.has(redirectUrl)) {
               logger.error(
@@ -401,26 +513,26 @@ export function getDocumentLoader(
             return await load(redirectUrl, options, redirected + 1, visited);
           }
 
-          const result = await getRemoteDocument(
-            currentUrl,
-            response,
-            async (alternateUrl) => {
-              options?.signal?.throwIfAborted();
-              if (redirected >= DEFAULT_MAX_REDIRECTION) {
-                throw new FetchError(
-                  currentUrl,
-                  `Too many redirections (${redirected + 1})`,
-                );
-              }
-              if (visited.has(alternateUrl)) {
-                throw new FetchError(
-                  currentUrl,
-                  `Redirect loop detected: ${alternateUrl}`,
-                );
-              }
-              return await load(alternateUrl, options, redirected + 1, visited);
-            },
-          );
+          let result: RemoteDocument;
+          if ("alternate" in outcome) {
+            const alternateUrl = outcome.alternate;
+            options?.signal?.throwIfAborted();
+            if (redirected >= DEFAULT_MAX_REDIRECTION) {
+              throw new FetchError(
+                currentUrl,
+                `Too many redirections (${redirected + 1})`,
+              );
+            }
+            if (visited.has(alternateUrl)) {
+              throw new FetchError(
+                currentUrl,
+                `Redirect loop detected: ${alternateUrl}`,
+              );
+            }
+            result = await load(alternateUrl, options, redirected + 1, visited);
+          } else {
+            result = outcome.document;
+          }
           span.setAttribute("docloader.document_url", result.documentUrl);
           if (result.contextUrl != null) {
             span.setAttribute("docloader.context_url", result.contextUrl);

package/src/internal.ts

@@ -0,0 +1,13 @@
+/**
+ * Internal APIs shared among Fedify packages.  They are not part of the public
+ * API and have no semver guarantees; do not use them outside Fedify.
+ *
+ * @module
+ */
+export {
+  openDirectFetch,
+  openPublicFetch,
+  type PublicFetchOptions,
+  type PublicFetchSession,
+  withPublicFetch,
+} from "./public-fetch.ts";

package/src/pin-grant.test.ts

@@ -0,0 +1,118 @@
+import { test } from "@fedify/fixture";
+import { deepStrictEqual, ok, strictEqual, throws } from "node:assert";
+import type { LookupAddress } from "node:dns";
+import {
+  issuePinGrant,
+  type PinGrant,
+  redeemPinGrant,
+  revokePinGrant,
+} from "./pin-grant.ts";
+
+const addresses: readonly LookupAddress[] = [
+  { address: "8.8.8.8", family: 4 },
+  { address: "2001:4860:4860::8888", family: 6 },
+];
+
+test("redeemPinGrant() returns what the grant was issued for", () => {
+  const url = new URL("https://pinned-host.invalid:8443/path?query");
+  const input = addresses.map((address) => ({ ...address }));
+  const grant = issuePinGrant(url, input);
+  const record = redeemPinGrant(grant, url);
+  deepStrictEqual(record.origin, "https://pinned-host.invalid:8443");
+  deepStrictEqual(record.hostname, "pinned-host.invalid");
+  deepStrictEqual(record.addresses, addresses);
+
+  // The record holds a frozen copy of the addresses:
+  input[0].address = "127.0.0.1";
+  input.push({ address: "10.0.0.1", family: 4 });
+  deepStrictEqual(redeemPinGrant(grant, url).addresses, addresses);
+  ok(Object.isFrozen(record));
+  ok(Object.isFrozen(record.addresses));
+  ok(Object.isFrozen(record.addresses[0]));
+
+  // Other paths and queries of the same origin are accepted:
+  strictEqual(
+    redeemPinGrant(grant, new URL("https://pinned-host.invalid:8443/other")),
+    record,
+  );
+});
+
+test("redeemPinGrant() strips brackets from IPv6 hostnames", () => {
+  const url = new URL("http://[2001:4860:4860::8888]/");
+  const grant = issuePinGrant(url, [addresses[1]]);
+  const record = redeemPinGrant(grant, url);
+  deepStrictEqual(record.hostname, "2001:4860:4860::8888");
+  deepStrictEqual(record.origin, "http://[2001:4860:4860::8888]");
+});
+
+test("redeemPinGrant() rejects grants not issued by issuePinGrant()", () => {
+  const url = new URL("https://pinned-host.invalid/");
+  const issued = issuePinGrant(url, addresses);
+  const forgeries: Record<string, unknown> = {
+    "object literal": {},
+    "null-prototype object": Object.create(null),
+    "frozen null-prototype object": Object.freeze(Object.create(null)),
+    "copy of a grant": { ...issued },
+    "structured clone of a grant": structuredClone(issued),
+    "object inheriting from a grant": Object.create(issued),
+    "null": null,
+    "undefined": undefined,
+    "string": "grant",
+  };
+  for (const [name, forgery] of Object.entries(forgeries)) {
+    throws(
+      () => redeemPinGrant(forgery as PinGrant, url),
+      TypeError,
+      name,
+    );
+  }
+  // The issued grant itself still works:
+  ok(redeemPinGrant(issued, url));
+});
+
+test("redeemPinGrant() rejects URLs of other origins", () => {
+  const grant = issuePinGrant(
+    new URL("https://pinned-host.invalid/"),
+    addresses,
+  );
+  for (
+    const url of [
+      "http://pinned-host.invalid/",
+      "https://other-host.invalid/",
+      "https://sub.pinned-host.invalid/",
+      "https://pinned-host.invalid:8443/",
+      "https://8.8.8.8/",
+    ]
+  ) {
+    throws(() => redeemPinGrant(grant, new URL(url)), TypeError, url);
+  }
+  // The default port is the same origin:
+  ok(redeemPinGrant(grant, new URL("https://pinned-host.invalid:443/")));
+});
+
+test("redeemPinGrant() rejects revoked grants", () => {
+  const url = new URL("https://pinned-host.invalid/");
+  const grant = issuePinGrant(url, addresses);
+  const other = issuePinGrant(url, addresses);
+  revokePinGrant(grant);
+  throws(() => redeemPinGrant(grant, url), TypeError);
+  // Revoking is idempotent, and does not affect other grants:
+  revokePinGrant(grant);
+  throws(() => redeemPinGrant(grant, url), TypeError);
+  ok(redeemPinGrant(other, url));
+});
+
+test("revokePinGrant() ignores unknown objects", () => {
+  revokePinGrant({} as PinGrant);
+  revokePinGrant(Object.create(null) as PinGrant);
+});
+
+test("issuePinGrant() returns a distinct frozen object every time", () => {
+  const url = new URL("https://pinned-host.invalid/");
+  const a = issuePinGrant(url, addresses);
+  const b = issuePinGrant(url, addresses);
+  ok(a !== b);
+  ok(Object.isFrozen(a));
+  strictEqual(Object.getPrototypeOf(a), null);
+  deepStrictEqual(Reflect.ownKeys(a), []);
+});

package/src/pin-grant.ts

@@ -0,0 +1,99 @@
+/**
+ * Unforgeable capabilities that hand the addresses validated by
+ * `openPublicFetch()` over to a connection pinner.
+ *
+ * This module is not exported from any entry point of the package.
+ * @module
+ */
+
+import type { ResolvedAddress } from "./url.ts";
+
+declare const brand: unique symbol;
+
+/**
+ * An opaque token that stands for a set of validated addresses of an origin.
+ * Only objects returned by {@link issuePinGrant} are accepted by
+ * {@link redeemPinGrant}; objects of the same shape are not.
+ */
+export type PinGrant = { readonly [brand]: never };
+
+/**
+ * What a {@link PinGrant} stands for.
+ */
+export interface PinGrantRecord {
+  /** The origin (scheme, host, and port) the grant was issued for. */
+  readonly origin: string;
+  /** The hostname the grant was issued for, without IPv6 brackets. */
+  readonly hostname: string;
+  /** The validated addresses, in lookup order. */
+  readonly addresses: readonly ResolvedAddress[];
+}
+
+interface GrantState {
+  readonly record: PinGrantRecord;
+  revoked: boolean;
+}
+
+const grants = new WeakMap<PinGrant, GrantState>();
+
+/**
+ * Issues a grant for the given URL's origin and addresses.  This does not
+ * validate the addresses; callers validate them before issuing, and pinners
+ * validate them again right before connecting.
+ * @param url The URL whose origin the grant is bound to.
+ * @param addresses The addresses to connect to.
+ * @returns A new grant.
+ */
+export function issuePinGrant(
+  url: URL,
+  addresses: readonly ResolvedAddress[],
+): PinGrant {
+  const grant = Object.freeze(Object.create(null)) as PinGrant;
+  let hostname = url.hostname;
+  if (hostname.startsWith("[") && hostname.endsWith("]")) {
+    hostname = hostname.slice(1, -1);
+  }
+  const record: PinGrantRecord = Object.freeze({
+    origin: url.origin,
+    hostname,
+    addresses: Object.freeze(
+      addresses.map(({ address, family }) =>
+        Object.freeze({ address, family })
+      ),
+    ),
+  });
+  grants.set(grant, { record, revoked: false });
+  return grant;
+}
+
+/**
+ * Checks that the given grant was issued by {@link issuePinGrant}, has not
+ * been revoked, and was issued for the origin of the given URL.
+ * @param grant The grant to redeem.
+ * @param url The URL to connect to.
+ * @returns What the grant stands for.
+ * @throws {TypeError} If the grant cannot be used for the URL.
+ */
+export function redeemPinGrant(grant: PinGrant, url: URL): PinGrantRecord {
+  const state = typeof grant === "object" && grant !== null
+    ? grants.get(grant)
+    : undefined;
+  if (state == null) throw new TypeError("Invalid pin grant.");
+  if (state.revoked) throw new TypeError("The pin grant has been revoked.");
+  if (url.origin !== state.record.origin) {
+    throw new TypeError(
+      `The pin grant for ${state.record.origin} cannot be used for ` +
+        `${url.origin}.`,
+    );
+  }
+  return state.record;
+}
+
+/**
+ * Revokes the given grant so that it can no longer be redeemed.
+ * @param grant The grant to revoke.
+ */
+export function revokePinGrant(grant: PinGrant): void {
+  const state = grants.get(grant);
+  if (state != null) state.revoked = true;
+}