@fedify/vocab-runtime 2.5.0-dev.2448 → 2.5.0-dev.2494

package/src/docloader.ts

@@ -1,5 +1,5 @@
 import { getLogger } from "@logtape/logtape";
-import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
+import { type Span, SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
 import metadata from "../deno.json" with { type: "json" };
 import {
   BodyTooLargeError,
@@ -9,13 +9,18 @@ import {
 } from "./body.ts";
 import preloadedContexts from "./contexts.ts";
 import { HttpHeaderLink } from "./link.ts";
+import {
+  openDirectFetch,
+  openPublicFetch,
+  type PublicFetchSession,
+} from "./public-fetch.ts";
 import {
   createActivityPubRequest,
   FetchError,
   type GetUserAgentOptions,
   logRequest,
 } from "./request.ts";
-import { UrlError, validatePublicUrl } from "./url.ts";
+import { UrlError } from "./url.ts";
 
 const logger = getLogger(["fedify", "runtime", "docloader"]);
 const DEFAULT_MAX_REDIRECTION = 20;
@@ -187,7 +192,8 @@ async function bufferErrorResponse(
     try {
       await readBoundedBytes(response, MAX_ERROR_BODY_SIZE, url);
     } catch (error) {
-      await clone.body?.cancel().catch(() => {});
+      // Cancelling a tee branch must not block scoped session cleanup.
+      clone.body?.cancel().catch(() => {});
       if (signal?.aborted) throw error;
       logger.debug(
         "Failed to read the error response body from {url}: {error}",
@@ -222,7 +228,9 @@ async function bufferErrorResponse(
  * Gets a {@link RemoteDocument} from the given response.
  * @param url The URL of the document to load.
  * @param response The response to get the document from.
- * @param fetch The function to fetch the document.
+ * @param fetch The function to fetch the document.  It is called with the URL
+ *              of an alternate document, if any, after the response body has
+ *              been either read or its cancellation has been started.
  * @param options The options for loading the document.
  * @returns The loaded remote document.
  * @throws {FetchError} If the response is not OK.
@@ -237,7 +245,55 @@ export async function getRemoteDocument(
   ) => Promise<RemoteDocument>,
   options?: DocumentLoaderOptions,
 ): Promise<RemoteDocument> {
-  const documentUrl = response.url === "" ? url : response.url;
+  const result = await parseRemoteDocument(url, response, options);
+  if ("alternate" in result) return await fetch(result.alternate, options);
+  return result.document;
+}
+
+/**
+ * Starts cancelling the body of a response without waiting for it to settle.
+ * The cancellation of a branch of a tee (e.g., a body cloned by a fetch mock)
+ * settles only after the other branch is cancelled too, per the Streams
+ * standard, so awaiting it could block forever.  Failures are only logged.
+ * @param response The response whose body is cancelled.
+ * @param url The URL of the response, for logging.
+ */
+function cancelResponseBody(response: Response, url: string): void {
+  response.body?.cancel().catch((error) => {
+    logger.debug(
+      "Failed to cancel the response body from {url}: {error}",
+      { url, error },
+    );
+  });
+}
+
+/**
+ * The result of {@link parseRemoteDocument}: either the loaded document, or
+ * the URL of an alternate document to load instead.
+ */
+type ParsedRemoteDocument =
+  | { readonly document: RemoteDocument }
+  | { readonly alternate: string };
+
+/**
+ * Gets a {@link RemoteDocument} from the given response, or the URL of an
+ * alternate document to load instead.  When an alternate document is found,
+ * the response body has been either read or its cancellation has been started
+ * (without waiting for it), so that the alternate document can be loaded after
+ * the response's session is closed.
+ * @param url The URL of the document to load.
+ * @param response The response to get the document from.
+ * @returns The loaded remote document, or the alternate document URL.
+ * @throws {FetchError} If the response is not OK.
+ */
+async function parseRemoteDocument(
+  url: string,
+  response: Response,
+  options?: DocumentLoaderOptions,
+): Promise<ParsedRemoteDocument> {
+  // Since the response URL is an IP address URL on runtimes that pin
+  // connections, the request URL is used instead:
+  const documentUrl = url;
   const docUrl = new URL(documentUrl);
   if (!response.ok) {
     if (options?.suppressError) {
@@ -308,7 +364,8 @@ export async function getRemoteDocument(
             "Found alternate document: {alternateUrl} from {url}",
             { alternateUrl: altUri.href, url: documentUrl },
           );
-          return await fetch(altUri.href, options);
+          cancelResponseBody(response, documentUrl);
+          return { alternate: altUri.href };
         }
       }
     }
@@ -370,7 +427,7 @@ export async function getRemoteDocument(
           "Found alternate document: {alternateUrl} from {url}",
           { alternateUrl: attribs.href, url: documentUrl },
         );
-        return await fetch(new URL(attribs.href, docUrl).href, options);
+        return { alternate: new URL(attribs.href, docUrl).href };
       }
     }
     try {
@@ -397,7 +454,7 @@ export async function getRemoteDocument(
       headers: Object.fromEntries(response.headers.entries()),
     },
   );
-  return { contextUrl, document, documentUrl };
+  return { document: { contextUrl, document, documentUrl } };
 }
 
 /**
@@ -537,6 +594,37 @@ export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
  * @since 1.3.0
  */
 export function getDocumentLoader(
+  options: GetDocumentLoaderOptions = {},
+): DocumentLoader {
+  return createDocumentLoader(options);
+}
+
+/**
+ * The dependencies of {@link createDocumentLoader}.
+ */
+export interface DocumentLoaderDependencies {
+  /**
+   * Opens the fetch session for a single hop, instead of
+   * {@link openPublicFetch} (or {@link openDirectFetch} if private addresses
+   * are allowed).
+   */
+  readonly openHop?: (url: string) => Promise<PublicFetchSession>;
+}
+
+/**
+ * The outcome of a single hop of {@link createDocumentLoader}, decided while
+ * its fetch session is open.
+ */
+type HopOutcome = ParsedRemoteDocument | { readonly location: string };
+
+/**
+ * The implementation of {@link getDocumentLoader}, with injectable
+ * dependencies.  This is not exported from any entry point.
+ * @param options Options for the document loader.
+ * @param deps The dependencies.
+ * @returns The document loader.
+ */
+export function createDocumentLoader(
   {
     allowPrivateAddress,
     maxRedirection,
@@ -544,12 +632,76 @@ export function getDocumentLoader(
     timeout,
     userAgent,
   }: GetDocumentLoaderOptions = {},
+  deps: DocumentLoaderDependencies = {},
 ): DocumentLoader {
   const resolvedTimeout = resolveDocumentLoaderTimeout(timeout);
   const tracerProvider = trace.getTracerProvider();
   const tracer = tracerProvider.getTracer(metadata.name, metadata.version);
   const maximumRedirection = maxRedirection ?? DEFAULT_MAX_REDIRECTION;
 
+  async function openHop(
+    url: string,
+    options?: DocumentLoaderOptions,
+  ): Promise<PublicFetchSession> {
+    try {
+      const session = deps.openHop != null
+        ? await deps.openHop(url)
+        : allowPrivateAddress
+        ? await openDirectFetch(url)
+        : await openPublicFetch(url, { signal: options?.signal });
+      if (options?.signal?.aborted) {
+        await session.close();
+        options.signal.throwIfAborted();
+      }
+      return session;
+    } catch (error) {
+      if (error instanceof UrlError) {
+        if (error.reason === "dns") {
+          logger.debug("DNS lookup failed for {url}", { url, error });
+        } else {
+          logger[options?.suppressError ? "warn" : "error"](
+            "Disallowed private URL: {url}",
+            { url, error },
+          );
+        }
+      }
+      throw error;
+    }
+  }
+
+  async function fetchHop(
+    session: PublicFetchSession,
+    url: string,
+    span: Span,
+    options?: DocumentLoaderOptions,
+  ): Promise<HopOutcome> {
+    try {
+      const request = createActivityPubRequest(url, { userAgent });
+      logRequest(logger, request);
+      const response = await session.fetch(request, {
+        // Since Bun has a bug that ignores the `Request.redirect` option,
+        // to work around it we specify `redirect: "manual"` here too:
+        // https://github.com/oven-sh/bun/issues/10754
+        redirect: "manual",
+        signal: options?.signal,
+      });
+      span.setAttribute("http.response.status_code", response.status);
+
+      // Follow redirects manually to get the final URL:
+      if (
+        response.status >= 300 && response.status < 400 &&
+        response.headers.has("Location")
+      ) {
+        const location = response.headers.get("Location")!;
+        cancelResponseBody(response, url);
+        return { location };
+      }
+      return await parseRemoteDocument(url, response, options);
+    } finally {
+      await session.close();
+    }
+  }
+
   async function load(
     url: string,
     options?: DocumentLoaderOptions,
@@ -575,32 +727,9 @@ export function getDocumentLoader(
         documentUrl: currentUrl,
       };
     }
-    if (!allowPrivateAddress) {
-      try {
-        await validatePublicUrl(currentUrl);
-      } catch (error) {
-        if (error instanceof UrlError) {
-          if (error.reason === "dns") {
-            logger.debug("DNS lookup failed for {url}", {
-              url: currentUrl,
-              error,
-            });
-          } else {
-            logger[options?.suppressError ? "warn" : "error"](
-              "Disallowed private URL: {url}",
-              {
-                url: currentUrl,
-                error,
-              },
-            );
-          }
-        }
-        throw error;
-      }
-      // The DNS lookup cannot be aborted, so do not go on if the call was
-      // aborted or timed out in the meantime:
-      options?.signal?.throwIfAborted();
-    }
+    // Opening the session validates the URL; the session is closed by
+    // fetchHop() before the next hop is opened:
+    const session = await openHop(currentUrl, options);
     visited.add(currentUrl);
 
     return await tracer.startActiveSpan(
@@ -613,22 +742,8 @@ export function getDocumentLoader(
       },
       async (span) => {
         try {
-          const request = createActivityPubRequest(currentUrl, { userAgent });
-          logRequest(logger, request);
-          const response = await fetch(request, {
-            // Since Bun has a bug that ignores the `Request.redirect` option,
-            // to work around it we specify `redirect: "manual"` here too:
-            // https://github.com/oven-sh/bun/issues/10754
-            redirect: "manual",
-            signal: options?.signal,
-          });
-          span.setAttribute("http.response.status_code", response.status);
-
-          // Follow redirects manually to get the final URL:
-          if (
-            response.status >= 300 && response.status < 400 &&
-            response.headers.has("Location")
-          ) {
+          const outcome = await fetchHop(session, currentUrl, span, options);
+          if ("location" in outcome) {
             if (redirected >= maximumRedirection) {
               logger[options?.suppressError ? "warn" : "error"](
                 "Too many redirections ({redirections}) while fetching document.",
@@ -639,10 +754,7 @@ export function getDocumentLoader(
                 `Too many redirections (${redirected + 1})`,
               );
             }
-            const redirectUrl = new URL(
-              response.headers.get("Location")!,
-              response.url === "" ? currentUrl : response.url,
-            ).href;
+            const redirectUrl = new URL(outcome.location, currentUrl).href;
             span.setAttribute("http.redirect.url", redirectUrl);
             if (visited.has(redirectUrl)) {
               logger[options?.suppressError ? "warn" : "error"](
@@ -658,27 +770,26 @@ export function getDocumentLoader(
             return await load(redirectUrl, options, redirected + 1, visited);
           }
 
-          const result = await getRemoteDocument(
-            currentUrl,
-            response,
-            async (alternateUrl) => {
-              options?.signal?.throwIfAborted();
-              if (redirected >= DEFAULT_MAX_REDIRECTION) {
-                throw new FetchError(
-                  currentUrl,
-                  `Too many redirections (${redirected + 1})`,
-                );
-              }
-              if (visited.has(alternateUrl)) {
-                throw new FetchError(
-                  currentUrl,
-                  `Redirect loop detected: ${alternateUrl}`,
-                );
-              }
-              return await load(alternateUrl, options, redirected + 1, visited);
-            },
-            options,
-          );
+          let result: RemoteDocument;
+          if ("alternate" in outcome) {
+            const alternateUrl = outcome.alternate;
+            options?.signal?.throwIfAborted();
+            if (redirected >= DEFAULT_MAX_REDIRECTION) {
+              throw new FetchError(
+                currentUrl,
+                `Too many redirections (${redirected + 1})`,
+              );
+            }
+            if (visited.has(alternateUrl)) {
+              throw new FetchError(
+                currentUrl,
+                `Redirect loop detected: ${alternateUrl}`,
+              );
+            }
+            result = await load(alternateUrl, options, redirected + 1, visited);
+          } else {
+            result = outcome.document;
+          }
           span.setAttribute("docloader.document_url", result.documentUrl);
           if (result.contextUrl != null) {
             span.setAttribute("docloader.context_url", result.contextUrl);

package/src/internal.ts

@@ -0,0 +1,13 @@
+/**
+ * Internal APIs shared among Fedify packages.  They are not part of the public
+ * API and have no semver guarantees; do not use them outside Fedify.
+ *
+ * @module
+ */
+export {
+  openDirectFetch,
+  openPublicFetch,
+  type PublicFetchOptions,
+  type PublicFetchSession,
+  withPublicFetch,
+} from "./public-fetch.ts";

package/src/pin-grant.test.ts

@@ -0,0 +1,118 @@
+import { test } from "@fedify/fixture";
+import { deepStrictEqual, ok, strictEqual, throws } from "node:assert";
+import type { LookupAddress } from "node:dns";
+import {
+  issuePinGrant,
+  type PinGrant,
+  redeemPinGrant,
+  revokePinGrant,
+} from "./pin-grant.ts";
+
+const addresses: readonly LookupAddress[] = [
+  { address: "8.8.8.8", family: 4 },
+  { address: "2001:4860:4860::8888", family: 6 },
+];
+
+test("redeemPinGrant() returns what the grant was issued for", () => {
+  const url = new URL("https://pinned-host.invalid:8443/path?query");
+  const input = addresses.map((address) => ({ ...address }));
+  const grant = issuePinGrant(url, input);
+  const record = redeemPinGrant(grant, url);
+  deepStrictEqual(record.origin, "https://pinned-host.invalid:8443");
+  deepStrictEqual(record.hostname, "pinned-host.invalid");
+  deepStrictEqual(record.addresses, addresses);
+
+  // The record holds a frozen copy of the addresses:
+  input[0].address = "127.0.0.1";
+  input.push({ address: "10.0.0.1", family: 4 });
+  deepStrictEqual(redeemPinGrant(grant, url).addresses, addresses);
+  ok(Object.isFrozen(record));
+  ok(Object.isFrozen(record.addresses));
+  ok(Object.isFrozen(record.addresses[0]));
+
+  // Other paths and queries of the same origin are accepted:
+  strictEqual(
+    redeemPinGrant(grant, new URL("https://pinned-host.invalid:8443/other")),
+    record,
+  );
+});
+
+test("redeemPinGrant() strips brackets from IPv6 hostnames", () => {
+  const url = new URL("http://[2001:4860:4860::8888]/");
+  const grant = issuePinGrant(url, [addresses[1]]);
+  const record = redeemPinGrant(grant, url);
+  deepStrictEqual(record.hostname, "2001:4860:4860::8888");
+  deepStrictEqual(record.origin, "http://[2001:4860:4860::8888]");
+});
+
+test("redeemPinGrant() rejects grants not issued by issuePinGrant()", () => {
+  const url = new URL("https://pinned-host.invalid/");
+  const issued = issuePinGrant(url, addresses);
+  const forgeries: Record<string, unknown> = {
+    "object literal": {},
+    "null-prototype object": Object.create(null),
+    "frozen null-prototype object": Object.freeze(Object.create(null)),
+    "copy of a grant": { ...issued },
+    "structured clone of a grant": structuredClone(issued),
+    "object inheriting from a grant": Object.create(issued),
+    "null": null,
+    "undefined": undefined,
+    "string": "grant",
+  };
+  for (const [name, forgery] of Object.entries(forgeries)) {
+    throws(
+      () => redeemPinGrant(forgery as PinGrant, url),
+      TypeError,
+      name,
+    );
+  }
+  // The issued grant itself still works:
+  ok(redeemPinGrant(issued, url));
+});
+
+test("redeemPinGrant() rejects URLs of other origins", () => {
+  const grant = issuePinGrant(
+    new URL("https://pinned-host.invalid/"),
+    addresses,
+  );
+  for (
+    const url of [
+      "http://pinned-host.invalid/",
+      "https://other-host.invalid/",
+      "https://sub.pinned-host.invalid/",
+      "https://pinned-host.invalid:8443/",
+      "https://8.8.8.8/",
+    ]
+  ) {
+    throws(() => redeemPinGrant(grant, new URL(url)), TypeError, url);
+  }
+  // The default port is the same origin:
+  ok(redeemPinGrant(grant, new URL("https://pinned-host.invalid:443/")));
+});
+
+test("redeemPinGrant() rejects revoked grants", () => {
+  const url = new URL("https://pinned-host.invalid/");
+  const grant = issuePinGrant(url, addresses);
+  const other = issuePinGrant(url, addresses);
+  revokePinGrant(grant);
+  throws(() => redeemPinGrant(grant, url), TypeError);
+  // Revoking is idempotent, and does not affect other grants:
+  revokePinGrant(grant);
+  throws(() => redeemPinGrant(grant, url), TypeError);
+  ok(redeemPinGrant(other, url));
+});
+
+test("revokePinGrant() ignores unknown objects", () => {
+  revokePinGrant({} as PinGrant);
+  revokePinGrant(Object.create(null) as PinGrant);
+});
+
+test("issuePinGrant() returns a distinct frozen object every time", () => {
+  const url = new URL("https://pinned-host.invalid/");
+  const a = issuePinGrant(url, addresses);
+  const b = issuePinGrant(url, addresses);
+  ok(a !== b);
+  ok(Object.isFrozen(a));
+  strictEqual(Object.getPrototypeOf(a), null);
+  deepStrictEqual(Reflect.ownKeys(a), []);
+});

package/src/pin-grant.ts

@@ -0,0 +1,99 @@
+/**
+ * Unforgeable capabilities that hand the addresses validated by
+ * `openPublicFetch()` over to a connection pinner.
+ *
+ * This module is not exported from any entry point of the package.
+ * @module
+ */
+
+import type { ResolvedAddress } from "./url.ts";
+
+declare const brand: unique symbol;
+
+/**
+ * An opaque token that stands for a set of validated addresses of an origin.
+ * Only objects returned by {@link issuePinGrant} are accepted by
+ * {@link redeemPinGrant}; objects of the same shape are not.
+ */
+export type PinGrant = { readonly [brand]: never };
+
+/**
+ * What a {@link PinGrant} stands for.
+ */
+export interface PinGrantRecord {
+  /** The origin (scheme, host, and port) the grant was issued for. */
+  readonly origin: string;
+  /** The hostname the grant was issued for, without IPv6 brackets. */
+  readonly hostname: string;
+  /** The validated addresses, in lookup order. */
+  readonly addresses: readonly ResolvedAddress[];
+}
+
+interface GrantState {
+  readonly record: PinGrantRecord;
+  revoked: boolean;
+}
+
+const grants = new WeakMap<PinGrant, GrantState>();
+
+/**
+ * Issues a grant for the given URL's origin and addresses.  This does not
+ * validate the addresses; callers validate them before issuing, and pinners
+ * validate them again right before connecting.
+ * @param url The URL whose origin the grant is bound to.
+ * @param addresses The addresses to connect to.
+ * @returns A new grant.
+ */
+export function issuePinGrant(
+  url: URL,
+  addresses: readonly ResolvedAddress[],
+): PinGrant {
+  const grant = Object.freeze(Object.create(null)) as PinGrant;
+  let hostname = url.hostname;
+  if (hostname.startsWith("[") && hostname.endsWith("]")) {
+    hostname = hostname.slice(1, -1);
+  }
+  const record: PinGrantRecord = Object.freeze({
+    origin: url.origin,
+    hostname,
+    addresses: Object.freeze(
+      addresses.map(({ address, family }) =>
+        Object.freeze({ address, family })
+      ),
+    ),
+  });
+  grants.set(grant, { record, revoked: false });
+  return grant;
+}
+
+/**
+ * Checks that the given grant was issued by {@link issuePinGrant}, has not
+ * been revoked, and was issued for the origin of the given URL.
+ * @param grant The grant to redeem.
+ * @param url The URL to connect to.
+ * @returns What the grant stands for.
+ * @throws {TypeError} If the grant cannot be used for the URL.
+ */
+export function redeemPinGrant(grant: PinGrant, url: URL): PinGrantRecord {
+  const state = typeof grant === "object" && grant !== null
+    ? grants.get(grant)
+    : undefined;
+  if (state == null) throw new TypeError("Invalid pin grant.");
+  if (state.revoked) throw new TypeError("The pin grant has been revoked.");
+  if (url.origin !== state.record.origin) {
+    throw new TypeError(
+      `The pin grant for ${state.record.origin} cannot be used for ` +
+        `${url.origin}.`,
+    );
+  }
+  return state.record;
+}
+
+/**
+ * Revokes the given grant so that it can no longer be redeemed.
+ * @param grant The grant to revoke.
+ */
+export function revokePinGrant(grant: PinGrant): void {
+  const state = grants.get(grant);
+  if (state != null) state.revoked = true;
+}