@fedify/vocab-runtime 2.0.0-pr.451.1730

package/src/docloader.test.ts

@@ -0,0 +1,365 @@
+import fetchMock from "fetch-mock";
+import { deepStrictEqual, rejects } from "node:assert";
+import { test } from "node:test";
+import preloadedContexts from "./contexts.ts";
+import { getDocumentLoader } from "./docloader.ts";
+import { FetchError } from "./request.ts";
+import { UrlError } from "./url.ts";
+
+test("new FetchError()", () => {
+  const e = new FetchError("https://example.com/", "An error message.");
+  deepStrictEqual(e.name, "FetchError");
+  deepStrictEqual(e.url, new URL("https://example.com/"));
+  deepStrictEqual(e.message, "https://example.com/: An error message.");
+
+  const e2 = new FetchError(new URL("https://example.org/"));
+  deepStrictEqual(e2.url, new URL("https://example.org/"));
+  deepStrictEqual(e2.message, "https://example.org/");
+});
+
+test("getDocumentLoader()", async (t) => {
+  const fetchDocumentLoader = getDocumentLoader();
+
+  fetchMock.spyGlobal();
+
+  fetchMock.get("https://example.com/object", {
+    body: {
+      "@context": "https://www.w3.org/ns/activitystreams",
+      id: "https://example.com/object",
+      name: "Fetched object",
+      type: "Object",
+    },
+  });
+
+  await t.test("ok", async () => {
+    deepStrictEqual(await fetchDocumentLoader("https://example.com/object"), {
+      contextUrl: null,
+      documentUrl: "https://example.com/object",
+      document: {
+        "@context": "https://www.w3.org/ns/activitystreams",
+        id: "https://example.com/object",
+        name: "Fetched object",
+        type: "Object",
+      },
+    });
+  });
+
+  fetchMock.get("https://example.com/link-ctx", {
+    body: {
+      id: "https://example.com/link-ctx",
+      name: "Fetched object",
+      type: "Object",
+    },
+    headers: {
+      "Content-Type": "application/activity+json",
+      Link: "<https://www.w3.org/ns/activitystreams>; " +
+        'rel="http://www.w3.org/ns/json-ld#context"; ' +
+        'type="application/ld+json"',
+    },
+  });
+
+  fetchMock.get("https://example.com/link-obj", {
+    headers: {
+      "Content-Type": "text/html; charset=utf-8",
+      Link: '<https://example.com/object>; rel="alternate"; ' +
+        'type="application/activity+json"',
+    },
+  });
+
+  fetchMock.get("https://example.com/link-obj-relative", {
+    headers: {
+      "Content-Type": "text/html; charset=utf-8",
+      Link: '</object>; rel="alternate"; ' +
+        'type="application/activity+json"',
+    },
+  });
+
+  fetchMock.get("https://example.com/obj-w-wrong-link", {
+    body: {
+      "@context": "https://www.w3.org/ns/activitystreams",
+      id: "https://example.com/obj-w-wrong-link",
+      name: "Fetched object",
+      type: "Object",
+    },
+    headers: {
+      "Content-Type": "text/html; charset=utf-8",
+      Link: '<https://example.com/object>; rel="alternate"; ' +
+        'type="application/ld+json; profile="https://www.w3.org/ns/activitystreams""',
+    },
+  });
+
+  await t.test("Link header", async () => {
+    deepStrictEqual(await fetchDocumentLoader("https://example.com/link-ctx"), {
+      contextUrl: "https://www.w3.org/ns/activitystreams",
+      documentUrl: "https://example.com/link-ctx",
+      document: {
+        id: "https://example.com/link-ctx",
+        name: "Fetched object",
+        type: "Object",
+      },
+    });
+
+    deepStrictEqual(await fetchDocumentLoader("https://example.com/link-obj"), {
+      contextUrl: null,
+      documentUrl: "https://example.com/object",
+      document: {
+        "@context": "https://www.w3.org/ns/activitystreams",
+        id: "https://example.com/object",
+        name: "Fetched object",
+        type: "Object",
+      },
+    });
+  });
+
+  await t.test("Link header relative url", async () => {
+    deepStrictEqual(await fetchDocumentLoader("https://example.com/link-ctx"), {
+      contextUrl: "https://www.w3.org/ns/activitystreams",
+      documentUrl: "https://example.com/link-ctx",
+      document: {
+        id: "https://example.com/link-ctx",
+        name: "Fetched object",
+        type: "Object",
+      },
+    });
+
+    deepStrictEqual(
+      await fetchDocumentLoader("https://example.com/link-obj-relative"),
+      {
+        contextUrl: null,
+        documentUrl: "https://example.com/object",
+        document: {
+          "@context": "https://www.w3.org/ns/activitystreams",
+          id: "https://example.com/object",
+          name: "Fetched object",
+          type: "Object",
+        },
+      },
+    );
+  });
+
+  await t.test("wrong Link header syntax", async () => {
+    deepStrictEqual(
+      await fetchDocumentLoader("https://example.com/obj-w-wrong-link"),
+      {
+        contextUrl: null,
+        documentUrl: "https://example.com/obj-w-wrong-link",
+        document: {
+          "@context": "https://www.w3.org/ns/activitystreams",
+          id: "https://example.com/obj-w-wrong-link",
+          name: "Fetched object",
+          type: "Object",
+        },
+      },
+    );
+  });
+
+  fetchMock.get("https://example.com/html-link", {
+    body: `<html>
+        <head>
+          <meta charset=utf-8>
+          <link
+            rel=alternate
+            type='application/activity+json'
+            href="https://example.com/object">
+        </head>
+      </html>`,
+    headers: { "Content-Type": "text/html; charset=utf-8" },
+  });
+
+  await t.test("HTML <link>", async () => {
+    deepStrictEqual(
+      await fetchDocumentLoader("https://example.com/html-link"),
+      {
+        contextUrl: null,
+        documentUrl: "https://example.com/object",
+        document: {
+          "@context": "https://www.w3.org/ns/activitystreams",
+          id: "https://example.com/object",
+          name: "Fetched object",
+          type: "Object",
+        },
+      },
+    );
+  });
+
+  fetchMock.get("https://example.com/xhtml-link", {
+    body: `<html>
+        <head>
+          <meta charset="utf-8" />
+          <link
+            rel=alternate
+            type="application/activity+json"
+            href="https://example.com/object" />
+        </head>
+      </html>`,
+    headers: { "Content-Type": "application/xhtml+xml; charset=utf-8" },
+  });
+
+  await t.test("XHTML <link>", async () => {
+    deepStrictEqual(
+      await fetchDocumentLoader("https://example.com/xhtml-link"),
+      {
+        contextUrl: null,
+        documentUrl: "https://example.com/object",
+        document: {
+          "@context": "https://www.w3.org/ns/activitystreams",
+          id: "https://example.com/object",
+          name: "Fetched object",
+          type: "Object",
+        },
+      },
+    );
+  });
+
+  fetchMock.get("https://example.com/html-a", {
+    body: `<html>
+        <head>
+          <meta charset=utf-8>
+        </head>
+        <body>
+          <a
+            rel=alternate
+            type=application/activity+json
+            href=https://example.com/object>test</a>
+        </body>
+      </html>`,
+    headers: { "Content-Type": "text/html; charset=utf-8" },
+  });
+
+  await t.test("HTML <a>", async () => {
+    deepStrictEqual(await fetchDocumentLoader("https://example.com/html-a"), {
+      contextUrl: null,
+      documentUrl: "https://example.com/object",
+      document: {
+        "@context": "https://www.w3.org/ns/activitystreams",
+        id: "https://example.com/object",
+        name: "Fetched object",
+        type: "Object",
+      },
+    });
+  });
+
+  fetchMock.get("https://example.com/wrong-content-type", {
+    body: {
+      "@context": "https://www.w3.org/ns/activitystreams",
+      id: "https://example.com/wrong-content-type",
+      name: "Fetched object",
+      type: "Object",
+    },
+    headers: { "Content-Type": "text/html; charset=utf-8" },
+  });
+
+  await t.test("Wrong Content-Type", async () => {
+    deepStrictEqual(
+      await fetchDocumentLoader("https://example.com/wrong-content-type"),
+      {
+        contextUrl: null,
+        documentUrl: "https://example.com/wrong-content-type",
+        document: {
+          "@context": "https://www.w3.org/ns/activitystreams",
+          id: "https://example.com/wrong-content-type",
+          name: "Fetched object",
+          type: "Object",
+        },
+      },
+    );
+  });
+
+  fetchMock.get("https://example.com/404", { status: 404 });
+
+  await t.test("not ok", async () => {
+    await rejects(
+      () => fetchDocumentLoader("https://example.com/404"),
+      FetchError,
+      "HTTP 404: https://example.com/404",
+    );
+  });
+
+  await t.test("preloaded contexts", async () => {
+    for (const [url, document] of Object.entries(preloadedContexts)) {
+      deepStrictEqual(await fetchDocumentLoader(url), {
+        contextUrl: null,
+        documentUrl: url,
+        document,
+      });
+    }
+  });
+
+  await t.test("deny non-HTTP/HTTPS", async () => {
+    await rejects(
+      () => fetchDocumentLoader("ftp://localhost"),
+      UrlError,
+    );
+  });
+
+  fetchMock.get("https://example.com/localhost-redirect", {
+    status: 302,
+    headers: { Location: "https://localhost/object" },
+  });
+
+  fetchMock.get("https://example.com/localhost-link", {
+    body: `<html>
+        <head>
+          <meta charset=utf-8>
+          <link
+            rel=alternate
+            type='application/activity+json'
+            href="https://localhost/object">
+        </head>
+      </html>`,
+    headers: { "Content-Type": "text/html; charset=utf-8" },
+  });
+
+  fetchMock.get("https://localhost/object", {
+    body: {
+      "@context": "https://www.w3.org/ns/activitystreams",
+      id: "https://localhost/object",
+      name: "Fetched object",
+      type: "Object",
+    },
+  });
+
+  await t.test("allowPrivateAddress: false", async () => {
+    await rejects(
+      () => fetchDocumentLoader("https://localhost/object"),
+      UrlError,
+    );
+    await rejects(
+      () => fetchDocumentLoader("https://example.com/localhost-redirect"),
+      UrlError,
+    );
+    await rejects(
+      () => fetchDocumentLoader("https://example.com/localhost-link"),
+      UrlError,
+    );
+  });
+
+  const fetchDocumentLoader2 = getDocumentLoader({ allowPrivateAddress: true });
+
+  await t.test("allowPrivateAddress: true", async () => {
+    const expected = {
+      contextUrl: null,
+      documentUrl: "https://localhost/object",
+      document: {
+        "@context": "https://www.w3.org/ns/activitystreams",
+        id: "https://localhost/object",
+        name: "Fetched object",
+        type: "Object",
+      },
+    };
+    deepStrictEqual(
+      await fetchDocumentLoader2("https://localhost/object"),
+      expected,
+    );
+    deepStrictEqual(
+      await fetchDocumentLoader2("https://example.com/localhost-redirect"),
+      expected,
+    );
+    deepStrictEqual(
+      await fetchDocumentLoader2("https://example.com/localhost-link"),
+      expected,
+    );
+  });
+
+  fetchMock.hardReset();
+});

package/src/docloader.ts

@@ -0,0 +1,311 @@
+import { getLogger } from "@logtape/logtape";
+import preloadedContexts from "./contexts.ts";
+import { HttpHeaderLink } from "./link.ts";
+import {
+  createRequest,
+  FetchError,
+  type GetUserAgentOptions,
+  logRequest,
+} from "./request.ts";
+import { UrlError, validatePublicUrl } from "./url.ts";
+
+const logger = getLogger(["fedify", "runtime", "docloader"]);
+
+/**
+ * A remote JSON-LD document and its context fetched by
+ * a {@link DocumentLoader}.
+ */
+export interface RemoteDocument {
+  /**
+   * The URL of the context document.
+   */
+  contextUrl: string | null;
+
+  /**
+   * The fetched JSON-LD document.
+   */
+  document: unknown;
+
+  /**
+   * The URL of the fetched document.
+   */
+  documentUrl: string;
+}
+
+/**
+ * Options for {@link DocumentLoader}.
+ * @since 1.8.0
+ */
+export interface DocumentLoaderOptions {
+  /**
+   * An `AbortSignal` for cancellation.
+   * @since 1.8.0
+   */
+  signal?: AbortSignal;
+}
+
+/**
+ * A JSON-LD document loader that fetches documents from the Web.
+ * @param url The URL of the document to load.
+ * @param options The options for the document loader.
+ * @returns The loaded remote document.
+ */
+export type DocumentLoader = (
+  url: string,
+  options?: DocumentLoaderOptions,
+) => Promise<RemoteDocument>;
+
+/**
+ * A factory function that creates a {@link DocumentLoader} with options.
+ * @param options The options for the document loader.
+ * @returns The document loader.
+ * @since 1.4.0
+ */
+export type DocumentLoaderFactory = (
+  options?: DocumentLoaderFactoryOptions,
+) => DocumentLoader;
+
+/**
+ * Options for {@link DocumentLoaderFactory}.
+ * @see {@link DocumentLoaderFactory}
+ * @see {@link AuthenticatedDocumentLoaderFactory}
+ * @since 1.4.0
+ */
+export interface DocumentLoaderFactoryOptions {
+  /**
+   * Whether to allow fetching private network addresses.
+   * Turned off by default.
+   * @default `false``
+   */
+  allowPrivateAddress?: boolean;
+
+  /**
+   * Options for making `User-Agent` string.
+   * If a string is given, it is used as the `User-Agent` header value.
+   * If an object is given, it is passed to {@link getUserAgent} function.
+   */
+  userAgent?: GetUserAgentOptions | string;
+}
+
+/**
+ * A factory function that creates an authenticated {@link DocumentLoader} for
+ * a given identity.  This is used for fetching documents that require
+ * authentication.
+ * @param identity The identity to create the document loader for.
+ *                 The actor's key pair.
+ * @param options The options for the document loader.
+ * @returns The authenticated document loader.
+ * @since 0.4.0
+ */
+export type AuthenticatedDocumentLoaderFactory = (
+  identity: { keyId: URL; privateKey: CryptoKey },
+  options?: DocumentLoaderFactoryOptions,
+) => DocumentLoader;
+
+/**
+ * Gets a {@link RemoteDocument} from the given response.
+ * @param url The URL of the document to load.
+ * @param response The response to get the document from.
+ * @param fetch The function to fetch the document.
+ * @returns The loaded remote document.
+ * @throws {FetchError} If the response is not OK.
+ * @internal
+ */
+export async function getRemoteDocument(
+  url: string,
+  response: Response,
+  fetch: (
+    url: string,
+    options?: DocumentLoaderOptions,
+  ) => Promise<RemoteDocument>,
+): Promise<RemoteDocument> {
+  const documentUrl = response.url === "" ? url : response.url;
+  const docUrl = new URL(documentUrl);
+  if (!response.ok) {
+    logger.error(
+      "Failed to fetch document: {status} {url} {headers}",
+      {
+        status: response.status,
+        url: documentUrl,
+        headers: Object.fromEntries(response.headers.entries()),
+      },
+    );
+    throw new FetchError(
+      documentUrl,
+      `HTTP ${response.status}: ${documentUrl}`,
+    );
+  }
+  const contentType = response.headers.get("Content-Type");
+  const jsonLd = contentType == null ||
+    contentType === "application/activity+json" ||
+    contentType.startsWith("application/activity+json;") ||
+    contentType === "application/ld+json" ||
+    contentType.startsWith("application/ld+json;");
+  const linkHeader = response.headers.get("Link");
+  let contextUrl: string | null = null;
+  if (linkHeader != null) {
+    let link: HttpHeaderLink;
+    try {
+      link = new HttpHeaderLink(linkHeader);
+    } catch (e) {
+      if (e instanceof SyntaxError) {
+        link = new HttpHeaderLink();
+      } else {
+        throw e;
+      }
+    }
+    if (jsonLd) {
+      const entries = link.getByRel("http://www.w3.org/ns/json-ld#context");
+      for (const [uri, params] of entries) {
+        if ("type" in params && params.type === "application/ld+json") {
+          contextUrl = uri;
+          break;
+        }
+      }
+    } else {
+      const entries = link.getByRel("alternate");
+      for (const [uri, params] of entries) {
+        const altUri = new URL(uri, docUrl);
+        if (
+          "type" in params &&
+          (params.type === "application/activity+json" ||
+            params.type === "application/ld+json" ||
+            params.type.startsWith("application/ld+json;")) &&
+          altUri.href !== docUrl.href
+        ) {
+          logger.debug(
+            "Found alternate document: {alternateUrl} from {url}",
+            { alternateUrl: altUri.href, url: documentUrl },
+          );
+          return await fetch(altUri.href);
+        }
+      }
+    }
+  }
+  let document: unknown;
+  if (
+    !jsonLd &&
+    (contentType === "text/html" || contentType?.startsWith("text/html;") ||
+      contentType === "application/xhtml+xml" ||
+      contentType?.startsWith("application/xhtml+xml;"))
+  ) {
+    const p =
+      /<(a|link)((\s+[a-z][a-z:_-]*=("[^"]*"|'[^']*'|[^\s>]+))+)\s*\/?>/ig;
+    const p2 = /\s+([a-z][a-z:_-]*)=("([^"]*)"|'([^']*)'|([^\s>]+))/ig;
+    const html = await response.text();
+    let m: RegExpExecArray | null;
+    const rawAttribs: string[] = [];
+    while ((m = p.exec(html)) !== null) rawAttribs.push(m[2]);
+    for (const rawAttrs of rawAttribs) {
+      let m2: RegExpExecArray | null;
+      const attribs: Record<string, string> = {};
+      while ((m2 = p2.exec(rawAttrs)) !== null) {
+        const key = m2[1].toLowerCase();
+        const value = m2[3] ?? m2[4] ?? m2[5] ?? "";
+        attribs[key] = value;
+      }
+      if (
+        attribs.rel === "alternate" && "type" in attribs && (
+          attribs.type === "application/activity+json" ||
+          attribs.type === "application/ld+json" ||
+          attribs.type.startsWith("application/ld+json;")
+        ) && "href" in attribs &&
+        new URL(attribs.href, docUrl).href !== docUrl.href
+      ) {
+        logger.debug(
+          "Found alternate document: {alternateUrl} from {url}",
+          { alternateUrl: attribs.href, url: documentUrl },
+        );
+        return await fetch(new URL(attribs.href, docUrl).href);
+      }
+    }
+    document = JSON.parse(html);
+  } else {
+    document = await response.json();
+  }
+  logger.debug(
+    "Fetched document: {status} {url} {headers}",
+    {
+      status: response.status,
+      url: documentUrl,
+      headers: Object.fromEntries(response.headers.entries()),
+    },
+  );
+  return { contextUrl, document, documentUrl };
+}
+
+/**
+ * Options for {@link getDocumentLoader}.
+ * @since 1.3.0
+ */
+export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
+  /**
+   * Whether to preload the frequently used contexts.
+   */
+  skipPreloadedContexts?: boolean;
+}
+
+/**
+ * Creates a JSON-LD document loader that utilizes the browser's `fetch` API.
+ *
+ * The created loader preloads the below frequently used contexts by default
+ * (unless `options.ignorePreloadedContexts` is set to `true`):
+ *
+ * - <https://www.w3.org/ns/activitystreams>
+ * - <https://w3id.org/security/v1>
+ * - <https://w3id.org/security/data-integrity/v1>
+ * - <https://www.w3.org/ns/did/v1>
+ * - <https://w3id.org/security/multikey/v1>
+ * - <https://purl.archive.org/socialweb/webfinger>
+ * - <http://schema.org/>
+ * @param options Options for the document loader.
+ * @returns The document loader.
+ * @since 1.3.0
+ */
+export function getDocumentLoader(
+  { allowPrivateAddress, skipPreloadedContexts, userAgent }:
+    GetDocumentLoaderOptions = {},
+): DocumentLoader {
+  async function load(
+    url: string,
+    options?: DocumentLoaderOptions,
+  ): Promise<RemoteDocument> {
+    options?.signal?.throwIfAborted();
+    if (!skipPreloadedContexts && url in preloadedContexts) {
+      logger.debug("Using preloaded context: {url}.", { url });
+      return {
+        contextUrl: null,
+        document: preloadedContexts[url],
+        documentUrl: url,
+      };
+    }
+    if (!allowPrivateAddress) {
+      try {
+        await validatePublicUrl(url);
+      } catch (error) {
+        if (error instanceof UrlError) {
+          logger.error("Disallowed private URL: {url}", { url, error });
+        }
+        throw error;
+      }
+    }
+    const request = createRequest(url, { userAgent });
+    logRequest(logger, request);
+    const response = await fetch(request, {
+      // Since Bun has a bug that ignores the `Request.redirect` option,
+      // to work around it we specify `redirect: "manual"` here too:
+      // https://github.com/oven-sh/bun/issues/10754
+      redirect: "manual",
+      signal: options?.signal,
+    });
+    // Follow redirects manually to get the final URL:
+    if (
+      response.status >= 300 && response.status < 400 &&
+      response.headers.has("Location")
+    ) {
+      return load(response.headers.get("Location")!, options);
+    }
+    return getRemoteDocument(url, response, load);
+  }
+  return load;
+}