@fedify/vocab-runtime 2.4.0-dev.2030 → 2.4.0-dev.2047

package/src/internal/portable-dereference.test.ts

@@ -0,0 +1,74 @@
+import { deepStrictEqual, ok, throws } from "node:assert/strict";
+import { test } from "node:test";
+import { parseIri } from "../url.ts";
+import {
+  getPortableGatewayCandidates,
+  isPortableIri,
+} from "./portable-dereference.ts";
+
+const hrefs = (urls: URL[]) => urls.map((url) => url.href);
+
+test("isPortableIri()", () => {
+  ok(isPortableIri(parseIri("ap://did:key:z6Mkabc/actor")));
+  ok(isPortableIri(parseIri("ap+ef61://did:key:z6Mkabc/actor")));
+  ok(isPortableIri(new URL("ap://did%3Akey%3Az6Mkabc/actor")));
+  ok(!isPortableIri(new URL("https://example.com/actor")));
+  ok(!isPortableIri(new URL("did:key:z6Mkabc")));
+});
+
+test("getPortableGatewayCandidates() uses explicit gateways", () => {
+  const url = parseIri(
+    "ap://did:key:z6Mkabc/actor?@gateway=https%3A%2F%2Fhint.example",
+  );
+  deepStrictEqual(
+    hrefs(getPortableGatewayCandidates(url, [
+      "https://b.example",
+      new URL("https://a.example/"),
+      "https://b.example/",
+      "http://c.example:8080",
+    ])),
+    ["https://b.example/", "https://a.example/", "http://c.example:8080/"],
+  );
+  deepStrictEqual(getPortableGatewayCandidates(url, []), []);
+  for (
+    const gateway of [
+      "https://a.example/path",
+      "https://a.example/?",
+      "https://a.example/#",
+      "https://user:pass@a.example",
+      "ftp://a.example",
+      "a.example",
+    ]
+  ) {
+    throws(() => getPortableGatewayCandidates(url, [gateway]), TypeError);
+  }
+});
+
+test("getPortableGatewayCandidates() reads @gateway hints", () => {
+  deepStrictEqual(
+    hrefs(getPortableGatewayCandidates(parseIri(
+      "ap://did:key:z6Mkabc/actor?page=1" +
+        "&@gateway=https%3A%2F%2Fa.example" +
+        "&gateways=https%3A%2F%2Flegacy.example" +
+        "&@gateway=not%20a%20URL" +
+        "&@gateway=https%3A%2F%2Fb.example%2Fpath" +
+        "&%40gateway=https%3A%2F%2Fb.example" +
+        "&@gateway=https%3A%2F%2Fa.example%2F",
+    ))),
+    ["https://a.example/", "https://b.example/"],
+  );
+  deepStrictEqual(
+    getPortableGatewayCandidates(parseIri("ap://did:key:z6Mkabc/actor")),
+    [],
+  );
+  const hints = Array.from(
+    { length: 7 },
+    (_, i) => `@gateway=https%3A%2F%2Fg${i}.example`,
+  );
+  deepStrictEqual(
+    hrefs(getPortableGatewayCandidates(
+      parseIri(`ap://did:key:z6Mkabc/actor?${hints.join("&")}`),
+    )),
+    [0, 1, 2, 3, 4].map((i) => `https://g${i}.example/`),
+  );
+});

package/src/internal/portable-dereference.ts

@@ -0,0 +1,426 @@
+import { getLogger } from "@logtape/logtape";
+import {
+  type Span,
+  SpanStatusCode,
+  type TracerProvider,
+} from "@opentelemetry/api";
+import preloadedContexts from "../contexts.ts";
+import type { DocumentLoader, RemoteDocument } from "../docloader.ts";
+import jsonld from "../jsonld.ts";
+import type { PortableObjectVerifier } from "../portable.ts";
+import {
+  canonicalizePortableUri,
+  formatIri,
+  parseIri,
+  toCompatibleEf61Id,
+} from "../url.ts";
+
+const logger = getLogger(["fedify", "vocab", "gateway"]);
+
+/**
+ * The maximum number of `@gateway` location hints to try for one reference.
+ * Hints come from possibly untrusted documents, so they are bounded to keep
+ * a single accessor call from fanning out to many servers.
+ */
+const MAX_GATEWAY_HINTS = 5;
+
+const LOCATION_HINT_PARAMETER = "@gateway";
+
+// The same baseline contexts that Fedify's proof verifier always resolves
+// from its built-in copies (see getNormalizationContextLoader() in
+// @fedify/fedify).  Serving them identically here keeps the identity check,
+// the verifier, and the parser on the same context documents.  Other
+// contexts go through the caller's context loader.
+const BASELINE_CONTEXT_URLS: ReadonlySet<string> = new Set([
+  "https://w3id.org/identity/v1",
+  "https://www.w3.org/ns/activitystreams",
+  "https://w3id.org/security/v1",
+  "https://w3id.org/security/data-integrity/v1",
+]);
+
+/**
+ * Checks whether a URL is an FEP-ef61 portable ActivityPub IRI.
+ *
+ * @internal Technically exported for generated vocabulary classes, but not
+ * part of the public API contract.  This is not considered public API for
+ * Semantic Versioning decisions.
+ */
+export function isPortableIri(url: URL): boolean {
+  return url.protocol === "ap+ef61:" || url.protocol === "ap:";
+}
+
+/**
+ * Picks the ordered list of FEP-ef61 gateways to fetch a portable IRI from.
+ *
+ * If `gateways` is given, it is used as is (even when empty), and `@gateway`
+ * location hints in the IRI are ignored.  Otherwise, up to
+ * {@link MAX_GATEWAY_HINTS} valid `@gateway` hints are used.  Duplicate
+ * gateways are dropped in both cases.
+ *
+ * @throws {TypeError} If an explicit gateway is not an HTTP(S) origin.
+ * @internal
+ */
+export function getPortableGatewayCandidates(
+  url: URL,
+  gateways?: readonly (string | URL)[],
+): URL[] {
+  const candidates: URL[] = [];
+  const seen = new Set<string>();
+  const add = (gateway: URL) => {
+    if (seen.has(gateway.href)) return;
+    seen.add(gateway.href);
+    candidates.push(gateway);
+  };
+  if (gateways != null) {
+    for (const gateway of gateways) {
+      const parsed = parseGatewayOrigin(gateway);
+      if (parsed == null) {
+        throw new TypeError(
+          "FEP-ef61 gateways must be HTTP(S) origins with no credentials, " +
+            "path, query, or fragment: " + String(gateway),
+        );
+      }
+      add(parsed);
+    }
+    return candidates;
+  }
+  for (
+    const hint of new URLSearchParams(url.search).getAll(
+      LOCATION_HINT_PARAMETER,
+    )
+  ) {
+    if (candidates.length >= MAX_GATEWAY_HINTS) break;
+    const parsed = parseGatewayOrigin(hint);
+    if (parsed == null) {
+      logger.debug(
+        "Ignoring an invalid FEP-ef61 gateway hint {hint} in {url}.",
+        { hint, url: formatIri(url) },
+      );
+      continue;
+    }
+    add(parsed);
+  }
+  return candidates;
+}
+
+function parseGatewayOrigin(gateway: string | URL): URL | null {
+  let url: URL;
+  if (gateway instanceof URL) url = new URL(gateway.href);
+  else if (typeof gateway === "string" && URL.canParse(gateway)) {
+    url = new URL(gateway);
+  } else return null;
+  // Comparing href with the origin also rejects credentials, a path, and
+  // query and fragment components, including empty ? and # delimiters.
+  if (
+    (url.protocol !== "http:" && url.protocol !== "https:") ||
+    url.href !== `${url.origin}/`
+  ) {
+    return null;
+  }
+  return url;
+}
+
+/**
+ * Creates a context loader that returns the same context documents for the
+ * whole dereference operation, so that the identity check, the proof
+ * verifier, and the parser interpret the fetched document identically even
+ * if the underlying loader is nondeterministic.  Failed loads are not
+ * remembered, so a transient failure does not affect the next gateway.
+ *
+ * The parsed object keeps the loader for its own later dereferences, so
+ * `release()` turns it into a plain pass-through to the underlying loader
+ * once the operation is over.
+ */
+function createSnapshotContextLoader(
+  contextLoader: DocumentLoader,
+): { loader: DocumentLoader; release: () => void } {
+  const cache = new Map<string, Promise<RemoteDocument>>();
+  let released = false;
+  const release = () => {
+    released = true;
+    cache.clear();
+  };
+  const loader: DocumentLoader = async (url, options) => {
+    if (released) return await contextLoader(url, options);
+    const key = URL.canParse(url) ? new URL(url).href : url;
+    if (BASELINE_CONTEXT_URLS.has(key)) {
+      return {
+        contextUrl: null,
+        document: structuredClone(preloadedContexts[key]),
+        documentUrl: key,
+      };
+    }
+    let promise = cache.get(key);
+    if (promise == null) {
+      const loading = contextLoader(url, options).then((document) =>
+        structuredClone(document)
+      );
+      promise = loading;
+      cache.set(key, loading);
+      loading.catch(() => {
+        if (cache.get(key) === loading) cache.delete(key);
+      });
+    }
+    return structuredClone(await promise);
+  };
+  return { loader, release };
+}
+
+/**
+ * Options for {@link dereferencePortableIri}.
+ *
+ * @internal Technically exported for generated vocabulary classes, but not
+ * part of the public API contract.  This is not considered public API for
+ * Semantic Versioning decisions.
+ */
+export interface DereferencePortableIriOptions<T> {
+  documentLoader: DocumentLoader;
+  contextLoader: DocumentLoader;
+  tracerProvider: TracerProvider;
+  gateways?: readonly (string | URL)[];
+  verifyPortableObject?: PortableObjectVerifier;
+  suppressError?: boolean;
+  crossOrigin?: "ignore" | "throw" | "trust";
+  /**
+   * The signal for cancelling the dereference.  It is passed to the document
+   * loader for requests to gateways, and checked between attempts.  Requests
+   * for JSON-LD contexts and verification methods may not be cancelled.
+   */
+  signal?: AbortSignal;
+  parse: (
+    document: unknown,
+    options: { contextLoader: DocumentLoader; baseUrl: URL },
+  ) => Promise<T>;
+  span?: Span;
+}
+
+/**
+ * The error thrown by {@link dereferencePortableIri} under
+ * `crossOrigin: "throw"` when gateways returned objects but none of them
+ * satisfied the identity and proof checks.
+ *
+ * @internal Not part of the public API contract.
+ */
+export class PortableObjectRejectedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "PortableObjectRejectedError";
+  }
+}
+
+type Attempt =
+  | { readonly type: "rejected"; readonly source: string }
+  | { readonly type: "error"; readonly error: unknown };
+
+class PortableObjectRejection {
+  constructor(
+    readonly message: string,
+    readonly values: Record<string, unknown> = {},
+  ) {}
+}
+
+/**
+ * Dereferences an FEP-ef61 portable IRI through its gateways.
+ *
+ * Gateways are tried one by one until one of them returns a single-node
+ * document whose `@id`, if any, canonically matches the requested IRI and
+ * which passes `verifyPortableObject`.  When there is no gateway to try, the document
+ * loader is asked for the portable IRI itself, and its result is validated
+ * the same way.
+ *
+ * @returns The parsed object, or `null` if no valid object was retrieved and
+ *          `suppressError` is set or the failure was a rejected object
+ *          (unless `crossOrigin` is `"throw"`).
+ * @throws {TypeError} If `gateways` has an invalid entry (always), if no
+ *                     verifier is given, or if the IRI cannot be turned into
+ *                     a gateway URL (unless `suppressError` is set).
+ * @internal Technically exported for generated vocabulary classes, but not
+ * part of the public API contract.  This is not considered public API for
+ * Semantic Versioning decisions.
+ */
+export async function dereferencePortableIri<T extends { id: URL | null }>(
+  url: URL,
+  options: DereferencePortableIriOptions<T>,
+): Promise<T | null> {
+  url = parseIri(url);
+  const lookupUrl = formatIri(url);
+  const { span } = options;
+  // Invalid gateways are a programming error, so they are never suppressed:
+  const gateways = getPortableGatewayCandidates(url, options.gateways);
+  const fail = (error: unknown): null => {
+    span?.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+    if (options.suppressError) {
+      logger.error("Failed to dereference {url}: {error}", {
+        url: lookupUrl,
+        error,
+      });
+      return null;
+    }
+    throw error;
+  };
+  const verify = options.verifyPortableObject;
+  if (verify == null) {
+    return fail(
+      new TypeError(
+        "Dereferencing the portable object " + lookupUrl + " requires the " +
+          "verifyPortableObject option, e.g., verifyPortableObjectProof() " +
+          "from @fedify/fedify.",
+      ),
+    );
+  }
+  const requestUrls: { url: string; gateway: URL | null }[] = [];
+  let expectedId: string;
+  try {
+    expectedId = canonicalizePortableUri(lookupUrl);
+    for (const gateway of gateways) {
+      requestUrls.push({ url: toCompatibleEf61Id(url, gateway).href, gateway });
+    }
+  } catch (error) {
+    return fail(error);
+  }
+  if (requestUrls.length < 1) {
+    // No gateway to ask; custom document loaders may still know how to
+    // retrieve the portable IRI itself:
+    requestUrls.push({ url: lookupUrl, gateway: null });
+  }
+  const snapshot = createSnapshotContextLoader(options.contextLoader);
+  const contextLoader = snapshot.loader;
+  const attempts: Attempt[] = [];
+  const { signal } = options;
+  try {
+    for (const { url: requestUrl, gateway } of requestUrls) {
+      signal?.throwIfAborted();
+      let remoteDocument: RemoteDocument;
+      try {
+        remoteDocument = await options.documentLoader(
+          requestUrl,
+          signal == null ? undefined : { signal },
+        );
+      } catch (error) {
+        signal?.throwIfAborted();
+        logger.debug("Failed to fetch {url} from {requestUrl}: {error}", {
+          url: lookupUrl,
+          requestUrl,
+          error,
+        });
+        attempts.push({ type: "error", error });
+        continue;
+      }
+      const { document } = remoteDocument;
+      try {
+        await checkPortableObjectId(document, expectedId, contextLoader);
+        let result: { readonly verified: boolean };
+        try {
+          result = await verify(document, {
+            documentLoader: options.documentLoader,
+            contextLoader,
+            tracerProvider: options.tracerProvider,
+          });
+        } catch (error) {
+          throw new PortableObjectRejection(
+            "the proof verifier failed: {error}",
+            { error },
+          );
+        }
+        if (!result.verified) {
+          throw new PortableObjectRejection(
+            "it does not satisfy the FEP-ef61 proof policy: {result}",
+            { result },
+          );
+        }
+        const object = await options.parse(document, {
+          contextLoader,
+          baseUrl: url,
+        });
+        signal?.throwIfAborted();
+        if (gateway != null) {
+          span?.setAttribute("activitypub.gateway", gateway.href);
+        }
+        return object;
+      } catch (error) {
+        signal?.throwIfAborted();
+        if (error instanceof PortableObjectRejection) {
+          logger.warn(
+            "Rejected the portable object {url} served from {requestUrl}, " +
+              "because " + error.message,
+            { ...error.values, url: lookupUrl, requestUrl },
+          );
+          attempts.push({ type: "rejected", source: requestUrl });
+          continue;
+        }
+        logger.debug("Failed to parse {url} from {requestUrl}: {error}", {
+          url: lookupUrl,
+          requestUrl,
+          error,
+        });
+        attempts.push({ type: "error", error });
+      }
+    }
+  } finally {
+    snapshot.release();
+  }
+  const rejected = attempts.filter((a) => a.type === "rejected");
+  if (rejected.length > 0) {
+    const message = "No gateway returned a valid portable object for " +
+      lookupUrl + "; refusing to return the object.  Objects retrieved " +
+      "from: " + rejected.map((a) => a.source).join(", ") + ".";
+    if (options.suppressError || options.crossOrigin !== "throw") {
+      span?.setStatus({ code: SpanStatusCode.ERROR, message });
+      logger.warn(message);
+      return null;
+    }
+    throw new PortableObjectRejectedError(message);
+  }
+  const errors = attempts.flatMap((a) => a.type === "error" ? [a.error] : []);
+  return fail(
+    errors.length === 1 ? errors[0] : new AggregateError(
+      errors,
+      "Failed to dereference " + lookupUrl + " through any of its " +
+        "gateways: " + requestUrls.map((r) => r.url).join(", "),
+    ),
+  );
+}
+
+async function checkPortableObjectId(
+  document: unknown,
+  expectedId: string,
+  contextLoader: DocumentLoader,
+): Promise<void> {
+  if (
+    document == null || typeof document !== "object" || Array.isArray(document)
+  ) {
+    throw new PortableObjectRejection("it is not a single JSON object");
+  }
+  const expanded = await jsonld.expand(document, {
+    documentLoader: contextLoader,
+    keepFreeFloatingNodes: true,
+  });
+  if (expanded.length !== 1) {
+    throw new PortableObjectRejection(
+      "it does not have exactly one top-level node",
+    );
+  }
+  const id = expanded[0]["@id"];
+  // A document without @id, such as a proof document, is left to the
+  // verifier; verifyPortableObjectProof() rejects it, because there is no
+  // portable ID to check its proof against:
+  if (id == null) return;
+  if (typeof id !== "string") {
+    throw new PortableObjectRejection("it has an invalid @id");
+  }
+  // Compare the raw @id rather than a parsed URL, because URL parsing
+  // normalizes opaque path segments (e.g., dot segments):
+  let actualId: string | null;
+  try {
+    actualId = canonicalizePortableUri(id);
+  } catch (error) {
+    if (!(error instanceof TypeError)) throw error;
+    actualId = null;
+  }
+  if (actualId !== expectedId) {
+    throw new PortableObjectRejection(
+      "its @id ({objectId}) does not match the requested portable ID",
+      { objectId: id },
+    );
+  }
+}

package/src/mod.ts

@@ -45,6 +45,10 @@ export {
   verifyHashlink,
 } from "./digest.ts";
 export { LanguageString } from "./langstr.ts";
+export type {
+  PortableObjectVerifier,
+  PortableObjectVerifierOptions,
+} from "./portable.ts";
 export {
   decodeMultibase,
   encodeMultibase,
@@ -68,6 +72,7 @@ export {
   canonicalizePortableUri,
   expandIPv6Address,
   formatIri,
+  fromCompatibleEf61Id,
   getFe34Origin,
   haveSameFe34Origin,
   haveSameIriOrigin,
@@ -77,6 +82,7 @@ export {
   parseGatewayUrl,
   parseIri,
   parseJsonLdId,
+  toCompatibleEf61Id,
   UrlError,
   validatePublicUrl,
 } from "./url.ts";

package/src/portable.ts

@@ -0,0 +1,52 @@
+import type { TracerProvider } from "@opentelemetry/api";
+import type { DocumentLoader } from "./docloader.ts";
+
+/**
+ * Options passed to a {@link PortableObjectVerifier}.
+ * @since 2.4.0
+ */
+export interface PortableObjectVerifierOptions {
+  /**
+   * The document loader for fetching remote documents, such as verification
+   * methods that are not `did:key` URLs.
+   */
+  readonly documentLoader?: DocumentLoader;
+
+  /**
+   * The document loader for fetching remote JSON-LD contexts.  During
+   * gateway dereferencing, this loader returns the same context documents
+   * that the vocabulary parser sees.
+   */
+  readonly contextLoader?: DocumentLoader;
+
+  /**
+   * The OpenTelemetry tracer provider.
+   */
+  readonly tracerProvider?: TracerProvider;
+}
+
+/**
+ * A function that applies the [FEP-ef61] proof policy to a portable object
+ * fetched through a gateway.
+ *
+ * It receives the fetched JSON-LD document as is, and resolves to an object
+ * whose `verified` property tells whether the document carries a valid
+ * [FEP-8b32] Object Integrity Proof made by the DID in its portable ID.
+ * `verifyPortableObjectProof()` from `@fedify/fedify` satisfies this type, so
+ * it can be passed as the `verifyPortableObject` option of generated
+ * vocabulary accessors such as `Activity.getObject()`.
+ *
+ * [FEP-ef61]: https://w3id.org/fep/ef61
+ * [FEP-8b32]: https://w3id.org/fep/8b32
+ *
+ * @param document The fetched JSON-LD document.
+ * @param options Loaders and tracing options for verification.
+ * @returns An object whose `verified` property is `true` if and only if the
+ *          document satisfies the proof policy.  Other properties, such as
+ *          a failure reason, are logged when verification fails.
+ * @since 2.4.0
+ */
+export type PortableObjectVerifier = (
+  document: unknown,
+  options: PortableObjectVerifierOptions,
+) => Promise<{ readonly verified: boolean }>;

package/src/request.test.ts

@@ -2,7 +2,7 @@ import { deepStrictEqual } from "node:assert";
 import process from "node:process";
 import { test } from "node:test";
 import metadata from "../deno.json" with { type: "json" };
-import { getUserAgent } from "./request.ts";
+import { createActivityPubRequest, getUserAgent } from "./request.ts";
 
 test("getUserAgent()", () => {
   if ("Deno" in globalThis) {
@@ -91,3 +91,12 @@ test("getUserAgent()", () => {
     );
   }
 });
+
+test("createActivityPubRequest() asks for the ActivityStreams profile", () => {
+  const request = createActivityPubRequest("https://example.com/object");
+  deepStrictEqual(
+    request.headers.get("Accept"),
+    "application/activity+json, " +
+      'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+  );
+});

package/src/request.ts

@@ -31,6 +31,14 @@ export class FetchError extends Error {
   }
 }
 
+/**
+ * The `Accept` header value for fetching ActivityPub objects.  ActivityPub
+ * and FEP-ef61 gateways require the ActivityStreams profile on the JSON-LD
+ * media type.
+ */
+const ACTIVITYPUB_ACCEPT =
+  'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"';
+
 /**
  * Options for creating a request.
  * @internal
@@ -52,7 +60,7 @@ export function createActivityPubRequest(
 ): Request {
   return new Request(url, {
     headers: {
-      Accept: "application/activity+json, application/ld+json",
+      Accept: ACTIVITYPUB_ACCEPT,
       "User-Agent": typeof options.userAgent === "string"
         ? options.userAgent
         : getUserAgent(options.userAgent),