@fedify/vocab-runtime 2.4.0-dev.2169 → 2.4.0-dev.2190

package/src/docloader.test.ts

@@ -1,13 +1,22 @@
 import { configure, type LogRecord, reset } from "@logtape/logtape";
 import fetchMock from "fetch-mock";
-import { deepStrictEqual, ok, rejects } from "node:assert";
+import { deepStrictEqual, ok, rejects, throws } from "node:assert";
 import dns from "node:dns/promises";
 import { test } from "node:test";
-import { createServer } from "node:http";
+import {
+  createServer,
+  type IncomingMessage,
+  type ServerResponse,
+} from "node:http";
 import { gzipSync } from "node:zlib";
 import preloadedContexts from "./contexts.ts";
 import cidV1Context from "./contexts/cid-v1.json" with { type: "json" };
-import { getDocumentLoader, getRemoteDocument } from "./docloader.ts";
+import {
+  getDocumentLoader,
+  getRemoteDocument,
+  resolveDocumentLoaderTimeout,
+  withDocumentLoaderTimeout,
+} from "./docloader.ts";
 import { FetchError } from "./request.ts";
 import { UrlError } from "./url.ts";
 
@@ -1118,3 +1127,332 @@ test("getDocumentLoader() rejects cancellation before fetching", async () => {
     fetchMock.hardReset();
   }
 });
+
+async function withServer<T>(
+  handler: (request: IncomingMessage, response: ServerResponse) => void,
+  callback: (baseUrl: string) => Promise<T>,
+): Promise<T> {
+  const server = createServer(handler);
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", resolve);
+  });
+  try {
+    const address = server.address();
+    ok(address != null && typeof address !== "string");
+    return await callback(`http://127.0.0.1:${address.port}`);
+  } finally {
+    await new Promise<void>((resolve, reject) => {
+      server.close((error) => error == null ? resolve() : reject(error));
+      server.closeAllConnections();
+    });
+  }
+}
+
+function isTimeoutFetchError(error: unknown, url: string): boolean {
+  ok(error instanceof FetchError, String(error));
+  deepStrictEqual(error.url.href, url);
+  deepStrictEqual(error.response, undefined);
+  ok(error.cause instanceof DOMException);
+  deepStrictEqual(error.cause.name, "TimeoutError");
+  return true;
+}
+
+test("resolveDocumentLoaderTimeout() validates timeouts", () => {
+  deepStrictEqual(resolveDocumentLoaderTimeout(undefined), 10_000);
+  deepStrictEqual(resolveDocumentLoaderTimeout(null), null);
+  deepStrictEqual(resolveDocumentLoaderTimeout(1500), 1500);
+  deepStrictEqual(resolveDocumentLoaderTimeout(0.5), 1);
+  deepStrictEqual(resolveDocumentLoaderTimeout(2_147_483_647), 2_147_483_647);
+  for (const invalid of [0, -1, NaN, Infinity, 2_147_483_648]) {
+    throws(() => resolveDocumentLoaderTimeout(invalid), RangeError);
+  }
+  throws(() => getDocumentLoader({ timeout: 0 }), RangeError);
+});
+
+test("getDocumentLoader() times out a request without a response", async () => {
+  await withServer(() => {
+    // Never respond.
+  }, async (baseUrl) => {
+    const url = `${baseUrl}/hang`;
+    const loader = getDocumentLoader({
+      allowPrivateAddress: true,
+      timeout: 100,
+    });
+    await rejects(loader(url), (e) => isTimeoutFetchError(e, url));
+  });
+});
+
+test("getDocumentLoader() times out a stalled response body", async () => {
+  for (const status of [200, 404]) {
+    await withServer((_, response) => {
+      response.writeHead(status, {
+        "Content-Type": "application/activity+json",
+      });
+      response.write('{"id":');
+      // Never finish the body.
+    }, async (baseUrl) => {
+      const url = `${baseUrl}/stalled`;
+      const loader = getDocumentLoader({
+        allowPrivateAddress: true,
+        timeout: 200,
+      });
+      await rejects(loader(url), (e) => isTimeoutFetchError(e, url));
+    });
+  }
+});
+
+test("getDocumentLoader() shares the timeout across redirects", async () => {
+  let requests = 0;
+  await withServer((request, response) => {
+    requests++;
+    const index = Number(request.url?.split("/").at(-1));
+    setTimeout(() => {
+      if (index >= 15) {
+        response.writeHead(200, {
+          "Content-Type": "application/activity+json",
+        });
+        response.end('{"done":true}');
+        return;
+      }
+      response.writeHead(302, { Location: `/redirect/${index + 1}` });
+      response.end();
+    }, 50);
+  }, async (baseUrl) => {
+    const url = `${baseUrl}/redirect/0`;
+    // Each hop is well within the timeout, but the whole chain is not:
+    const loader = getDocumentLoader({
+      allowPrivateAddress: true,
+      timeout: 300,
+    });
+    await rejects(loader(url), (e) => isTimeoutFetchError(e, url));
+    ok(requests < 16, `requests: ${requests}`);
+    // Without a timeout, the chain completes:
+    const unbounded = getDocumentLoader({
+      allowPrivateAddress: true,
+      timeout: null,
+    });
+    deepStrictEqual((await unbounded(url)).document, { done: true });
+  });
+});
+
+test("getDocumentLoader() shares the timeout across alternates", async (t) => {
+  for (const html of [false, true]) {
+    await t.test(html ? "HTML" : "Link header", async () => {
+      await withServer((request, response) => {
+        const index = Number(request.url?.split("/").at(-1));
+        setTimeout(() => {
+          if (index >= 15) {
+            response.writeHead(200, {
+              "Content-Type": "application/activity+json",
+            });
+            response.end('{"done":true}');
+            return;
+          }
+          const next = `/alternate/${index + 1}`;
+          response.writeHead(
+            200,
+            html ? { "Content-Type": "text/html" } : {
+              "Content-Type": "text/plain",
+              Link:
+                `<${next}>; rel="alternate"; type="application/activity+json"`,
+            },
+          );
+          response.end(
+            html
+              ? `<link rel="alternate" type="application/activity+json" href="${next}">`
+              : "not JSON",
+          );
+        }, 50);
+      }, async (baseUrl) => {
+        const url = `${baseUrl}/alternate/0`;
+        const loader = getDocumentLoader({
+          allowPrivateAddress: true,
+          timeout: 300,
+        });
+        await rejects(loader(url), (e) => isTimeoutFetchError(e, url));
+      });
+    });
+  }
+});
+
+test("getDocumentLoader() lets the caller's signal abort a request", async () => {
+  await withServer(() => {
+    // Never respond.
+  }, async (baseUrl) => {
+    const url = `${baseUrl}/hang`;
+    for (const timeout of [undefined, 60_000, null]) {
+      const loader = getDocumentLoader({ allowPrivateAddress: true, timeout });
+      const controller = new AbortController();
+      const reason = new Error("Canceled by the caller");
+      setTimeout(() => controller.abort(reason), 50);
+      await rejects(loader(url, { signal: controller.signal }), (e) => {
+        deepStrictEqual(e, reason);
+        return true;
+      });
+    }
+    // The caller's signal wins even if the timeout fires too:
+    const loader = getDocumentLoader({
+      allowPrivateAddress: true,
+      timeout: 50,
+    });
+    const controller = new AbortController();
+    const reason = new Error("Canceled by the caller");
+    controller.abort(reason);
+    await rejects(loader(url, { signal: controller.signal }), (e) => {
+      deepStrictEqual(e, reason);
+      return true;
+    });
+  });
+});
+
+test("getDocumentLoader() bounds DNS lookups that ignore the timeout", async (t) => {
+  // The validator skips DNS when Deno has no network permission.
+  if (
+    "Deno" in globalThis &&
+    (await Deno.permissions.query({ name: "net" })).state !== "granted"
+  ) {
+    t.skip("requires the net permission");
+    return;
+  }
+  fetchMock.mockGlobal();
+  let requests = 0;
+  const url = "https://slow-dns.example/object";
+  fetchMock.get(url, () => {
+    requests++;
+    return Response.json({ id: url });
+  });
+  // Stubbing works only because url.ts uses the default node:dns/promises
+  // import; see the FIXME there.
+  const originalLookup = dns.lookup;
+  let resolveLookup: () => void = () => {};
+  const lookedUp = new Promise<void>((resolve) => resolveLookup = resolve);
+  dns.lookup = (() =>
+    new Promise((resolve) =>
+      setTimeout(() => {
+        resolve([{ address: "93.184.215.14", family: 4 }]);
+        resolveLookup();
+      }, 200)
+    )) as typeof dns.lookup;
+  try {
+    const loader = getDocumentLoader({ timeout: 50 });
+    await rejects(loader(url), (e) => isTimeoutFetchError(e, url));
+    // A late DNS answer must not start the request anymore:
+    await lookedUp;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    deepStrictEqual(requests, 0);
+  } finally {
+    dns.lookup = originalLookup;
+    fetchMock.hardReset();
+  }
+});
+
+test("withDocumentLoaderTimeout() cleans up after each call", async () => {
+  const signals: AbortSignal[] = [];
+  const loader = withDocumentLoaderTimeout((url, options) => {
+    signals.push(options!.signal!);
+    return Promise.resolve({
+      contextUrl: null,
+      document: {},
+      documentUrl: url,
+    });
+  }, 50);
+  const results = await Promise.all([
+    loader("https://example.com/a"),
+    loader("https://example.com/b"),
+  ]);
+  deepStrictEqual(results.map((r) => r.documentUrl), [
+    "https://example.com/a",
+    "https://example.com/b",
+  ]);
+  ok(signals[0] !== signals[1]);
+  // The timers are cleared, so the signals are never aborted:
+  await new Promise((resolve) => setTimeout(resolve, 100));
+  ok(signals.every((signal) => !signal.aborted));
+  // A loader that is given no timeout is returned as is:
+  const inner = () => Promise.reject(new Error("unused"));
+  ok(withDocumentLoaderTimeout(inner, null) === inner);
+});
+
+test("getRemoteDocument() keeps error bodies byte for byte", async () => {
+  const bytes = new Uint8Array([0xef, 0xbb, 0xbf, 0x4e, 0x6f, 0xff, 0xfe]);
+  const url = "https://example.com/error";
+  let error: unknown;
+  await rejects(
+    getRemoteDocument(
+      url,
+      new Response(bytes, { status: 404 }),
+      () => Promise.reject(new Error("unused")),
+    ),
+    (e) => {
+      error = e;
+      return true;
+    },
+  );
+  ok(error instanceof FetchError);
+  ok(error.response != null);
+  deepStrictEqual(error.response.status, 404);
+  deepStrictEqual(new Uint8Array(await error.response.arrayBuffer()), bytes);
+});
+
+test("getRemoteDocument() keeps metadata of bodiless or large errors", async () => {
+  const url = "https://example.com/error";
+  for (
+    const response of [
+      new Response(null, { status: 304, headers: { ETag: '"a"' } }),
+      new Response(new Uint8Array(1024 * 1024 + 1), {
+        status: 500,
+        headers: { ETag: '"a"' },
+      }),
+    ]
+  ) {
+    await rejects(
+      getRemoteDocument(
+        url,
+        response,
+        () => Promise.reject(new Error("unused")),
+      ),
+      (e) => {
+        ok(e instanceof FetchError);
+        ok(e.response != null);
+        deepStrictEqual(e.response.status, response.status);
+        deepStrictEqual(e.response.headers.get("ETag"), '"a"');
+        deepStrictEqual(e.response.body, null);
+        return true;
+      },
+    );
+  }
+});
+
+test("getDocumentLoader() reports statuses that Response cannot hold", async () => {
+  await withServer((_, response) => {
+    response.writeHead(999);
+    response.end("Request denied");
+  }, async (baseUrl) => {
+    const url = `${baseUrl}/denied`;
+    const loader = getDocumentLoader({ allowPrivateAddress: true });
+    let error: unknown;
+    await rejects(loader(url), (e) => {
+      error = e;
+      return true;
+    });
+    ok(error instanceof FetchError, String(error));
+    deepStrictEqual(error.response?.status, 999);
+    deepStrictEqual(await error.response.text(), "Request denied");
+  });
+});
+
+test("getDocumentLoader() times out a stalled body of such a status", async () => {
+  await withServer((_, response) => {
+    response.writeHead(999);
+    response.write("Request");
+    // Never finish the body.
+  }, async (baseUrl) => {
+    const url = `${baseUrl}/stalled-denied`;
+    const loader = getDocumentLoader({
+      allowPrivateAddress: true,
+      timeout: 200,
+    });
+    await rejects(loader(url), (e) => isTimeoutFetchError(e, url));
+  });
+});

package/src/docloader.ts

@@ -1,7 +1,12 @@
 import { getLogger } from "@logtape/logtape";
 import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
 import metadata from "../deno.json" with { type: "json" };
-import { BodyTooLargeError, MAX_BODY_SIZE, readBoundedText } from "./body.ts";
+import {
+  BodyTooLargeError,
+  MAX_BODY_SIZE,
+  readBoundedBytes,
+  readBoundedText,
+} from "./body.ts";
 import preloadedContexts from "./contexts.ts";
 import { HttpHeaderLink } from "./link.ts";
 import {
@@ -15,6 +20,11 @@ import { UrlError, validatePublicUrl } from "./url.ts";
 const logger = getLogger(["fedify", "runtime", "docloader"]);
 const DEFAULT_MAX_REDIRECTION = 20;
 const MAX_HTML_SIZE = 1024 * 1024; // 1MB
+const MAX_ERROR_BODY_SIZE = 1024 * 1024; // 1MB
+const DEFAULT_TIMEOUT = 10_000; // 10 seconds
+// The maximum delay setTimeout() accepts; larger values fire immediately
+// on some runtimes:
+const MAX_TIMEOUT = 2_147_483_647;
 
 /**
  * A remote JSON-LD document and its context fetched by
@@ -104,6 +114,29 @@ export interface DocumentLoaderFactoryOptions {
    * @since 2.2.0
    */
   maxRedirection?: number;
+
+  /**
+   * The timeout in milliseconds for each call of the created document
+   * loader.  The timeout is shared by all the steps of a call, including
+   * URL validation, every HTTP redirect and alternate document link it
+   * follows, retries, and reading the response body; it does not restart
+   * for each of them.  It does not interrupt synchronous work such as
+   * parsing a document that has already been received.
+   *
+   * When a call times out, the loader throws a {@link FetchError} without
+   * a {@link FetchError.response}, whose `cause` is a `DOMException` named
+   * `"TimeoutError"`.  An `AbortSignal` passed through
+   * {@link DocumentLoaderOptions.signal} still cancels a call; in that case
+   * the loader throws the signal's reason as before.
+   *
+   * Fractional values are rounded up.  Set it to `null` to turn off the
+   * timeout.
+   * @default `10000` (10 seconds)
+   * @throws {RangeError} If the value is not a positive finite number or is
+   *                      greater than 2,147,483,647 (about 24.8 days).
+   * @since 2.4.0
+   */
+  timeout?: number | null;
 }
 
 /**
@@ -129,6 +162,61 @@ function createResponseMetadata(response: Response): Response {
   });
 }
 
+const NULL_BODY_STATUSES: ReadonlySet<number> = new Set([204, 205, 304]);
+
+/**
+ * Reads the body of an error response while the document loader is still
+ * running, so that its timeout and `AbortSignal` also bound the read, and
+ * nothing reading {@link FetchError.response} later waits on the network.
+ * The body is kept byte for byte, unless it is too large or cannot be read;
+ * then only the status and headers are kept.  A response whose status
+ * the `Response` constructor does not accept (e.g., 999) is kept as a clone
+ * whose body has been read in full; if its body is too large or cannot be
+ * read, no response is kept at all.
+ */
+async function bufferErrorResponse(
+  response: Response,
+  url: string,
+  signal?: AbortSignal,
+): Promise<Response | undefined> {
+  if (response.status < 200 || response.status > 599) {
+    // Such a response cannot be rebuilt, so keep a clone instead, and read
+    // the original to the end so that the clone's body is buffered too:
+    const clone = response.clone();
+    try {
+      await readBoundedBytes(response, MAX_ERROR_BODY_SIZE, url);
+    } catch (error) {
+      await clone.body?.cancel().catch(() => {});
+      if (signal?.aborted) throw error;
+      logger.debug(
+        "Failed to read the error response body from {url}: {error}",
+        { url, error },
+      );
+      return undefined;
+    }
+    return clone;
+  }
+  if (response.body == null || NULL_BODY_STATUSES.has(response.status)) {
+    return createResponseMetadata(response);
+  }
+  let body: Uint8Array<ArrayBuffer>;
+  try {
+    body = await readBoundedBytes(response, MAX_ERROR_BODY_SIZE, url);
+  } catch (error) {
+    if (signal?.aborted) throw error;
+    logger.debug(
+      "Failed to read the error response body from {url}: {error}",
+      { url, error },
+    );
+    return createResponseMetadata(response);
+  }
+  return new Response(body, {
+    headers: response.headers,
+    status: response.status,
+    statusText: response.statusText,
+  });
+}
+
 /**
  * Gets a {@link RemoteDocument} from the given response.
  * @param url The URL of the document to load.
@@ -174,7 +262,7 @@ export async function getRemoteDocument(
     throw new FetchError(
       documentUrl,
       `HTTP ${response.status}: ${documentUrl}`,
-      response.clone(),
+      await bufferErrorResponse(response, documentUrl, options?.signal),
     );
   }
   const contentType = response.headers.get("Content-Type");
@@ -311,6 +399,104 @@ export async function getRemoteDocument(
   return { contextUrl, document, documentUrl };
 }
 
+/**
+ * Resolves {@link DocumentLoaderFactoryOptions.timeout} into milliseconds.
+ * @param timeout The timeout option.  `undefined` means the default timeout,
+ *                and `null` means no timeout.
+ * @returns The timeout in milliseconds, or `null` if it is turned off.
+ * @throws {RangeError} If the timeout is invalid.
+ * @internal
+ */
+export function resolveDocumentLoaderTimeout(
+  timeout: number | null | undefined,
+): number | null {
+  if (timeout === undefined) return DEFAULT_TIMEOUT;
+  if (timeout === null) return null;
+  if (
+    typeof timeout !== "number" || !Number.isFinite(timeout) || timeout <= 0
+  ) {
+    throw new RangeError(
+      `The document loader timeout must be a positive finite number of ` +
+        `milliseconds, but got ${String(timeout)}.`,
+    );
+  }
+  const ms = Math.ceil(timeout);
+  if (ms > MAX_TIMEOUT) {
+    throw new RangeError(
+      `The document loader timeout must not be greater than ${MAX_TIMEOUT} ` +
+        `milliseconds, but got ${timeout}.`,
+    );
+  }
+  return ms;
+}
+
+/**
+ * Bounds each call of the given document loader by the given timeout.
+ * The timeout is combined with the caller's `signal`, and the combined
+ * signal is passed to the loader.  The call settles no later than the
+ * timeout even if the loader is stuck in a step that cannot be aborted,
+ * e.g., a DNS lookup.
+ *
+ * A timed-out call throws a {@link FetchError} without a response, whose
+ * `cause` is a `DOMException` named `"TimeoutError"`.  If the caller's
+ * signal is aborted, its reason is thrown instead.
+ * @param loader The document loader to bound.
+ * @param timeout The timeout in milliseconds, or `null` for no timeout.
+ *                It is assumed to have been resolved by
+ *                {@link resolveDocumentLoaderTimeout}.
+ * @returns The bounded document loader.
+ * @internal
+ */
+export function withDocumentLoaderTimeout(
+  loader: DocumentLoader,
+  timeout: number | null,
+): DocumentLoader {
+  if (timeout == null) return loader;
+  return async (url, options) => {
+    const callerSignal = options?.signal;
+    callerSignal?.throwIfAborted();
+    const controller = new AbortController();
+    const timeoutReason = new DOMException(
+      `The document loader timed out after ${timeout} ms.`,
+      "TimeoutError",
+    );
+    let timedOut = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      controller.abort(timeoutReason);
+    }, timeout);
+    const onCallerAbort = () => controller.abort(callerSignal?.reason);
+    callerSignal?.addEventListener("abort", onCallerAbort, { once: true });
+    let onAbort: (() => void) | undefined;
+    const aborted = new Promise<never>((_, reject) => {
+      onAbort = () => reject(controller.signal.reason);
+      controller.signal.addEventListener("abort", onAbort, { once: true });
+    });
+    const loading = loader(url, { ...options, signal: controller.signal });
+    // If the abort wins the race, the loader's late rejection is ignored:
+    loading.catch(() => {});
+    try {
+      return await Promise.race([loading, aborted]);
+    } catch (error) {
+      if (callerSignal?.aborted) throw callerSignal.reason;
+      if (!timedOut) throw error;
+      logger[options?.suppressError ? "warn" : "error"](
+        "Timed out after {timeout} ms while fetching document: {url}",
+        { timeout, url },
+      );
+      const fetchError = new FetchError(url, `Timed out after ${timeout} ms`);
+      fetchError.cause = timeoutReason;
+      throw fetchError;
+    } finally {
+      clearTimeout(timer);
+      callerSignal?.removeEventListener("abort", onCallerAbort);
+      if (onAbort != null) {
+        controller.signal.removeEventListener("abort", onAbort);
+      }
+    }
+  };
+}
+
 /**
  * Options for {@link getDocumentLoader}.
  * @since 1.3.0
@@ -326,6 +512,8 @@ export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
  * Creates a JSON-LD document loader that utilizes the browser's `fetch` API.
  * At most 20 HTTP redirects and alternate document links are followed in total
  * per call.  Revisiting a URL within that chain throws a {@link FetchError}.
+ * Each call times out after 10 seconds by default; see
+ * {@link DocumentLoaderFactoryOptions.timeout}.
  *
  * The created loader preloads the below frequently used contexts by default
  * (unless `options.skipPreloadedContexts` is set to `true`):
@@ -346,9 +534,15 @@ export interface GetDocumentLoaderOptions extends DocumentLoaderFactoryOptions {
  * @since 1.3.0
  */
 export function getDocumentLoader(
-  { allowPrivateAddress, maxRedirection, skipPreloadedContexts, userAgent }:
-    GetDocumentLoaderOptions = {},
+  {
+    allowPrivateAddress,
+    maxRedirection,
+    skipPreloadedContexts,
+    timeout,
+    userAgent,
+  }: GetDocumentLoaderOptions = {},
 ): DocumentLoader {
+  const resolvedTimeout = resolveDocumentLoaderTimeout(timeout);
   const tracerProvider = trace.getTracerProvider();
   const tracer = tracerProvider.getTracer(metadata.name, metadata.version);
   const maximumRedirection = maxRedirection ?? DEFAULT_MAX_REDIRECTION;
@@ -391,6 +585,9 @@ export function getDocumentLoader(
         }
         throw error;
       }
+      // The DNS lookup cannot be aborted, so do not go on if the call was
+      // aborted or timed out in the meantime:
+      options?.signal?.throwIfAborted();
     }
     visited.add(currentUrl);
 
@@ -488,5 +685,8 @@ export function getDocumentLoader(
       },
     );
   }
-  return (url, options) => load(url, options);
+  return withDocumentLoaderTimeout(
+    (url, options) => load(url, options),
+    resolvedTimeout,
+  );
 }

package/src/mod.ts

@@ -15,6 +15,8 @@ export {
   type GetDocumentLoaderOptions,
   getRemoteDocument,
   type RemoteDocument,
+  resolveDocumentLoaderTimeout,
+  withDocumentLoaderTimeout,
 } from "./docloader.ts";
 export {
   type DidKeyVerificationMethod,