@dwk/microsub 0.1.0-beta.0

package/src/discovery.ts

@@ -0,0 +1,270 @@
+/**
+ * `@dwk/microsub` — feed discovery and fetching.
+ *
+ * `follow` and `preview` take a URL the user typed; the server must work out
+ * what to actually poll. {@link discoverFeed} fetches that URL (through the
+ * SSRF-safe wrapper), and:
+ *
+ * - if it is already a syndication feed (Atom / RSS / JSON Feed), parses it;
+ * - if it is HTML, looks for a `<link rel="alternate">` feed and follows the
+ *   first one; failing that, parses the page's own `h-feed` microformats.
+ *
+ * {@link fetchFeed} re-fetches an already-resolved feed URL on each poll,
+ * sending the cached `ETag` / `Last-Modified` so an unchanged feed returns `304`
+ * and is skipped. Every outbound request goes through {@link safeFetch}, and the
+ * body is read with a hard size cap.
+ *
+ * @packageDocumentation
+ */
+
+import { noopLogger, noopMetrics, type Logger, type Metrics } from "@dwk/log";
+
+import { readTextCapped, type FetchLike } from "./fetch";
+import { parseHFeed } from "./hfeed";
+import { parseFeed, type Jf2Entry } from "./jf2";
+import { safeFetch } from "./safe-fetch";
+
+/** Shared options for the discovery/fetch helpers. */
+export interface DiscoveryOptions {
+  readonly fetch?: FetchLike;
+  readonly logger?: Logger;
+  readonly metrics?: Metrics;
+}
+
+/** A resolved feed: the URL to poll and the entries parsed from its first fetch. */
+export interface DiscoveredFeed {
+  /** The URL the poller should re-fetch (a feed, or an HTML `h-feed` page). */
+  readonly feedUrl: string;
+  /** The entries parsed from the discovery fetch. */
+  readonly entries: readonly Jf2Entry[];
+}
+
+/** A completed feed fetch. */
+export interface FetchedFeed {
+  /** The parsed entries (empty when not modified or unparseable). */
+  readonly entries: readonly Jf2Entry[];
+  /** `true` when the server answered `304 Not Modified`. */
+  readonly notModified: boolean;
+  readonly etag: string | null;
+  readonly lastModified: string | null;
+}
+
+const FEED_ACCEPT =
+  "application/atom+xml, application/rss+xml, application/feed+json, " +
+  "application/json, text/xml, application/xml, text/html;q=0.9, */*;q=0.8";
+
+/** Feed media types we recognise on a `Content-Type` or a `<link type>`. */
+function isFeedType(contentType: string): boolean {
+  const essence = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
+  return (
+    essence === "application/atom+xml" ||
+    essence === "application/rss+xml" ||
+    essence === "application/feed+json" ||
+    essence === "application/json" ||
+    essence === "text/xml" ||
+    essence === "application/xml" ||
+    essence === "application/rdf+xml"
+  );
+}
+
+function isHtmlType(contentType: string): boolean {
+  const essence = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
+  return essence === "text/html" || essence === "application/xhtml+xml";
+}
+
+/** A `<link>` discovered in an HTML head: its rel, type, and resolved href. */
+interface FeedLink {
+  readonly rels: string[];
+  readonly type: string;
+  readonly href: string;
+}
+
+/** Collect `<link>` elements (rel/type/href) from an HTML document. */
+async function findFeedLinks(
+  html: string,
+  baseUrl: string,
+): Promise<FeedLink[]> {
+  const links: FeedLink[] = [];
+  const rewriter = new HTMLRewriter().on("link", {
+    element(el) {
+      const href = el.getAttribute("href");
+      if (href === null || href === "") return;
+      const rels = (el.getAttribute("rel") ?? "")
+        .toLowerCase()
+        .split(/\s+/)
+        .filter(Boolean);
+      const type = (el.getAttribute("type") ?? "").toLowerCase();
+      let resolved: string;
+      try {
+        resolved = new URL(href, baseUrl).toString();
+      } catch {
+        return;
+      }
+      links.push({ rels, type, href: resolved });
+    },
+  });
+  await rewriter.transform(new Response(html)).text();
+  return links;
+}
+
+/** The first alternate feed `<link>` in document order, if any. */
+function pickFeedLink(links: readonly FeedLink[]): FeedLink | null {
+  for (const link of links) {
+    if (!link.rels.includes("alternate") && link.rels.length > 0) continue;
+    if (
+      link.type === "application/atom+xml" ||
+      link.type === "application/rss+xml" ||
+      link.type === "application/feed+json" ||
+      link.type === "application/json"
+    ) {
+      return link;
+    }
+  }
+  return null;
+}
+
+/** Parse a fetched body into entries, choosing the parser by content type. */
+async function parseBody(
+  body: string,
+  contentType: string,
+  url: string,
+): Promise<Jf2Entry[]> {
+  if (isHtmlType(contentType)) {
+    return parseHFeed(body, url);
+  }
+  return parseFeed(body, contentType, url);
+}
+
+function resolveDeps(options?: DiscoveryOptions): {
+  doFetch: FetchLike;
+  logger: Logger;
+  metrics: Metrics;
+} {
+  return {
+    doFetch: options?.fetch ?? ((input, init) => fetch(input, init)),
+    logger: options?.logger ?? noopLogger,
+    metrics: options?.metrics ?? noopMetrics,
+  };
+}
+
+/**
+ * Discover the feed at `target`. Returns the URL to poll plus the entries from
+ * this first fetch, or `null` when the target is unreachable, blocked, or has no
+ * feed and no `h-entry` content.
+ */
+export async function discoverFeed(
+  target: string,
+  options?: DiscoveryOptions,
+): Promise<DiscoveredFeed | null> {
+  const { doFetch, logger, metrics } = resolveDeps(options);
+
+  let response: Response;
+  let finalUrl: string;
+  try {
+    const result = await safeFetch(
+      doFetch,
+      target,
+      { method: "GET", headers: { accept: FEED_ACCEPT } },
+      { logger, metrics },
+    );
+    response = result.response;
+    finalUrl = result.url;
+  } catch {
+    return null;
+  }
+
+  if (!response.ok) return null;
+  const contentType = response.headers.get("content-type") ?? "";
+  const body = await readTextCapped(response);
+  if (body === null) return null;
+
+  // A syndication feed: parse it directly.
+  if (
+    isFeedType(contentType) ||
+    (!isHtmlType(contentType) && looksLikeFeedBody(body))
+  ) {
+    return {
+      feedUrl: finalUrl,
+      entries: parseFeed(body, contentType, finalUrl),
+    };
+  }
+
+  // HTML: prefer a declared alternate feed, else fall back to h-feed.
+  if (isHtmlType(contentType) || body.trimStart().startsWith("<")) {
+    const links = await findFeedLinks(body, finalUrl);
+    const feedLink = pickFeedLink(links);
+    if (feedLink !== null) {
+      const fetched = await fetchFeed(feedLink.href, options);
+      if (fetched !== null) {
+        return { feedUrl: feedLink.href, entries: fetched.entries };
+      }
+    }
+    const entries = await parseHFeed(body, finalUrl);
+    if (entries.length > 0) {
+      return { feedUrl: finalUrl, entries };
+    }
+  }
+
+  return null;
+}
+
+/** Cheap sniff for a feed root when the content type is unhelpful. */
+function looksLikeFeedBody(body: string): boolean {
+  const head = body.trimStart().slice(0, 512).toLowerCase();
+  return (
+    head.includes("<rss") ||
+    head.includes("<feed") ||
+    head.includes("<rdf:rdf") ||
+    (head.startsWith("{") && head.includes("jsonfeed.org"))
+  );
+}
+
+/**
+ * Fetch and parse an already-resolved feed URL, sending conditional-request
+ * validators when supplied. Returns `notModified` on a `304`, the parsed
+ * entries otherwise, or `null` when the fetch fails or is blocked.
+ */
+export async function fetchFeed(
+  feedUrl: string,
+  options?: DiscoveryOptions,
+  cache?: { etag: string | null; lastModified: string | null },
+): Promise<FetchedFeed | null> {
+  const { doFetch, logger, metrics } = resolveDeps(options);
+
+  const headers: Record<string, string> = { accept: FEED_ACCEPT };
+  if (cache?.etag) headers["if-none-match"] = cache.etag;
+  if (cache?.lastModified) headers["if-modified-since"] = cache.lastModified;
+
+  let response: Response;
+  let finalUrl: string;
+  try {
+    const result = await safeFetch(
+      doFetch,
+      feedUrl,
+      { method: "GET", headers },
+      { logger, metrics },
+    );
+    response = result.response;
+    finalUrl = result.url;
+  } catch {
+    return null;
+  }
+
+  const etag = response.headers.get("etag");
+  const lastModified = response.headers.get("last-modified");
+
+  if (response.status === 304) {
+    await response.body?.cancel().catch(() => undefined);
+    return { entries: [], notModified: true, etag, lastModified };
+  }
+  if (!response.ok) {
+    await response.body?.cancel().catch(() => undefined);
+    return null;
+  }
+
+  const contentType = response.headers.get("content-type") ?? "";
+  const body = await readTextCapped(response);
+  if (body === null) return null;
+  const entries = await parseBody(body, contentType, finalUrl);
+  return { entries, notModified: false, etag, lastModified };
+}

package/src/fetch.ts

@@ -0,0 +1,82 @@
+/**
+ * `@dwk/microsub` — injectable `fetch` type and a body-size-capped text reader.
+ *
+ * Discovery, polling, preview, and search all perform HTTP I/O against
+ * attacker-influenced URLs. They accept a {@link FetchLike} so callers can
+ * inject a stub in tests (no network) and so the package never reaches for a
+ * global it did not receive.
+ *
+ * @packageDocumentation
+ */
+
+/** A minimal, injectable `fetch` signature. */
+export type FetchLike = (
+  input: string,
+  init?: RequestInit,
+) => Promise<Response>;
+
+/**
+ * Default cap on a fetched feed body (4 MB). A feed is modest; a larger body is
+ * almost certainly hostile or irrelevant, and buffering it would risk an OOM
+ * (the Worker memory limit is 128 MB). See `spec/non-functional-requirements.md`.
+ */
+export const MAX_BODY_BYTES = 4 * 1024 * 1024;
+
+/**
+ * Read a response body as text, refusing bodies larger than `maxBytes`.
+ *
+ * A declared `Content-Length` over the cap is rejected up front; the stream is
+ * then read incrementally and aborted the moment the cap is exceeded, so a
+ * missing or lying `Content-Length` cannot force the whole body into memory.
+ * Returns `null` when the body is too large or cannot be read.
+ */
+export async function readTextCapped(
+  response: Response,
+  maxBytes = MAX_BODY_BYTES,
+): Promise<string | null> {
+  const declared = response.headers.get("content-length");
+  if (declared !== null) {
+    const length = Number.parseInt(declared, 10);
+    if (Number.isFinite(length) && length > maxBytes) {
+      return null;
+    }
+  }
+
+  const body = response.body;
+  if (body === null) {
+    try {
+      const text = await response.text();
+      return text.length > maxBytes ? null : text;
+    } catch {
+      return null;
+    }
+  }
+
+  const reader = body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (value !== undefined) {
+        total += value.byteLength;
+        if (total > maxBytes) {
+          await reader.cancel();
+          return null;
+        }
+        chunks.push(value);
+      }
+    }
+  } catch {
+    return null;
+  }
+
+  const merged = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return new TextDecoder("utf-8").decode(merged);
+}