@dwk/webmention 0.1.0-beta.0

package/src/fetch.ts

@@ -0,0 +1,84 @@
+/**
+ * `@dwk/webmention` — injectable `fetch` type.
+ *
+ * Endpoint discovery, source verification, and sending all perform HTTP I/O.
+ * They accept a {@link FetchLike} so callers can inject a stub in tests (no
+ * network) and so the package never reaches for a global it didn't receive.
+ *
+ * @packageDocumentation
+ */
+
+/** A minimal, injectable `fetch` signature. */
+export type FetchLike = (
+  input: string,
+  init?: RequestInit,
+) => Promise<Response>;
+
+/**
+ * Default cap on a fetched document body (2 MB). Discovery and verification
+ * only need to scan markup for links; a larger body is almost certainly hostile
+ * or irrelevant, and buffering it would risk an OOM (the Worker memory limit is
+ * 128 MB). See `spec/non-functional-requirements.md`.
+ */
+export const MAX_BODY_BYTES = 2 * 1024 * 1024;
+
+/**
+ * Read a response body as text, refusing bodies larger than `maxBytes`.
+ *
+ * A declared `Content-Length` over the cap is rejected up front; the stream is
+ * then read incrementally and aborted the moment the cap is exceeded, so a
+ * missing or lying `Content-Length` cannot force the whole body into memory.
+ * Returns `null` when the body is too large or cannot be read.
+ */
+export async function readBodyCapped(
+  response: Response,
+  maxBytes = MAX_BODY_BYTES,
+): Promise<string | null> {
+  const declared = response.headers.get("content-length");
+  if (declared !== null) {
+    const length = Number.parseInt(declared, 10);
+    if (Number.isFinite(length) && length > maxBytes) {
+      return null;
+    }
+  }
+
+  const body = response.body;
+  if (body === null) {
+    try {
+      const text = await response.text();
+      return text.length > maxBytes ? null : text;
+    } catch {
+      return null;
+    }
+  }
+
+  const reader = body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) {
+        break;
+      }
+      if (value !== undefined) {
+        total += value.byteLength;
+        if (total > maxBytes) {
+          await reader.cancel();
+          return null;
+        }
+        chunks.push(value);
+      }
+    }
+  } catch {
+    return null;
+  }
+
+  const merged = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return new TextDecoder().decode(merged);
+}

package/src/html.ts

@@ -0,0 +1,206 @@
+/**
+ * `@dwk/webmention` — HTML / `Link`-header parsing helpers.
+ *
+ * Shared by endpoint discovery (sender) and source verification (receiver).
+ * `Link`-header parsing is plain string scanning; HTML scanning uses the
+ * Workers runtime's streaming `HTMLRewriter` rather than regex tag matching, so
+ * it handles comments, attribute quoting, and malformed markup correctly
+ * without pulling a parser into the bundle (`HTMLRewriter` is built into the
+ * runtime — zero script-size cost; see `spec/non-functional-requirements.md`).
+ *
+ * Because `HTMLRewriter` is a `workerd` global, the HTML scanners are async and
+ * exercised under the Workers test pool, not bare Node.
+ *
+ * @packageDocumentation
+ */
+
+/** A parsed `Link` header entry: its target URI and `rel` tokens. */
+export interface LinkHeaderEntry {
+  readonly uri: string;
+  readonly rels: readonly string[];
+}
+
+/**
+ * Parse an HTTP `Link` header into entries. Handles multiple comma-separated
+ * links and semicolon-separated parameters, e.g.
+ * `<https://a.example/webmention>; rel="webmention"`.
+ *
+ * Entries with no `rel` parameter are dropped; an empty URI (`<>`) is kept so
+ * the caller can resolve it against the document URL (a Webmention endpoint
+ * advertised at the page itself).
+ */
+export function parseLinkHeader(value: string | null): LinkHeaderEntry[] {
+  if (value === null || value.trim() === "") {
+    return [];
+  }
+  const entries: LinkHeaderEntry[] = [];
+  for (const part of splitLinks(value)) {
+    const match = /^\s*<([^>]*)>\s*(.*)$/.exec(part);
+    if (match === null) {
+      continue;
+    }
+    const uri = match[1] ?? "";
+    const rels = splitTokens(extractRel(match[2] ?? ""));
+    if (rels.length > 0) {
+      entries.push({ uri, rels });
+    }
+  }
+  return entries;
+}
+
+/**
+ * Split a `Link` header on top-level commas, respecting both the angle-bracket
+ * URI reference and double-quoted parameter values — so a comma inside
+ * `title="A, B"` or inside `<…>` does not split the entry.
+ */
+function splitLinks(value: string): string[] {
+  const result: string[] = [];
+  let depth = 0;
+  let inQuotes = false;
+  let current = "";
+  for (let i = 0; i < value.length; i++) {
+    const char = value[i] as string;
+    if (char === '"' && value[i - 1] !== "\\") {
+      inQuotes = !inQuotes;
+    } else if (!inQuotes && char === "<") {
+      depth++;
+    } else if (!inQuotes && char === ">") {
+      depth--;
+    }
+    if (char === "," && depth === 0 && !inQuotes) {
+      result.push(current);
+      current = "";
+    } else {
+      current += char;
+    }
+  }
+  if (current.trim() !== "") {
+    result.push(current);
+  }
+  return result;
+}
+
+/**
+ * Split a `Link` entry's parameter string on top-level semicolons, respecting
+ * double-quoted values so a `;` inside a quoted value does not split a param.
+ */
+function splitParams(paramString: string): string[] {
+  const result: string[] = [];
+  let inQuotes = false;
+  let current = "";
+  for (let i = 0; i < paramString.length; i++) {
+    const char = paramString[i] as string;
+    if (char === '"' && paramString[i - 1] !== "\\") {
+      inQuotes = !inQuotes;
+    }
+    if (char === ";" && !inQuotes) {
+      result.push(current);
+      current = "";
+    } else {
+      current += char;
+    }
+  }
+  if (current.trim() !== "") {
+    result.push(current);
+  }
+  return result;
+}
+
+/**
+ * Extract the `rel` parameter from a `Link` entry's parameter string. Matches
+ * the `rel` parameter exactly (per-parameter), so a `rel=` substring inside
+ * another parameter's quoted value (e.g. `title="my rel=x"`) is not mistaken
+ * for it.
+ */
+function extractRel(paramString: string): string | null {
+  for (const param of splitParams(paramString)) {
+    const match = /^\s*rel\s*=\s*("([^"]*)"|'([^']*)'|[^;\s]+)\s*$/i.exec(
+      param,
+    );
+    if (match !== null) {
+      return match[2] ?? match[3] ?? match[1] ?? null;
+    }
+  }
+  return null;
+}
+
+/**
+ * Whether a `Content-Type` value names an HTML document (`text/html` or
+ * `application/xhtml+xml`). Compares the media type's essence — the part before
+ * any `;` parameters — case-insensitively, so `text/html; charset=utf-8`
+ * matches but an unrelated type carrying `text/html` inside a parameter does
+ * not.
+ */
+export function isHtmlContentType(contentType: string): boolean {
+  const essence = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
+  return essence === "text/html" || essence === "application/xhtml+xml";
+}
+
+/**
+ * Whether a `Content-Type` value names a JSON document (`application/json` or a
+ * `+json`-suffixed type such as `application/activity+json`). Compares the media
+ * type's essence — the part before any `;` parameters — case-insensitively.
+ */
+export function isJsonContentType(contentType: string): boolean {
+  const essence = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
+  return essence === "application/json" || essence.endsWith("+json");
+}
+
+/** Split a whitespace-separated token list (e.g. a `rel` value) into tokens. */
+export function splitTokens(value: string | null): string[] {
+  if (value === null) {
+    return [];
+  }
+  return value
+    .trim()
+    .split(/\s+/)
+    .filter((token) => token !== "");
+}
+
+/** Resolve `uri` against `base`, returning a normalized absolute URL or `null`. */
+export function resolveUrl(uri: string, base: string): string | null {
+  try {
+    return new URL(uri, base).toString();
+  } catch {
+    return null;
+  }
+}
+
+/** An element seen by {@link scanElements}: its (lowercased) tag name and the requested attributes. */
+export interface ScannedElement {
+  /** Lowercased tag name, e.g. `"a"`, `"link"`, `"base"`. */
+  readonly name: string;
+  /** Requested attribute values; `null` when the attribute is absent, `""` when present but empty. */
+  readonly attrs: Readonly<Record<string, string | null>>;
+}
+
+/**
+ * Scan `html` with the runtime's streaming `HTMLRewriter`, returning — in
+ * document order — every element matching `selector` together with the
+ * requested attribute values.
+ *
+ * Using a real tokenizer (rather than regex) means elements inside comments are
+ * never reported (the parser treats comment contents as text, satisfying
+ * webmention.rocks discovery test 13), attribute quoting is handled correctly,
+ * and a `data-href` attribute is never mistaken for `href`. An absent attribute
+ * is reported as `null`; a present-but-empty one (`href=""`) as `""`.
+ */
+export async function scanElements(
+  html: string,
+  selector: string,
+  attrNames: readonly string[],
+): Promise<ScannedElement[]> {
+  const elements: ScannedElement[] = [];
+  const rewriter = new HTMLRewriter().on(selector, {
+    element(el) {
+      const attrs: Record<string, string | null> = {};
+      for (const name of attrNames) {
+        attrs[name] = el.getAttribute(name);
+      }
+      elements.push({ name: el.tagName, attrs });
+    },
+  });
+  // Drive the parser to completion by consuming the transformed body.
+  await rewriter.transform(new Response(html)).text();
+  return elements;
+}

package/src/inbox.ts

@@ -0,0 +1,121 @@
+/**
+ * `@dwk/webmention` — inbox store.
+ *
+ * Verified mentions are persisted to an inbox so they can be surfaced on the
+ * target resource. The default is a D1-backed store (strongly consistent —
+ * never KV, per `spec/non-functional-requirements.md`); when composed into a
+ * Solid Pod, a caller can supply an {@link InboxStore} backed by the
+ * `@dwk/solid-pod` Durable Object instead. The store keys on the
+ * `(source, target)` pair so re-verifying a mention updates it in place and a
+ * source that drops the link can be removed. See `spec/packages/webmention.md`.
+ *
+ * @packageDocumentation
+ */
+
+import type { D1Database } from "@cloudflare/workers-types";
+
+/** A verified Webmention: `source` links to `target`, confirmed at `verifiedAt`. */
+export interface VerifiedMention {
+  readonly source: string;
+  readonly target: string;
+  /** Verification time, epoch milliseconds. */
+  readonly verifiedAt: number;
+}
+
+/** Persistence surface for verified mentions. */
+export interface InboxStore {
+  /** Upsert a verified mention, keyed on `(source, target)`. */
+  store(mention: VerifiedMention): Promise<void>;
+  /** Remove a mention (e.g. the source dropped the link); no-op when absent. */
+  remove(source: string, target: string): Promise<void>;
+  /** List mentions, newest first; scoped to `target` when given. */
+  list(target?: string): Promise<VerifiedMention[]>;
+}
+
+/** Options for {@link createD1Inbox}. */
+export interface D1InboxOptions {
+  /** Table name to use; created if absent. Defaults to `webmentions`. */
+  readonly table?: string;
+}
+
+interface MentionRow {
+  readonly source: string;
+  readonly target: string;
+  readonly verified_at: number;
+}
+
+/**
+ * Build a D1-backed {@link InboxStore}. The backing table is created on first
+ * use if it does not already exist.
+ */
+export function createD1Inbox(
+  db: D1Database,
+  options?: D1InboxOptions,
+): InboxStore {
+  const table = options?.table ?? "webmentions";
+  // Guard the identifier: it is interpolated into DDL, so only allow a safe
+  // set of characters rather than trusting the caller blindly.
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(table)) {
+    throw new Error(`@dwk/webmention: invalid inbox table name "${table}".`);
+  }
+
+  let ready: Promise<void> | null = null;
+  const ensureSchema = (): Promise<void> => {
+    ready ??= db
+      .prepare(
+        `CREATE TABLE IF NOT EXISTS ${table} (` +
+          `source TEXT NOT NULL, ` +
+          `target TEXT NOT NULL, ` +
+          `verified_at INTEGER NOT NULL, ` +
+          `PRIMARY KEY (source, target))`,
+      )
+      .run()
+      .then(() => undefined);
+    return ready;
+  };
+
+  return {
+    async store(mention) {
+      await ensureSchema();
+      await db
+        .prepare(
+          `INSERT INTO ${table} (source, target, verified_at) ` +
+            `VALUES (?1, ?2, ?3) ` +
+            `ON CONFLICT (source, target) ` +
+            `DO UPDATE SET verified_at = excluded.verified_at`,
+        )
+        .bind(mention.source, mention.target, mention.verifiedAt)
+        .run();
+    },
+
+    async remove(source, target) {
+      await ensureSchema();
+      await db
+        .prepare(`DELETE FROM ${table} WHERE source = ?1 AND target = ?2`)
+        .bind(source, target)
+        .run();
+    },
+
+    async list(target) {
+      await ensureSchema();
+      const statement =
+        target === undefined
+          ? db.prepare(
+              `SELECT source, target, verified_at FROM ${table} ` +
+                `ORDER BY verified_at DESC`,
+            )
+          : db
+              .prepare(
+                `SELECT source, target, verified_at FROM ${table} ` +
+                  `WHERE target = ?1 ORDER BY verified_at DESC`,
+              )
+              .bind(target);
+      const { results } = await statement.all<MentionRow>();
+      return results.map((row) => ({
+        source: row.source,
+        target: row.target,
+        verifiedAt: row.verified_at,
+      }));
+    },
+  };
+}

package/src/index.ts

@@ -0,0 +1,297 @@
+/**
+ * `@dwk/webmention` — Webmention (W3C) receiver + sender.
+ *
+ * Endpoint package. The receiver validates `source`/`target` synchronously,
+ * returns `202 Accepted`, and enqueues the pair for asynchronous link
+ * verification; the queue consumer fetches the source, confirms it links to the
+ * target, and persists (or removes) the mention in an inbox. The sender
+ * discovers a target's Webmention endpoint and notifies it on publish. Cloud
+ * specifics (Queue, D1) are confined here; HTML scanning uses the runtime's
+ * streaming `HTMLRewriter`, so the parsing/verification helpers are async and
+ * exercised under the Workers test pool.
+ *
+ * @see spec/packages/webmention.md
+ * @packageDocumentation
+ */
+
+import type {
+  D1Database,
+  ExecutionContext,
+  MessageBatch,
+  Queue,
+} from "@cloudflare/workers-types";
+import {
+  hostFromUrl,
+  noopLogger,
+  noopMetrics,
+  type Logger,
+  type Metrics,
+} from "@dwk/log";
+import { createD1Inbox, type InboxStore } from "./inbox";
+import type { FetchLike } from "./fetch";
+import { WebmentionLogEvent } from "./log";
+import { validateWebmentionParams } from "./validate";
+import { verifySource } from "./verify";
+
+export {
+  validateWebmentionParams,
+  type ValidateParams,
+  type ValidationResult,
+  type WebmentionValidationError,
+} from "./validate";
+export {
+  discoverEndpoint,
+  findWebmentionEndpoint,
+  type DiscoverOptions,
+} from "./discovery";
+export {
+  sendWebmention,
+  sendWebmentions,
+  type SendOptions,
+  type SendResult,
+} from "./sender";
+export {
+  verifySource,
+  sourceLinksTo,
+  extractLinks,
+  type VerifyOptions,
+  type VerifyResult,
+} from "./verify";
+export {
+  createD1Inbox,
+  type InboxStore,
+  type VerifiedMention,
+  type D1InboxOptions,
+} from "./inbox";
+export type { FetchLike } from "./fetch";
+export {
+  safeFetch,
+  assertPublicUrl,
+  isPrivateOrReservedHost,
+  SsrfError,
+  DEFAULT_MAX_REDIRECTS,
+  DEFAULT_TIMEOUT_MS,
+  type SafeFetchOptions,
+  type SafeFetchResult,
+  type SsrfReason,
+} from "./safe-fetch";
+export { WebmentionLogEvent } from "./log";
+export type { Logger, Metrics } from "@dwk/log";
+
+/** A queued verification job: confirm that `source` links to `target`. */
+export interface WebmentionJob {
+  readonly source: string;
+  readonly target: string;
+}
+
+/** Cloudflare bindings required by the Webmention handler and queue consumer. */
+export interface WebmentionEnv {
+  /** Queue producer for async verification of received mentions. */
+  readonly WEBMENTION_QUEUE: Queue<WebmentionJob>;
+  /**
+   * D1 database backing the default inbox. Optional only when an
+   * {@link InboxStore} is supplied via {@link WebmentionConfig.inbox} (e.g. a
+   * Solid Pod DO-backed inbox).
+   */
+  readonly WEBMENTION_INBOX?: D1Database;
+}
+
+/** Configuration passed to {@link createWebmention}. */
+export interface WebmentionConfig {
+  /** Base URL of this receiver; a `target` must live under its origin. */
+  readonly baseUrl: string;
+  /** Additional controlled hostnames besides `baseUrl`'s. */
+  readonly allowedHosts?: readonly string[];
+  /**
+   * Inbox store for verified mentions. Defaults to a D1 store built from
+   * {@link WebmentionEnv.WEBMENTION_INBOX}; supply one to back the inbox with
+   * the `@dwk/solid-pod` Durable Object instead.
+   */
+  readonly inbox?: InboxStore;
+  /** `fetch` implementation for verification; defaults to the global `fetch`. */
+  readonly fetch?: FetchLike;
+  /**
+   * Logger for receiver/queue events; defaults to a no-op. Wire a real logger
+   * (see `@dwk/log`) to surface SSRF blocks, validation rejections, and
+   * poison-message retries instead of swallowing them.
+   */
+  readonly logger?: Logger;
+  /**
+   * Metrics sink for receiver/queue counters; defaults to a no-op. Wire an
+   * adapter (e.g. `analyticsEngineMetrics` from `@dwk/log`, bound to an
+   * `AnalyticsEngineDataset`) to chart the same events the logger names —
+   * "SSRF blocks/min", "verification success rate", "queue retries by reason".
+   */
+  readonly metrics?: Metrics;
+}
+
+/** A `fetch`-compatible Worker handler. */
+export type WebmentionHandler = (
+  request: Request,
+  env: WebmentionEnv,
+  ctx: ExecutionContext,
+) => Promise<Response>;
+
+/** A Queue consumer for asynchronous Webmention verification. */
+export type WebmentionQueueConsumer = (
+  batch: MessageBatch<WebmentionJob>,
+  env: WebmentionEnv,
+  ctx: ExecutionContext,
+) => Promise<void>;
+
+function textResponse(status: number, body: string): Response {
+  return new Response(body, {
+    status,
+    headers: { "content-type": "text/plain; charset=utf-8" },
+  });
+}
+
+function resolveInbox(
+  config: WebmentionConfig,
+  env: WebmentionEnv,
+): InboxStore {
+  if (config.inbox !== undefined) {
+    return config.inbox;
+  }
+  if (env.WEBMENTION_INBOX !== undefined) {
+    return createD1Inbox(env.WEBMENTION_INBOX);
+  }
+  throw new Error(
+    "@dwk/webmention: no inbox configured — provide config.inbox or bind " +
+      "WEBMENTION_INBOX (D1).",
+  );
+}
+
+function formValue(value: string | File | null): string | null {
+  return typeof value === "string" ? value : null;
+}
+
+/**
+ * Whether the request body is `application/x-www-form-urlencoded` — the encoding
+ * Webmention §3.1.3 requires. `Request.formData()` would also accept
+ * `multipart/form-data`, so the essence is checked up front rather than relying
+ * on it.
+ */
+function isFormUrlEncoded(contentType: string | null): boolean {
+  const essence = contentType?.split(";")[0]?.trim().toLowerCase() ?? "";
+  return essence === "application/x-www-form-urlencoded";
+}
+
+/**
+ * Build the Webmention receiver handler from configuration.
+ *
+ * The returned handler is mountable under any path prefix. It accepts a
+ * form-encoded `POST` (`source` + `target`), validates synchronously, enqueues
+ * the pair for verification, and returns `202 Accepted`. Invalid requests get
+ * `400`; other methods get `405`. Fails loudly if the required `WEBMENTION_QUEUE`
+ * binding is missing.
+ */
+export function createWebmention(config: WebmentionConfig): WebmentionHandler {
+  const logger = config.logger ?? noopLogger;
+  const metrics = config.metrics ?? noopMetrics;
+  return async (request, env, _ctx) => {
+    if (request.method !== "POST") {
+      return new Response("Method Not Allowed", {
+        status: 405,
+        headers: { allow: "POST" },
+      });
+    }
+
+    if (env.WEBMENTION_QUEUE === undefined) {
+      throw new Error(
+        "@dwk/webmention: missing required binding WEBMENTION_QUEUE.",
+      );
+    }
+
+    if (!isFormUrlEncoded(request.headers.get("content-type"))) {
+      const fields = { reason: "invalid_content_type" as const };
+      logger.warn(WebmentionLogEvent.ReceiveRejected, fields);
+      metrics.count(WebmentionLogEvent.ReceiveRejected, fields);
+      return textResponse(
+        400,
+        "invalid_request: Content-Type must be application/x-www-form-urlencoded",
+      );
+    }
+
+    let form: FormData;
+    try {
+      form = await request.formData();
+    } catch {
+      return textResponse(
+        400,
+        "invalid_request: expected a form-encoded body with source and target",
+      );
+    }
+
+    const result = validateWebmentionParams({
+      source: formValue(form.get("source")),
+      target: formValue(form.get("target")),
+      baseUrl: config.baseUrl,
+      allowedHosts: config.allowedHosts,
+    });
+    if (!result.ok) {
+      const fields = { reason: result.error };
+      logger.warn(WebmentionLogEvent.ReceiveRejected, fields);
+      metrics.count(WebmentionLogEvent.ReceiveRejected, fields);
+      return textResponse(400, result.error);
+    }
+
+    await env.WEBMENTION_QUEUE.send({
+      source: result.source,
+      target: result.target,
+    });
+
+    const fields = {
+      sourceHost: hostFromUrl(result.source),
+      targetHost: hostFromUrl(result.target),
+    };
+    logger.info(WebmentionLogEvent.ReceiveAccepted, fields);
+    metrics.count(WebmentionLogEvent.ReceiveAccepted, fields);
+    return new Response(null, { status: 202 });
+  };
+}
+
+/**
+ * Build the Queue consumer that verifies received mentions.
+ *
+ * For each job it fetches the source and checks for a link to the target: a
+ * verified mention is upserted into the inbox, while a source that no longer
+ * links is removed. A job that throws is retried; otherwise it is acked. Fails
+ * loudly if no inbox is configured.
+ */
+export function createWebmentionQueueConsumer(
+  config: WebmentionConfig,
+): WebmentionQueueConsumer {
+  const logger = config.logger ?? noopLogger;
+  const metrics = config.metrics ?? noopMetrics;
+  return async (batch, env, _ctx) => {
+    const inbox = resolveInbox(config, env);
+    for (const message of batch.messages) {
+      const { source, target } = message.body;
+      try {
+        const result = await verifySource(source, target, {
+          fetch: config.fetch,
+          logger,
+          metrics,
+        });
+        if (result.links) {
+          await inbox.store({ source, target, verifiedAt: Date.now() });
+        } else {
+          await inbox.remove(source, target);
+        }
+        message.ack();
+      } catch (err) {
+        // A poison message must not retry silently — record why so an operator
+        // can tell a transient failure from a wedged one.
+        const fields = {
+          sourceHost: hostFromUrl(source),
+          targetHost: hostFromUrl(target),
+          error: err instanceof Error ? err.name : "unknown",
+        };
+        logger.warn(WebmentionLogEvent.QueueRetry, fields);
+        metrics.count(WebmentionLogEvent.QueueRetry, fields);
+        message.retry();
+      }
+    }
+  };
+}