@dwk/microsub 0.1.0-beta.0

package/src/jf2.ts

@@ -0,0 +1,394 @@
+/**
+ * `@dwk/microsub` — feed → JF2 normalisation.
+ *
+ * Microsub serves a single normalised timeline regardless of a source's wire
+ * format. This module turns the four feed formats a reader meets — JSON Feed,
+ * Atom, RSS 2.0, and `h-feed` microformats (the last parsed upstream in
+ * {@link ./hfeed}) — into [JF2](https://jf2.spec.indieweb.org/) `entry` objects.
+ *
+ * Everything here is **pure**: a feed body (and the format-sniffing on its
+ * content type) in, an array of JF2 entries out, each carrying a stable `_id`
+ * derived from the source's own identifier so the store can dedupe across polls.
+ * The XML formats go through the dependency-free reader in {@link ./xml}.
+ *
+ * @packageDocumentation
+ */
+
+import {
+  child,
+  children,
+  childText,
+  parseXml,
+  text,
+  type XmlElement,
+} from "./xml";
+
+/** A JF2 author card. */
+export interface Jf2Author {
+  readonly type: "card";
+  readonly name?: string;
+  readonly url?: string;
+  readonly photo?: string;
+}
+
+/** Structured JF2 content (`html` and/or its plain-text rendering). */
+export interface Jf2Content {
+  readonly html?: string;
+  readonly text?: string;
+}
+
+/** A normalised JF2 timeline entry. */
+export interface Jf2Entry {
+  readonly type: "entry";
+  /**
+   * Stable per-entry identifier, derived from the source's own id/guid/url. The
+   * store keys timeline rows on this so re-polling a feed does not duplicate
+   * entries.
+   */
+  readonly _id: string;
+  readonly url?: string;
+  readonly published?: string;
+  readonly name?: string;
+  readonly summary?: string;
+  readonly content?: Jf2Content;
+  readonly author?: Jf2Author;
+  readonly category?: readonly string[];
+  readonly photo?: readonly string[];
+  readonly "in-reply-to"?: string;
+  readonly "like-of"?: string;
+  /** Read flag, attached by the store on read (never by the parser). */
+  readonly _is_read?: boolean;
+}
+
+/** Resolve `href` against `base`, returning an absolute URL or the input. */
+function absolute(href: string, base: string): string {
+  try {
+    return new URL(href, base).toString();
+  } catch {
+    return href;
+  }
+}
+
+/**
+ * Derive a stable entry id from the best available identifier. A feed-supplied
+ * id/guid/url is used verbatim; otherwise a short hash of the entry's content
+ * keeps re-polls from duplicating an entry that has no stable id of its own.
+ */
+function entryId(candidate: string | undefined, fallback: string): string {
+  const basis =
+    candidate && candidate.trim() !== "" ? candidate.trim() : fallback;
+  return basis;
+}
+
+/** A small, stable non-cryptographic hash (FNV-1a) rendered as base36. */
+function hash(input: string): string {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < input.length; i++) {
+    h ^= input.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return (h >>> 0).toString(36);
+}
+
+function defined<T extends object>(obj: T): T {
+  for (const key of Object.keys(obj) as (keyof T)[]) {
+    if (obj[key] === undefined) delete obj[key];
+  }
+  return obj;
+}
+
+// --- JSON Feed --------------------------------------------------------------
+
+interface JsonFeedItem {
+  id?: unknown;
+  url?: unknown;
+  title?: unknown;
+  content_html?: unknown;
+  content_text?: unknown;
+  summary?: unknown;
+  date_published?: unknown;
+  tags?: unknown;
+  image?: unknown;
+  banner_image?: unknown;
+  authors?: unknown;
+  author?: unknown;
+}
+
+function str(value: unknown): string | undefined {
+  return typeof value === "string" && value !== "" ? value : undefined;
+}
+
+function jsonFeedAuthor(
+  item: JsonFeedItem,
+  feedAuthor?: Jf2Author,
+): Jf2Author | undefined {
+  const authors = Array.isArray(item.authors) ? item.authors : undefined;
+  const raw =
+    (authors?.[0] as Record<string, unknown> | undefined) ??
+    (item.author as Record<string, unknown> | undefined);
+  if (raw && typeof raw === "object") {
+    const card = defined({
+      type: "card" as const,
+      name: str(raw.name),
+      url: str(raw.url),
+      photo: str(raw.avatar),
+    });
+    if (card.name || card.url || card.photo) return card;
+  }
+  return feedAuthor;
+}
+
+function parseJsonFeed(body: string, baseUrl: string): Jf2Entry[] {
+  let doc: Record<string, unknown>;
+  try {
+    doc = JSON.parse(body) as Record<string, unknown>;
+  } catch {
+    return [];
+  }
+  const items = Array.isArray(doc.items) ? doc.items : [];
+  const feedAuthorRaw =
+    (Array.isArray(doc.authors)
+      ? (doc.authors[0] as Record<string, unknown>)
+      : undefined) ?? (doc.author as Record<string, unknown> | undefined);
+  const feedAuthor =
+    feedAuthorRaw && typeof feedAuthorRaw === "object"
+      ? defined({
+          type: "card" as const,
+          name: str(feedAuthorRaw.name),
+          url: str(feedAuthorRaw.url),
+          photo: str(feedAuthorRaw.avatar),
+        })
+      : undefined;
+
+  const entries: Jf2Entry[] = [];
+  for (const raw of items) {
+    if (!raw || typeof raw !== "object") continue;
+    const item = raw as JsonFeedItem;
+    const url = str(item.url);
+    const html = str(item.content_html);
+    const textContent = str(item.content_text);
+    const content =
+      html || textContent ? defined({ html, text: textContent }) : undefined;
+    const tags = Array.isArray(item.tags)
+      ? item.tags.filter((t): t is string => typeof t === "string")
+      : undefined;
+    const image = str(item.image) ?? str(item.banner_image);
+    const id = entryId(
+      str(item.id) ?? url,
+      hash(`${textContent ?? html ?? str(item.title) ?? ""}${url ?? ""}`),
+    );
+    entries.push(
+      defined({
+        type: "entry" as const,
+        _id: id,
+        url: url ? absolute(url, baseUrl) : undefined,
+        published: str(item.date_published),
+        name: str(item.title),
+        summary: str(item.summary),
+        content,
+        author: jsonFeedAuthor(item, feedAuthor),
+        category: tags && tags.length > 0 ? tags : undefined,
+        photo: image ? [absolute(image, baseUrl)] : undefined,
+      }) as Jf2Entry,
+    );
+  }
+  return entries;
+}
+
+// --- Atom -------------------------------------------------------------------
+
+/** The `href` of the best Atom `<link>`: prefer `rel="alternate"`, else first. */
+function atomLink(entry: XmlElement, baseUrl: string): string | undefined {
+  const links = children(entry, "link");
+  let fallback: string | undefined;
+  for (const link of links) {
+    const href = link.attrs.href;
+    if (!href) continue;
+    const rel = link.attrs.rel ?? "alternate";
+    if (rel === "alternate") return absolute(href, baseUrl);
+    fallback ??= absolute(href, baseUrl);
+  }
+  return fallback;
+}
+
+function atomAuthor(scope: XmlElement, baseUrl: string): Jf2Author | undefined {
+  const author = child(scope, "author");
+  if (author === null) return undefined;
+  const name = childText(author, "name");
+  const uri = childText(author, "uri");
+  const card = defined({
+    type: "card" as const,
+    name: name || undefined,
+    url: uri ? absolute(uri, baseUrl) : undefined,
+  });
+  return card.name || card.url ? card : undefined;
+}
+
+function parseAtom(feed: XmlElement, baseUrl: string): Jf2Entry[] {
+  const feedAuthor = atomAuthor(feed, baseUrl);
+  const entries: Jf2Entry[] = [];
+  for (const entry of children(feed, "entry")) {
+    const url = atomLink(entry, baseUrl);
+    const contentEl = child(entry, "content");
+    const summary = childText(entry, "summary");
+    const contentHtml = contentEl ? text(contentEl) : "";
+    const content = contentHtml
+      ? { html: contentHtml }
+      : summary
+        ? { html: summary }
+        : undefined;
+    const categories = children(entry, "category")
+      .map((c) => c.attrs.term ?? "")
+      .filter((t) => t !== "");
+    const id = entryId(
+      childText(entry, "id") || url,
+      hash(`${childText(entry, "title")}${url ?? ""}`),
+    );
+    entries.push(
+      defined({
+        type: "entry" as const,
+        _id: id,
+        url,
+        published:
+          childText(entry, "published") ||
+          childText(entry, "updated") ||
+          undefined,
+        name: childText(entry, "title") || undefined,
+        summary: summary || undefined,
+        content,
+        author: atomAuthor(entry, baseUrl) ?? feedAuthor,
+        category: categories.length > 0 ? categories : undefined,
+      }) as Jf2Entry,
+    );
+  }
+  return entries;
+}
+
+// --- RSS 2.0 / RDF ----------------------------------------------------------
+
+function rssAuthor(item: XmlElement): Jf2Author | undefined {
+  const creator = childText(item, "dc:creator") || childText(item, "author");
+  if (creator === "") return undefined;
+  // RSS `<author>` is an email; `dc:creator` is a display name. Either way we
+  // surface the string as the card name (an email is the best we have).
+  return { type: "card", name: creator };
+}
+
+function parseRss(root: XmlElement, baseUrl: string): Jf2Entry[] {
+  // RSS 2.0 nests items under `<channel>`; RDF (RSS 1.0) lists them at the root.
+  const channel = child(root, "channel");
+  const scope = channel ?? root;
+  const items = [...children(scope, "item"), ...children(root, "item")];
+  const seen = new Set<XmlElement>();
+  const entries: Jf2Entry[] = [];
+  for (const item of items) {
+    if (seen.has(item)) continue;
+    seen.add(item);
+    const link = childText(item, "link");
+    const guidEl = child(item, "guid");
+    const guid = guidEl ? text(guidEl) : "";
+    const url = link
+      ? absolute(link, baseUrl)
+      : guid
+        ? absolute(guid, baseUrl)
+        : undefined;
+    const encoded = childText(item, "content:encoded");
+    const description = childText(item, "description");
+    const body = encoded || description;
+    const content = body ? { html: body } : undefined;
+    const categories = children(item, "category")
+      .map((c) => text(c))
+      .filter((t) => t !== "");
+    const id = entryId(
+      guid || link,
+      hash(`${childText(item, "title")}${url ?? ""}`),
+    );
+    entries.push(
+      defined({
+        type: "entry" as const,
+        _id: id,
+        url,
+        published:
+          childText(item, "pubdate") || childText(item, "dc:date") || undefined,
+        name: childText(item, "title") || undefined,
+        summary: encoded && description ? description : undefined,
+        content,
+        author: rssAuthor(item),
+        category: categories.length > 0 ? categories : undefined,
+      }) as Jf2Entry,
+    );
+  }
+  return entries;
+}
+
+// --- Ordering ---------------------------------------------------------------
+
+/** Parse an entry's `published` to epoch milliseconds, or `null`. */
+function publishedMs(entry: Jf2Entry): number | null {
+  if (!entry.published) return null;
+  const ms = Date.parse(entry.published);
+  return Number.isFinite(ms) ? ms : null;
+}
+
+/**
+ * Order a feed's entries oldest-first for insertion, so the newest entry is
+ * appended last and receives the highest paging `seq`.
+ *
+ * When the entries carry dates they are sorted chronologically (a feed's own
+ * order is not trusted). When none do — some `h-feed` pages — the feed is
+ * assumed newest-first and simply reversed. The sort is stable, so undated
+ * entries keep their relative position among dated ones.
+ */
+export function orderEntriesForInsert(
+  entries: readonly Jf2Entry[],
+): Jf2Entry[] {
+  const anyDated = entries.some((entry) => publishedMs(entry) !== null);
+  if (!anyDated) return [...entries].reverse();
+  return entries
+    .map((entry, index) => ({ entry, index, ms: publishedMs(entry) }))
+    .sort((a, b) => {
+      // Undated entries sort as oldest (lowest seq); ties keep feed order.
+      const am = a.ms ?? Number.NEGATIVE_INFINITY;
+      const bm = b.ms ?? Number.NEGATIVE_INFINITY;
+      return am === bm ? a.index - b.index : am - bm;
+    })
+    .map((wrapped) => wrapped.entry);
+}
+
+// --- Dispatch ---------------------------------------------------------------
+
+/** Whether a content type / body looks like JSON Feed. */
+function looksLikeJson(contentType: string, body: string): boolean {
+  if (/\bjson\b/i.test(contentType)) return true;
+  const trimmed = body.trimStart();
+  return (
+    trimmed.startsWith("{") &&
+    /"version"\s*:\s*"https:\/\/jsonfeed\.org/.test(trimmed)
+  );
+}
+
+/**
+ * Parse a fetched feed body into JF2 entries, sniffing the format from the
+ * content type and the document root. Returns `[]` for an unrecognised or
+ * unparseable body (the caller treats an empty parse as "nothing new").
+ *
+ * `h-feed` HTML is **not** handled here — it needs the runtime's `HTMLRewriter`
+ * and lives in {@link ./hfeed}; this module is pure and runtime-free.
+ */
+export function parseFeed(
+  body: string,
+  contentType: string,
+  baseUrl: string,
+): Jf2Entry[] {
+  if (looksLikeJson(contentType, body)) {
+    return parseJsonFeed(body, baseUrl);
+  }
+  const root = parseXml(body);
+  if (root === null) return [];
+  if (root.name === "feed") return parseAtom(root, baseUrl);
+  if (root.name === "rss" || root.name === "rdf:rdf")
+    return parseRss(root, baseUrl);
+  // Some JSON feeds are served with a generic content type and no version tag.
+  if (body.trimStart().startsWith("{")) return parseJsonFeed(body, baseUrl);
+  return [];
+}

package/src/log.ts

@@ -0,0 +1,46 @@
+/**
+ * `@dwk/microsub` — structured observability event taxonomy.
+ *
+ * Microsub consumes DPoP-bound access tokens, mutates the user's reading state,
+ * and fetches arbitrary URLs, so an authorization rejection, a validation
+ * rejection, or an SSRF block that is silently swallowed is an operational blind
+ * spot. Logging and metrics are opt-in via an injected {@link Logger} and
+ * {@link Metrics} (see `@dwk/log`) and **share this one vocabulary**: the same
+ * dotted event name is passed to the logger and the metrics sink so a log line
+ * and its counter line up. See `spec/observability.md`.
+ *
+ * Fields follow the redaction policy: only machine-readable reason codes, the
+ * action verb, scopes, counts, and a sanitized host — never tokens, entry
+ * bodies, or full URLs.
+ *
+ * @packageDocumentation
+ */
+
+/** Stable event names emitted by `@dwk/microsub`. */
+export const MicrosubLogEvent = {
+  /**
+   * A request failed authorization (missing/invalid token, subject mismatch,
+   * DPoP failure, revocation, or insufficient scope). Fields: `reason` (the
+   * OAuth/Microsub error code), `status`.
+   */
+  AuthRejected: "microsub.auth.rejected",
+  /**
+   * A well-formed-but-invalid request was rejected before any mutation (unknown
+   * action/method, missing `channel`/`url`, bad cursor). Field: `reason`.
+   */
+  RequestRejected: "microsub.request.rejected",
+  /** A channel/follow/timeline action completed. Fields: `action`, `method`. */
+  ActionCompleted: "microsub.action.completed",
+  /** A scheduled poll enqueued jobs for the followed feeds. Field: `feeds`. */
+  PollScheduled: "microsub.poll.scheduled",
+  /** A feed poll completed. Fields: `added`, `host`. */
+  FeedPolled: "microsub.feed.polled",
+  /** A queued poll job threw and is being retried. Fields: `error`, `host`. */
+  PollRetry: "microsub.poll.retry",
+  /** An outbound fetch was refused on SSRF grounds. Fields: `reason`, `host`. */
+  SsrfBlocked: "microsub.ssrf.blocked",
+} as const;
+
+/** Union of the event-name string literals in {@link MicrosubLogEvent}. */
+export type MicrosubLogEvent =
+  (typeof MicrosubLogEvent)[keyof typeof MicrosubLogEvent];

package/src/poll.ts

@@ -0,0 +1,72 @@
+/**
+ * `@dwk/microsub` — the scheduled poller.
+ *
+ * A Cron Trigger drives {@link createMicrosubPoller}: on each run it lists every
+ * distinct followed feed and enqueues one poll job per feed onto
+ * `MICROSUB_QUEUE`, so the slow fetch-and-parse work happens off this path, on
+ * the queue, with retries and backoff (see {@link ./consumer}). Polling never
+ * runs inline on the Microsub read path — the read path serves stored entries
+ * only (`spec/packages/microsub.md`).
+ *
+ * @packageDocumentation
+ */
+
+import type {
+  ExecutionContext,
+  ScheduledController,
+} from "@cloudflare/workers-types";
+
+import {
+  resolveConfig,
+  type MicrosubConfig,
+  type MicrosubEnv,
+  type ResolvedConfig,
+} from "./config";
+import { MicrosubLogEvent } from "./log";
+import { createMicrosubStore } from "./store";
+
+/** A `scheduled`-compatible Worker handler. */
+export type MicrosubScheduledHandler = (
+  controller: ScheduledController,
+  env: MicrosubEnv,
+  ctx: ExecutionContext,
+) => Promise<void>;
+
+function emit(
+  config: ResolvedConfig,
+  event: string,
+  fields?: Record<string, unknown>,
+): void {
+  config.logger.info(event, fields);
+  config.metrics.count(event, fields);
+}
+
+/**
+ * Build the scheduled poller. Fails loudly if `MICROSUB_DB` or `MICROSUB_QUEUE`
+ * is missing — no silent degradation (composition contract).
+ */
+export function createMicrosubPoller(
+  config: MicrosubConfig,
+): MicrosubScheduledHandler {
+  const resolved = resolveConfig(config);
+  return async (_controller, env, _ctx) => {
+    if (!env.MICROSUB_DB) {
+      throw new Error(
+        "@dwk/microsub: missing required D1 binding `MICROSUB_DB`",
+      );
+    }
+    if (!env.MICROSUB_QUEUE) {
+      throw new Error(
+        "@dwk/microsub: missing required binding `MICROSUB_QUEUE`",
+      );
+    }
+    const store = createMicrosubStore(env);
+    const feeds = await store.distinctFeedUrls();
+    await Promise.all(
+      feeds.map((feedUrl) =>
+        env.MICROSUB_QUEUE.send({ kind: "poll", feedUrl }),
+      ),
+    );
+    emit(resolved, MicrosubLogEvent.PollScheduled, { feeds: feeds.length });
+  };
+}

package/src/queue.ts

@@ -0,0 +1,26 @@
+/**
+ * `@dwk/microsub` — queued job shapes.
+ *
+ * Polling — the slow, failure-prone work of fetching and parsing every followed
+ * feed — runs off the request path, on a queue with retries and backoff
+ * (`spec/packages/microsub.md`). The scheduled poller enqueues one job per
+ * distinct followed feed; the consumer fetches and appends. A single job kind
+ * flows through the queue today, discriminated by `kind` so the shape can grow.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Poll a single feed: fetch it (conditionally, via the stored `ETag` /
+ * `Last-Modified`), parse to JF2, and append new entries to every channel
+ * following it. The feed URL is the already-resolved subscription URL, not the
+ * page the user originally typed.
+ */
+export interface PollJob {
+  readonly kind: "poll";
+  /** The resolved feed URL to fetch. */
+  readonly feedUrl: string;
+}
+
+/** A job on the Microsub poll queue. */
+export type MicrosubJob = PollJob;

package/src/replay.ts

@@ -0,0 +1,68 @@
+/**
+ * Strongly-consistent, short-TTL record of accepted DPoP proof `jti`s, so a
+ * captured proof cannot be replayed within its acceptance window to repeat a
+ * state-changing request. `@dwk/dpop` verifies a proof's freshness but, per
+ * RFC 9449, delegates replay detection to the caller via the returned `jti`;
+ * this is that caller-side record.
+ *
+ * The table lives in D1 (the strongly-consistent `MICROSUB_DB`) — never KV,
+ * where ~60 s of staleness would let a replayed proof slip through (see
+ * `spec/non-functional-requirements.md`). Rows are reaped once their proof can
+ * no longer be cryptographically accepted, so the table only ever tracks the
+ * live window.
+ *
+ * @packageDocumentation
+ */
+
+import type { MicrosubStoreEnv } from "./store";
+
+/** Storage interface over accepted DPoP proof `jti`s. */
+export interface DpopReplayStore {
+  /** Create the schema if absent. Idempotent. */
+  init(): Promise<void>;
+  /**
+   * Atomically record an accepted proof `jti`. Returns `true` when the `jti`
+   * was unseen (and is now recorded), or `false` when it was already present —
+   * i.e. a replay. `expiresAt`/`now` are seconds since the epoch.
+   */
+  recordProof(jti: string, expiresAt: number, now: number): Promise<boolean>;
+}
+
+const SCHEMA = `CREATE TABLE IF NOT EXISTS microsub_dpop_proofs (
+   jti TEXT PRIMARY KEY,
+   expires_at INTEGER NOT NULL
+ )`;
+
+/**
+ * Create the D1-backed {@link DpopReplayStore}. Fails loudly if the required
+ * `MICROSUB_DB` binding is missing — no silent degradation (composition
+ * contract); silently skipping replay detection would be a security bug.
+ */
+export function createDpopReplayStore(env: MicrosubStoreEnv): DpopReplayStore {
+  if (!env.MICROSUB_DB) {
+    throw new Error("@dwk/microsub: missing required D1 binding `MICROSUB_DB`");
+  }
+  const db = env.MICROSUB_DB;
+
+  return {
+    async init() {
+      await db.prepare(SCHEMA).run();
+    },
+
+    async recordProof(jti, expiresAt, now) {
+      const results = await db.batch([
+        db.prepare(SCHEMA),
+        db
+          .prepare("DELETE FROM microsub_dpop_proofs WHERE expires_at <= ?")
+          .bind(now),
+        db
+          .prepare(
+            "INSERT OR IGNORE INTO microsub_dpop_proofs (jti, expires_at) VALUES (?, ?)",
+          )
+          .bind(jti, expiresAt),
+      ]);
+      // The INSERT is the third statement; `meta.changes` is 0 on a replay.
+      return (results[2]?.meta.changes ?? 0) > 0;
+    },
+  };
+}