@readymadeit/mcp 0.0.0-stage → 0.1.0

package/bundle/hkp-node/src/services/rss-parse.ts

@@ -0,0 +1,281 @@
+/**
+ * Reading a syndication feed, without a dependency to do it.
+ *
+ * RSS 2.0, RSS 1.0 (RDF) and Atom are three documents that say the same few
+ * things under different names, and this narrows them to one shape: an item has
+ * a title, somewhere to read it, when it was published, and a line about what
+ * it is. Everything else a feed carries is dropped here rather than downstream,
+ * because a board that merges several feeds can only sort and show fields all
+ * of them have.
+ *
+ * It is a scanner over the markup rather than a general XML parser. That is the
+ * honest description of what it can do: it finds the element boundaries these
+ * three formats use, unwraps CDATA, and decodes the entities that appear in
+ * feed text. A document that would need a real parser to read — arbitrary
+ * namespaces, nested elements of the same name — is out of its range, and the
+ * answer for one is a proper parser, not more regular expressions.
+ */
+
+/** One article, in the vocabulary every feed format can be reduced to. */
+export type FeedItem = {
+  /** The feed's own identity for the item — guid, id, or the link. */
+  id: string;
+  title: string;
+  /** Where to read it. Empty when the feed offered no readable address. */
+  link: string;
+  author: string;
+  /** A line about the item, with its markup stripped. */
+  summary: string;
+  /** When it was published, ISO 8601, or "" when the feed did not say. */
+  published: string;
+  /**
+   * The same instant as `YYYY-MM-DD HH:mm`, in UTC.
+   *
+   * A merged list shows a date on every row, so deriving one is work every
+   * consumer would otherwise repeat — and a board has no date formatter to
+   * repeat it with. UTC rather than a local zone because the formatting happens
+   * wherever the runtime is, which is not where the reader is.
+   */
+  publishedLabel: string;
+  /**
+   * The same instant in epoch milliseconds, which is what sorting uses.
+   * An item with no date sorts as 0 — last — rather than as now, so a feed
+   * that omits dates cannot crowd out the ones that keep them.
+   */
+  publishedMs: number;
+};
+
+export type ParsedFeed = {
+  /** What the feed calls itself, for a board that did not name it. */
+  title: string;
+  items: FeedItem[];
+};
+
+/**
+ * `<item>` in RSS and RDF, `<entry>` in Atom.
+ *
+ * The opening tag is captured alongside the body because RDF puts the item's
+ * address in an attribute of it (`rdf:about`) rather than in an element.
+ */
+const ITEM_BLOCK = /<(item|entry)((?:\s[^>]*)?)>([\s\S]*?)<\/\1\s*>/gi;
+
+const COMMENT = /<!--[\s\S]*?-->/g;
+
+const CDATA = /^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/;
+
+const NAMED_ENTITIES: Record<string, string> = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  nbsp: " ",
+};
+
+/** One pass of entity decoding — the unit the two callers compose. */
+function decodeOnce(value: string): string {
+  return value.replace(
+    /&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g,
+    (match, body: string): string => {
+      if (body.startsWith("#x") || body.startsWith("#X")) {
+        const code = Number.parseInt(body.slice(2), 16);
+        return Number.isFinite(code) ? String.fromCodePoint(code) : match;
+      }
+      if (body.startsWith("#")) {
+        const code = Number.parseInt(body.slice(1), 10);
+        return Number.isFinite(code) ? String.fromCodePoint(code) : match;
+      }
+      const named = NAMED_ENTITIES[body.toLowerCase()];
+      return named ?? match;
+    },
+  );
+}
+
+/**
+ * Decodes the entities feed text actually contains.
+ *
+ * Feeds are written by publishing systems escaping HTML into XML, so the same
+ * title arrives as `&amp;amp;` from one and `&amp;` from another. Decoding runs
+ * twice for that reason — a second pass turns what the first pass revealed as
+ * `&amp;` into `&` — and no further, since a third would start rewriting text
+ * that legitimately contains an ampersand followed by a word.
+ */
+function decodeEntities(text: string): string {
+  return decodeOnce(decodeOnce(text));
+}
+
+/** Unwraps CDATA, decodes entities, and collapses the whitespace markup left. */
+function text(raw: string | undefined): string {
+  if (!raw) {
+    return "";
+  }
+  const cdata = raw.match(CDATA);
+  return decodeEntities(cdata ? cdata[1] : raw)
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/**
+ * The same, for a value that arrived as HTML — a description, a summary.
+ *
+ * The markup may be escaped (`&lt;p&gt;`) or it may be markup (inside CDATA),
+ * and a feed reader sees both. So the passes go decode, strip, decode: the
+ * first reveals escaped tags as tags, the strip removes them whichever way they
+ * arrived, and the second decodes the entities the text itself contains. Doing
+ * it in the other order leaves a feed's `&lt;span&gt;` sitting in the summary as
+ * visible markup.
+ */
+function plainText(raw: string | undefined): string {
+  if (!raw) {
+    return "";
+  }
+  const cdata = raw.match(CDATA);
+  const body = decodeOnce(cdata ? cdata[1] : raw);
+  return decodeOnce(
+    body
+      .replace(COMMENT, "")
+      .replace(/<(script|style)\b[\s\S]*?<\/\1\s*>/gi, "")
+      .replace(/<[^>]*>/g, " "),
+  )
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/**
+ * The content of the first of `names` that appears, with any namespace prefix
+ * ignored — `dc:date` answers to "date" because what distinguishes the formats
+ * is the prefix, and the prefix is exactly what does not matter here.
+ */
+function firstTag(block: string, names: string[]): string | undefined {
+  for (const name of names) {
+    const pattern = new RegExp(
+      `<(?:[A-Za-z0-9_.-]+:)?${name}(?:\\s[^>]*)?>([\\s\\S]*?)</(?:[A-Za-z0-9_.-]+:)?${name}\\s*>`,
+      "i",
+    );
+    const match = block.match(pattern);
+    if (match) {
+      return match[1];
+    }
+  }
+  return undefined;
+}
+
+const ATTRIBUTE = (name: string) =>
+  new RegExp(`\\b${name}\\s*=\\s*("([^"]*)"|'([^']*)')`, "i");
+
+function attribute(tag: string, name: string): string | undefined {
+  const match = tag.match(ATTRIBUTE(name));
+  if (!match) {
+    return undefined;
+  }
+  return match[2] ?? match[3];
+}
+
+/**
+ * Where to read the item.
+ *
+ * RSS puts the address in the element's text; Atom puts it in the `href` of a
+ * `<link>` and may offer several. Of those, `alternate` is the human-readable
+ * one, which is the only kind worth handing a reader — `self` points back at
+ * the feed and `enclosure` at a media file.
+ */
+function itemLink(block: string, openTag: string): string {
+  const inline = firstTag(block, ["link"]);
+  if (inline && !/^\s*<|^\s*$/.test(inline)) {
+    return text(inline);
+  }
+
+  const links = block.match(/<(?:[A-Za-z0-9_.-]+:)?link\b[^>]*\/?>/gi) ?? [];
+  const usable = links.filter((tag) => {
+    const rel = (attribute(tag, "rel") ?? "alternate").toLowerCase();
+    return rel === "alternate";
+  });
+  const chosen = usable[0] ?? links[0];
+  const href = chosen ? attribute(chosen, "href") : undefined;
+  if (href) {
+    return decodeEntities(href).trim();
+  }
+
+  // RDF names the item by its address, on the item's own tag, which is the
+  // only place some 1.0 feeds put it at all.
+  const about = attribute(openTag, "about");
+  return about ? decodeEntities(about).trim() : "";
+}
+
+/**
+ * When the item was published.
+ *
+ * The publication date is preferred over the update date wherever a feed
+ * carries both: a list sorted by "last touched" reorders itself when a
+ * publisher fixes a typo in something from last week, which is not what a
+ * reader looking for what is new is asking to see.
+ */
+function itemDate(block: string): {
+  published: string;
+  publishedMs: number;
+  publishedLabel: string;
+} {
+  const raw = text(
+    firstTag(block, ["pubDate", "published", "date", "issued", "updated"]),
+  );
+  const parsed = raw ? Date.parse(raw) : Number.NaN;
+  if (!Number.isFinite(parsed)) {
+    return { published: "", publishedMs: 0, publishedLabel: "" };
+  }
+  const iso = new Date(parsed).toISOString();
+  return {
+    published: iso,
+    publishedMs: parsed,
+    publishedLabel: `${iso.slice(0, 10)} ${iso.slice(11, 16)}`,
+  };
+}
+
+function itemAuthor(block: string): string {
+  // Atom wraps the author in a <name>; RSS puts an address in <author> and a
+  // display name in <dc:creator>.
+  const author = firstTag(block, ["author"]);
+  if (author && /<(?:[A-Za-z0-9_.-]+:)?name[\s>]/i.test(author)) {
+    return text(firstTag(author, ["name"]));
+  }
+  return text(author ?? firstTag(block, ["creator"]));
+}
+
+/**
+ * Reads a feed document into items.
+ *
+ * Nothing is rejected for being the wrong format: a document with no items
+ * parses to none, which is what a board shows as a feed that said nothing
+ * rather than as an error it cannot act on.
+ */
+export function parseFeed(xml: string): ParsedFeed {
+  const document = xml.replace(COMMENT, "");
+
+  const blocks: { open: string; body: string }[] = [];
+  for (const match of document.matchAll(ITEM_BLOCK)) {
+    blocks.push({ open: match[2], body: match[3] });
+  }
+
+  // The feed's own title is the first one outside any item, so the items are
+  // taken out of the way before looking for it.
+  const channel = document.replace(ITEM_BLOCK, "");
+  const title = text(firstTag(channel, ["title"]));
+
+  const items = blocks.map(({ open, body: block }): FeedItem => {
+    const link = itemLink(block, open);
+    const { published, publishedMs, publishedLabel } = itemDate(block);
+    return {
+      id: text(firstTag(block, ["guid", "id"])) || link,
+      title: text(firstTag(block, ["title"])) || "(untitled)",
+      link,
+      author: itemAuthor(block),
+      summary: plainText(
+        firstTag(block, ["description", "summary", "content", "encoded"]),
+      ),
+      published,
+      publishedMs,
+      publishedLabel,
+    };
+  });
+
+  return { title, items };
+}

package/bundle/hkp-node/src/services/rss.ts

@@ -0,0 +1,392 @@
+/**
+ * Service Documentation
+ * Service ID: rss
+ * Service Name: RSS
+ * Runtime: hkp-node
+ * Modes: none
+ * Key Config: feeds, limit, timeoutMs, summaryChars, __hkpMount (a feed named
+ *             by reference); addFeed/removeFeed/refresh as commands
+ * IO: in=any (ignored, a trigger) -> out=null immediately; the merged article
+ *     list is pushed through the rest of the pipeline when the feeds answer
+ * Arrays: emits an array of articles
+ * Binary: not applicable
+ *
+ * Several feeds read as one list.
+ *
+ * A reader's unit is not the feed, it is the article: the question being asked
+ * is "what is new", and the answer spans every feed subscribed to. So this
+ * service fetches them together, reduces the three syndication formats to one
+ * article shape (`rss-parse`), merges the results and sorts them newest first.
+ * A board wanting one feed asks for one feed; nothing about the contract
+ * changes.
+ *
+ * It lives on hkp-node rather than the browser because a feed is served by
+ * whoever publishes it, and almost none of them send the CORS header a browser
+ * would need. Reading feeds is therefore something a board can only do from a
+ * runtime that is not a browser — not a preference.
+ *
+ * The feeds are **fetched concurrently and failures are kept apart**. One feed
+ * that is down, slow or no longer a feed produces an entry in `errors` and the
+ * others still arrive: a reader missing one source is still a reader, while one
+ * that shows nothing because a single site is having a bad morning is not.
+ */
+import {
+  HostedService,
+  JsonRecord,
+  ProcessContext,
+  RuntimeHost,
+  RuntimeNotification,
+  ServiceConfiguration,
+  ServiceRegistryEntry,
+} from "../types";
+import { FeedItem, parseFeed } from "./rss-parse";
+import { MOUNT_FIELD, parseMountRef } from "../coordinator/mount";
+import { failureReason } from "../egress";
+
+export const rssDescriptor: ServiceRegistryEntry = {
+  serviceId: "rss",
+  serviceName: "RSS",
+};
+
+/** A feed as a board subscribes to it: where it is, and what to call it. */
+type Feed = {
+  url: string;
+  /** What the list shows as the source. Empty until the feed says its own. */
+  name: string;
+};
+
+/** An article, as it travels: a feed item that remembers where it came from. */
+export type Article = FeedItem & {
+  feed: string;
+  feedUrl: string;
+};
+
+const DEFAULT_LIMIT = 60;
+const DEFAULT_TIMEOUT_MS = 10000;
+const DEFAULT_SUMMARY_CHARS = 240;
+
+/** A feed entry as it may be written: a bare URL, or a URL with a name. */
+function readFeed(value: unknown): Feed | null {
+  if (typeof value === "string") {
+    return value.trim() ? { url: value.trim(), name: "" } : null;
+  }
+  if (value && typeof value === "object") {
+    const record = value as JsonRecord;
+    const url = typeof record.url === "string" ? record.url.trim() : "";
+    const name = typeof record.name === "string" ? record.name.trim() : "";
+    return url ? { url, name } : null;
+  }
+  return null;
+}
+
+function readFeedList(value: unknown): Feed[] | null {
+  if (!Array.isArray(value)) {
+    return null;
+  }
+  const feeds: Feed[] = [];
+  for (const entry of value) {
+    const feed = readFeed(entry);
+    // A feed already subscribed to is not added twice: the list is what a
+    // person reads, and the same source listed twice doubles every article.
+    if (feed && !feeds.some((existing) => existing.url === feed.url)) {
+      feeds.push(feed);
+    }
+  }
+  return feeds;
+}
+
+/** The host, as the name to show for a feed that did not offer one. */
+function hostOf(url: string): string {
+  try {
+    return new URL(url).hostname.replace(/^www\./, "");
+  } catch {
+    return url;
+  }
+}
+
+export class RssService implements HostedService {
+  readonly serviceId = rssDescriptor.serviceId;
+  readonly serviceName = rssDescriptor.serviceName;
+  readonly uuid: string;
+
+  private host: RuntimeHost | null = null;
+  private feeds: Feed[] = [];
+  private limit = DEFAULT_LIMIT;
+  private timeoutMs = DEFAULT_TIMEOUT_MS;
+  private summaryChars = DEFAULT_SUMMARY_CHARS;
+  private fetching = false;
+  /** The latest round begun; an earlier one finishing later is dropped. */
+  private round = 0;
+  /**
+   * The address a feed named by reference resolves to.
+   *
+   * A feed on this board rather than out on the web — one unit publishing what
+   * another reads — has no address until the board loads, so it is named the
+   * way every other endpoint is and resolved by the coordinator. One address,
+   * so one feed of a list may be a reference; the rest are URLs, which is what
+   * they are anyway.
+   */
+  private mount = "";
+
+  constructor(config: ServiceConfiguration) {
+    this.uuid = config.uuid;
+    if (config.state) {
+      this.configure(config.state);
+    }
+  }
+
+  setHost(host: RuntimeHost): void {
+    this.host = host;
+  }
+
+  /**
+   * The address a feed's url stands for: itself when it is a URL, the resolved
+   * mount when it is a reference, and nothing while a reference is unresolved.
+   */
+  private addressOf(url: string): string | null {
+    if (!parseMountRef(url)) {
+      return url;
+    }
+    return this.mount && !parseMountRef(this.mount) ? this.mount : null;
+  }
+
+  /**
+   * What the service is configured as — and not what it last read.
+   *
+   * This is the state a board is saved with, so the articles stay out of it:
+   * they are a fetch's result rather than a setting, they are large, and they
+   * are stale the moment they are written. The list reaches a panel or a facade
+   * as a notification, and a board that has just loaded gets one as soon as
+   * something triggers it.
+   */
+  getState(): JsonRecord {
+    return {
+      feeds: this.feeds.map((feed) => ({ ...feed })),
+      limit: this.limit,
+      timeoutMs: this.timeoutMs,
+      summaryChars: this.summaryChars,
+      fetching: this.fetching,
+      [MOUNT_FIELD]: this.mount,
+    };
+  }
+
+  configure(config: JsonRecord): JsonRecord {
+    const feeds = readFeedList(config.feeds);
+    if (feeds) {
+      this.feeds = feeds;
+    }
+
+    // Adding and removing are commands rather than settings: a facade has a
+    // field and a button, not an editor for the whole list, and rewriting
+    // `feeds` from a text input would mean the widget had to know the list it
+    // was changing.
+    const added = readFeed(config.addFeed);
+    if (added && !this.feeds.some((feed) => feed.url === added.url)) {
+      this.feeds = [...this.feeds, added];
+    }
+
+    const removed = readFeed(config.removeFeed);
+    if (removed) {
+      this.feeds = this.feeds.filter((feed) => feed.url !== removed.url);
+    }
+
+    // The list is announced whenever it changes, not only returned to whoever
+    // changed it: a panel and a facade both render it, and only one of them
+    // made the call.
+    if (feeds || added || removed) {
+      this._notify({ feeds: this.feeds.map((feed) => ({ ...feed })) });
+    }
+
+    if (typeof config.limit === "number" && config.limit > 0) {
+      this.limit = Math.floor(config.limit);
+    }
+    if (typeof config.timeoutMs === "number" && config.timeoutMs > 0) {
+      this.timeoutMs = config.timeoutMs;
+    }
+    if (typeof config.summaryChars === "number" && config.summaryChars >= 0) {
+      this.summaryChars = Math.floor(config.summaryChars);
+    }
+
+    // Fetching now is a command here as well as the service's process(),
+    // because a panel is the one caller that cannot use process(): a service on
+    // a remote runtime has no local instance for its UI to call, so configure
+    // is the only verb that reaches it. Same work either way — Injector's
+    // `inject` is the same arrangement for the same reason.
+    if (typeof config[MOUNT_FIELD] === "string") {
+      this.mount = config[MOUNT_FIELD] as string;
+    }
+
+    if (config.refresh) {
+      this.start((payload) => this._notify(payload as JsonRecord));
+    }
+
+    return this.getState();
+  }
+
+  /**
+   * Starts the round of fetches and stops the synchronous push.
+   *
+   * Like every service whose answer is a response, the articles do not exist
+   * when this returns — the rest of the pipeline is called with them once the
+   * feeds have answered.
+   */
+  process(
+    _input: unknown,
+    notify: (payload: unknown, instanceId?: string) => void,
+  ): unknown {
+    this.start(notify);
+    return null;
+  }
+
+  destroy(): void {
+    this.host = null;
+  }
+
+  // ── Private ───────────────────────────────────────────────────────────────
+
+  private _notify(payload: JsonRecord, instanceId: string = this.uuid): void {
+    this.host?.notify(payload, instanceId);
+  }
+
+  /** Begins a round, or says why there is nothing to fetch. */
+  private start(notify: (payload: unknown, instanceId?: string) => void): void {
+    if (this.feeds.length === 0) {
+      notify({ fetching: false, error: "No feeds configured" });
+      return;
+    }
+    void this.refresh(notify, this.host?.currentContext() ?? undefined);
+  }
+
+  private async refresh(
+    notify: (payload: unknown, instanceId?: string) => void,
+    context?: ProcessContext,
+  ): Promise<void> {
+    const feeds = this.feeds;
+    const round = ++this.round;
+    this.fetching = true;
+    notify({ fetching: true, error: "", sources: feeds.length });
+
+    const results = await Promise.all(
+      feeds.map((feed) => this.readFeed(feed)),
+    );
+    // A newer round began while this one waited — after a feed was added or
+    // removed, say — and its list is the one that holds. Reporting this one
+    // last would bring a removed feed's articles back.
+    if (round !== this.round) {
+      return;
+    }
+
+    const articles: Article[] = [];
+    const errors: { url: string; error: string }[] = [];
+    for (const result of results) {
+      if (result.error) {
+        errors.push({ url: result.feed.url, error: result.error });
+        continue;
+      }
+      articles.push(...result.articles);
+    }
+
+    // Newest first, across every feed — which is the whole point of reading
+    // them together rather than one after another.
+    articles.sort((a, b) => b.publishedMs - a.publishedMs);
+    const items = articles.slice(0, this.limit);
+
+    this.fetching = false;
+    notify({
+      fetching: false,
+      fetchedAt: new Date().toISOString(),
+      sources: feeds.length,
+      count: items.length,
+      // Said on every round, so a panel showing a failure shows this round's
+      // and not one still standing from an earlier one.
+      error: errors.length ? `${errors.length} of ${feeds.length} failed` : "",
+      errors,
+      items,
+    });
+
+    await this.push(items, notify, context);
+  }
+
+  private async readFeed(
+    feed: Feed,
+  ): Promise<{ feed: Feed; articles: Article[]; error?: string }> {
+    const address = this.addressOf(feed.url);
+    if (!address) {
+      // A reference nobody has published yet: the unit that serves this feed is
+      // still coming up. Not an error to act on, and not a URL to dial — the
+      // next round tries again, by which time the coordinator has usually
+      // handed the address over.
+      return { feed, articles: [], error: `Waiting for ${feed.url}` };
+    }
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.timeoutMs);
+    try {
+      const response = await fetch(address, {
+        headers: {
+          accept:
+            "application/rss+xml, application/atom+xml, application/xml, text/xml;q=0.9, */*;q=0.8",
+          // Some publishers answer a request without one with a redirect to a
+          // consent page, which is not a feed.
+          "user-agent": "Readymade/1.0 (+https://readymadeit.com)",
+        },
+        signal: controller.signal,
+      });
+      if (!response.ok) {
+        return {
+          feed,
+          articles: [],
+          error: `HTTP ${response.status}`,
+        };
+      }
+      const parsed = parseFeed(await response.text());
+      const name = feed.name || parsed.title || hostOf(feed.url);
+      return {
+        feed,
+        articles: parsed.items.map((item) => ({
+          ...item,
+          summary: this.summaryChars
+            ? item.summary.slice(0, this.summaryChars)
+            : "",
+          feed: name,
+          feedUrl: feed.url,
+        })),
+      };
+    } catch (err) {
+      const message = failureReason(err);
+      return {
+        feed,
+        articles: [],
+        // An abort is this service's own timeout, and says so — "This operation
+        // was aborted" tells a reader nothing about which feed was slow.
+        error: controller.signal.aborted
+          ? `Timed out after ${this.timeoutMs}ms`
+          : message,
+      };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  private async push(
+    items: Article[],
+    notify: (payload: unknown, instanceId?: string) => void,
+    context?: ProcessContext,
+  ): Promise<void> {
+    if (!this.host) {
+      return;
+    }
+    const output = await this.host.processFrom(
+      this.uuid,
+      items,
+      // No-op: the runtime already fans these out to its notification
+      // targets. Re-notifying through `notify` would deliver every one twice.
+      (_n: RuntimeNotification) => {},
+      context,
+    );
+    // A downstream service returning null means stop — honour it rather than
+    // driving the next runtime with a dead result.
+    if (output !== null && output !== undefined) {
+      this.host.emitResult(output);
+    }
+  }
+}