@transclude/core 0.1.0

package/src/extract.js

@@ -0,0 +1,339 @@
+// A fragment cut out of a document this framework did not compile.
+//
+// A page here declares its own regions: `<div id="x" fragment>` compiles to its
+// own render function and the same markup serves the region inline and alone.
+// Nothing in that path parses HTML, and nothing in this file runs against it.
+//
+// This is for the other case, a document somebody else wrote, where there is no
+// attribute to read and no compiler output to reuse. Extent is worked out from
+// what HTML already encodes: heading rank and definition-list structure. Nothing
+// else is guessed. An id says what a thing is, not where it ends.
+
+import { parse, serializeOuter } from 'parse5';
+
+/** Inline elements that may be standing in for the thing after them. */
+const INLINE = new Set(['a', 'span', 'em', 'i', 'b', 'strong', 'small', 'sub', 'sup']);
+
+/** Content that makes an element worth returning even with no text in it. */
+const EMBEDDED = new Set([
+  'img', 'svg', 'video', 'audio', 'iframe', 'canvas', 'object', 'picture', 'input',
+]);
+
+/** Elements that mean nothing outside a particular parent. */
+const CONTEXTUAL = new Set([
+  'li', 'dd', 'dt', 'td', 'th', 'tr', 'thead', 'tbody', 'tfoot', 'caption', 'col',
+  'colgroup', 'option', 'optgroup', 'figcaption', 'legend', 'summary', 'source',
+  'track', 'param',
+]);
+
+const HEADINGS = new Set(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']);
+
+// ---- node access -----------------------------------------------------------
+// Everything that touches a parse5 node is here, so a second tree shape is a
+// change to this block rather than to the rules below it.
+
+const isElement = (node) => typeof node.tagName === 'string';
+const tagOf = (node) => node.tagName ?? null;
+const attrOf = (node, name) => node.attrs?.find((a) => a.name === name)?.value ?? null;
+
+/**
+ * A `<template>`'s children live on `.content`, not `childNodes`, and they are
+ * inert: nothing in there is rendered. A fragment URL that returned template
+ * content would return something the source document never showed, so this does
+ * not descend into one and an id inside a template is not addressable.
+ */
+const kidsOf = (node) => (tagOf(node) === 'template' ? [] : (node.childNodes ?? []));
+
+const rankOf = (node) => (HEADINGS.has(tagOf(node)) ? Number(tagOf(node)[1]) : null);
+
+function textOf(node) {
+  if (node.nodeName === '#text') return node.value ?? '';
+  let out = '';
+  for (const child of kidsOf(node)) out += textOf(child);
+  return out;
+}
+
+function hasEmbedded(node) {
+  if (EMBEDDED.has(tagOf(node))) return true;
+  return kidsOf(node).some(hasEmbedded);
+}
+
+/** Every element in document order, template content excluded. */
+function* walk(node) {
+  for (const child of kidsOf(node)) {
+    if (isElement(child)) yield child;
+    yield* walk(child);
+  }
+}
+
+// ---- slugs -----------------------------------------------------------------
+
+/**
+ * A heading's text as an id, the way GitHub and MDN write one.
+ *
+ * Unicode letters and numbers are kept rather than folded to ASCII: a document
+ * whose headings are not in English should still be addressable in its own
+ * script.
+ *
+ * @param {string} text
+ * @returns {string} lowercased, punctuation dropped, spaces hyphenated
+ */
+export function slugify(text) {
+  return text
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s_-]/gu, '')
+    .trim()
+    .replace(/\s+/g, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+/**
+ * Every id the document offers, explicit and generated, worked out in one pass.
+ *
+ * The whole table is built before any single fragment is resolved. Computing a
+ * slug on demand would make the suffix a heading gets depend on which fragment
+ * was asked for, so the same URL would mean different things on different
+ * requests.
+ *
+ * @param {string} html
+ * @returns {object} the indexed document
+ */
+export function readDocument(html) {
+  return indexDocument(parse(html));
+}
+
+/**
+ * The same table, over a tree that has already been parsed.
+ *
+ * The proxy sanitizes and rewrites a foreign document before indexing it, and
+ * indexing first would leave the table naming elements the cleaning removed.
+ *
+ * @param {object} root a parse5 tree
+ * @returns {object} the same root, with its id table
+ */
+export function indexDocument(root) {
+  const ids = new Map();
+  const duplicates = new Set();
+  const headings = [];
+  const order = [];
+  const position = new Map();
+
+  for (const element of walk(root)) {
+    position.set(element, position.size);
+    const id = attrOf(element, 'id');
+    if (id) {
+      if (ids.has(id)) duplicates.add(id);
+      else {
+        ids.set(id, element);
+        order.push({ id, element, implicit: false });
+      }
+    }
+    if (HEADINGS.has(tagOf(element))) headings.push(element);
+  }
+
+  // Explicit ids win outright, so they are all reserved before the first slug
+  // is handed out.
+  const taken = new Set(ids.keys());
+  const slugs = new Map();
+
+  for (const heading of headings) {
+    if (attrOf(heading, 'id')) continue;
+
+    const base = slugify(textOf(heading));
+    if (!base) continue;
+
+    let slug = base;
+    for (let n = 1; taken.has(slug); n += 1) slug = `${base}-${n}`;
+
+    taken.add(slug);
+    slugs.set(slug, heading);
+    order.push({ id: slug, element: heading, implicit: true });
+  }
+
+  // `order` gathered explicit ids in document order and then appended the
+  // generated ones, so it is sorted once here rather than interleaved above.
+  order.sort((a, b) => position.get(a.element) - position.get(b.element));
+
+  return { root, ids, slugs, duplicates, order };
+}
+
+// ---- extent ----------------------------------------------------------------
+
+const siblingsOf = (node) => (node.parentNode ? kidsOf(node.parentNode) : []);
+
+/**
+ * The rank a sibling terminates a run at, or null if it does not terminate one.
+ *
+ * An `<hgroup>` counts as the heading it holds, so a run started before one
+ * stops at it the way it would stop at a bare heading.
+ */
+function terminatingRank(node) {
+  if (!isElement(node)) return null;
+  if (HEADINGS.has(tagOf(node))) return rankOf(node);
+  if (tagOf(node) !== 'hgroup') return null;
+
+  const ranks = [...walk(node)].filter((n) => HEADINGS.has(tagOf(n))).map(rankOf);
+  return ranks.length ? Math.min(...ranks) : null;
+}
+
+/** A heading and everything under it, up to the next heading of its rank or above. */
+function headingRun(heading) {
+  // An `<hgroup>` is the unit: the run starts at the group's own position among
+  // its siblings, not at the heading's position inside the group.
+  const start = tagOf(heading.parentNode) === 'hgroup' ? heading.parentNode : heading;
+  const rank = rankOf(heading);
+
+  const siblings = siblingsOf(start);
+  const from = siblings.indexOf(start);
+  const nodes = [start];
+
+  for (const node of siblings.slice(from + 1)) {
+    const stop = terminatingRank(node);
+    if (stop !== null && stop <= rank) break;
+    nodes.push(node);
+  }
+  return nodes;
+}
+
+/** A `<dt>` and its definitions, up to the next term. */
+function termRun(dt) {
+  const siblings = siblingsOf(dt);
+  const nodes = [dt];
+
+  for (const node of siblings.slice(siblings.indexOf(dt) + 1)) {
+    if (isElement(node) && tagOf(node) === 'dt') break;
+    nodes.push(node);
+  }
+  return nodes;
+}
+
+/**
+ * An empty inline element is a bookmark for what follows it, not the thing
+ * being addressed. `<a id="install"></a><h2>Installing</h2>` is everywhere in
+ * documents old enough to predate ids on headings.
+ *
+ * One hop only. Chaining would walk past real content whenever two bookmarks
+ * sit together.
+ */
+function throughBookmark(element) {
+  if (!INLINE.has(tagOf(element))) return element;
+  if (textOf(element).trim() !== '') return element;
+  if (hasEmbedded(element)) return element;
+
+  const siblings = siblingsOf(element);
+  const after = siblings.slice(siblings.indexOf(element) + 1).find(isElement);
+  return after ?? element;
+}
+
+function ancestorsOf(element) {
+  const chain = [];
+  for (let node = element.parentNode; node && isElement(node); node = node.parentNode) {
+    if (tagOf(node) === 'html' || tagOf(node) === 'body') break;
+    chain.unshift(tagOf(node));
+  }
+  return chain;
+}
+
+// ---- what a caller gets ----------------------------------------------------
+
+/**
+ * The fragment named by `id`, or null if the document does not offer one.
+ *
+ * `nodes` is a list, not one element. A heading run is several siblings, and
+ * wrapping them in a container would mean the fetched fragment did not match
+ * what the source document renders in that place.
+ *
+ * @param {string|object} input the HTML, or an already indexed document
+ * @param {string} id
+ * @returns {{ id: string, implicit: boolean, nodes: object[], html: string,
+ *   kind: string, standalone: boolean, diagnostics: object[] }|null} null when
+ *   nothing answers to that id
+ */
+export function resolveFragment(input, id) {
+  const doc = typeof input === 'string' ? readDocument(input) : input;
+  const diagnostics = [];
+
+  const explicit = doc.ids.get(id);
+  const found = explicit ?? doc.slugs.get(id);
+  if (!found) return null;
+
+  if (doc.duplicates.has(id)) {
+    diagnostics.push({
+      code: 'duplicate-id',
+      message: `more than one element has id "${id}"; the first in document order was taken`,
+    });
+  }
+
+  // A generated slug names its heading, which is already the thing to return.
+  const element = explicit ? throughBookmark(found) : found;
+  if (element !== found) {
+    diagnostics.push({
+      code: 'bookmark',
+      message: `"${id}" is an empty <${tagOf(found)}>, so <${tagOf(element)}> after it was taken`,
+    });
+  }
+
+  let nodes;
+  let kind;
+
+  if (HEADINGS.has(tagOf(element))) {
+    nodes = headingRun(element);
+    kind = 'heading-run';
+  } else if (tagOf(element) === 'dt') {
+    nodes = termRun(element);
+    kind = 'dt-run';
+  } else {
+    nodes = [element];
+    kind = 'element';
+  }
+
+  const outer = tagOf(nodes[0]);
+  const standalone = !CONTEXTUAL.has(outer);
+  const ancestors = standalone ? [] : ancestorsOf(nodes[0]);
+
+  if (!standalone) {
+    diagnostics.push({
+      code: 'not-standalone',
+      message:
+        `<${outer}> means nothing on its own; it needs ` +
+        `${ancestors.length ? ancestors.join(' > ') : 'a parent'} around it`,
+      ancestors,
+    });
+  }
+
+  return {
+    id,
+    implicit: !explicit,
+    nodes,
+    html: nodes.map((node) => serializeOuter(node)).join(''),
+    kind,
+    standalone,
+    diagnostics,
+  };
+}
+
+/**
+ * Every fragment the document offers, in document order.
+ *
+ * This is the claim that the rules work on markup nobody wrote for us: run it
+ * over a page and the result should read like that page's outline.
+ *
+ * @param {string|object} input
+ * @returns {Array<{ id: string, implicit: boolean, tag: string, rank: number, kind: string, text: string }>}
+ */
+export function listFragments(input) {
+  const doc = typeof input === 'string' ? readDocument(input) : input;
+
+  return doc.order.map(({ id, element, implicit }) => {
+    const tag = tagOf(element);
+    const rank = rankOf(element);
+    return {
+      id,
+      implicit,
+      tag,
+      rank,
+      kind: rank ? 'heading-run' : tag === 'dt' ? 'dt-run' : 'element',
+      text: textOf(element).replace(/\s+/g, ' ').trim().slice(0, 120),
+    };
+  });
+}

package/src/feed.js

@@ -0,0 +1,194 @@
+// GET /feed.xml, from a list the app supplies.
+//
+// A sitemap comes from the route table, because a URL is all it lists. A feed
+// needs a title, a date and something to read, and none of that is in the route
+// table, so the app passes the items in. Everything else here is the format.
+
+/** How many items ship. A reader wants the recent ones, not the archive. */
+const LIMIT = 50;
+
+const escape = (text) =>
+  String(text)
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;');
+
+/**
+ * HTML, kept as HTML.
+ *
+ * `]]>` is the one sequence a CDATA section cannot hold, and it can appear in
+ * ordinary markup: `<script>if (a[b[c]]>0)</script>`. Splitting it across two
+ * sections is what the parser puts back together as the original three
+ * characters.
+ */
+const cdata = (html) => `<![CDATA[${String(html).replace(/]]>/g, ']]]]><![CDATA[>')}]]>`;
+
+const date = (value) => {
+  if (!value) return null;
+  const parsed = value instanceof Date ? value : new Date(value);
+  return Number.isNaN(parsed.getTime()) ? null : parsed;
+};
+
+/** An absolute URL, since a feed is read somewhere else by definition. */
+const absolute = (hostname, path) => `${hostname.replace(/\/$/, '')}${path}`;
+
+/**
+ * The items, newest first.
+ *
+ * Sorting is stable, so items with no date keep the order they were given in
+ * rather than being shuffled among the dated ones.
+ */
+/**
+ * @param {{ items?: object[] | (() => object[] | Promise<object[]>), limit?: number }} config
+ *   `items` may be a function, so an app builds them from its own data
+ * @returns {Promise<object[]>}
+ */
+async function itemsOf({ items = [], limit = LIMIT }) {
+  const list = typeof items === 'function' ? await items() : await items;
+
+  const dated = list.map((item, index) => ({ item, index, at: date(item.date) }));
+  dated.sort((a, b) => {
+    if (a.at && b.at && a.at.getTime() !== b.at.getTime()) return b.at - a.at;
+    if (a.at && !b.at) return -1;
+    if (!a.at && b.at) return 1;
+    return a.index - b.index;
+  });
+
+  return dated.slice(0, limit).map(({ item, at }) => ({ ...item, at }));
+}
+
+/** The feed's own timestamp: the newest thing in it. */
+const newest = (items) => items.find((item) => item.at)?.at ?? null;
+
+function rss(items, config, stamp) {
+  const { hostname, title, description = '', path, language } = config;
+  const self = absolute(hostname, path);
+
+  const body = items.map((item) => {
+    const url = absolute(hostname, item.path);
+    const parts = [
+      `      <title>${escape(item.title)}</title>`,
+      `      <link>${escape(url)}</link>`,
+      `      <guid isPermaLink="true">${escape(item.id ?? url)}</guid>`,
+    ];
+    if (item.at) parts.push(`      <pubDate>${item.at.toUTCString()}</pubDate>`);
+    if (item.description) parts.push(`      <description>${cdata(item.description)}</description>`);
+    if (item.content) {
+      parts.push(`      <content:encoded>${cdata(item.content)}</content:encoded>`);
+    }
+    return `    <item>\n${parts.join('\n')}\n    </item>`;
+  });
+
+  const head = [
+    `    <title>${escape(title)}</title>`,
+    `    <link>${escape(absolute(hostname, '/'))}</link>`,
+    `    <description>${escape(description)}</description>`,
+    `    <atom:link href="${escape(self)}" rel="self" type="application/rss+xml"/>`,
+  ];
+  if (language) head.push(`    <language>${escape(language)}</language>`);
+  if (stamp) head.push(`    <lastBuildDate>${stamp.toUTCString()}</lastBuildDate>`);
+
+  return (
+    `<?xml version="1.0" encoding="utf-8"?>\n` +
+    `<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" ` +
+    `xmlns:content="http://purl.org/rss/1.0/modules/content/">\n` +
+    `  <channel>\n${head.join('\n')}\n${body.join('\n')}\n  </channel>\n</rss>\n`
+  );
+}
+
+function atom(items, config, stamp) {
+  const { hostname, title, description = '', path, author } = config;
+  const self = absolute(hostname, path);
+
+  const person = (who) =>
+    `<author><name>${escape(who.name)}</name>` +
+    (who.email ? `<email>${escape(who.email)}</email>` : '') +
+    `</author>`;
+
+  const body = items.map((item) => {
+    const url = absolute(hostname, item.path);
+    const parts = [
+      `    <title>${escape(item.title)}</title>`,
+      `    <link href="${escape(url)}"/>`,
+      `    <id>${escape(item.id ?? url)}</id>`,
+      `    <updated>${(item.at ?? stamp).toISOString()}</updated>`,
+    ];
+    if (item.author) parts.push(`    ${person(item.author)}`);
+    if (item.description) parts.push(`    <summary>${escape(item.description)}</summary>`);
+    if (item.content) {
+      parts.push(`    <content type="html">${cdata(item.content)}</content>`);
+    }
+    return `  <entry>\n${parts.join('\n')}\n  </entry>`;
+  });
+
+  const head = [
+    `  <title>${escape(title)}</title>`,
+    `  <link href="${escape(absolute(hostname, '/'))}"/>`,
+    `  <link rel="self" href="${escape(self)}"/>`,
+    `  <id>${escape(absolute(hostname, '/'))}</id>`,
+    `  <updated>${stamp.toISOString()}</updated>`,
+  ];
+  if (description) head.push(`  <subtitle>${escape(description)}</subtitle>`);
+  if (author) head.push(`  ${person(author)}`);
+
+  return (
+    `<?xml version="1.0" encoding="utf-8"?>\n` +
+    `<feed xmlns="http://www.w3.org/2005/Atom">\n${head.join('\n')}\n${body.join('\n')}\n</feed>\n`
+  );
+}
+
+/**
+ * What the response is served as. A reader picks the parser from this.
+ *
+ * @param {object|null|undefined} config
+ * @returns {string}
+ */
+export const feedType = (config) =>
+  config?.format === 'atom'
+    ? 'application/atom+xml; charset=utf-8'
+    : 'application/rss+xml; charset=utf-8';
+
+/**
+ * Where it is mounted, and where the build writes it.
+ *
+ * @param {object|null|undefined} config
+ * @returns {string}
+ */
+export const feedPath = (config) => config?.path ?? '/feed.xml';
+
+/**
+ * @param {object} [config] the `feed` block, plus its `items`
+ * @returns {Promise<string>} an RSS or Atom document
+ * @throws when Atom is asked for without an author or a date
+ */
+export async function feed(config = {}) {
+  const { hostname, title, format = 'rss', author, updated } = config;
+
+  if (!hostname) throw new Error('[transclude] feed needs a hostname, since every link is absolute');
+  if (!title) throw new Error('[transclude] feed needs a title');
+
+  const items = await itemsOf(config);
+
+  // Nothing here may read the clock. A prerendered feed is written once and
+  // compressed once, and a timestamp from the build would change the bytes on
+  // every run for a file whose contents did not change.
+  const stamp = date(updated) ?? newest(items);
+
+  if (format === 'atom') {
+    if (!author && !items.every((item) => item.author)) {
+      throw new Error(
+        '[transclude] an Atom feed needs an author: give the feed one, or every item its own',
+      );
+    }
+    if (!stamp) {
+      throw new Error(
+        '[transclude] an Atom feed needs a date: give an item a `date`, or the feed an `updated`',
+      );
+    }
+    return atom(items, { ...config, path: feedPath(config) }, stamp);
+  }
+
+  return rss(items, { ...config, path: feedPath(config) }, stamp);
+}

package/src/include.js

@@ -0,0 +1,89 @@
+// What a `<transclude>` is resolved against, built once.
+//
+// Three servers render pages: the production app, the dev server and the build.
+// Each used to be handed a piece of this and the dev server was handed less than
+// the others, so an include that worked in production threw in dev with nothing
+// to suggest why. One function answers it for all three.
+
+import { paramsFor, renderFragment } from './document.js';
+import { includeResolver } from './proxy.js';
+
+/**
+ * `{ resolve, route }` for `renderRoute`.
+ *
+ * `resolve` reads another site and needs `proxy.allow`; without it an external
+ * include says so. `route` reads another route of this app and needs nothing,
+ * because nothing leaves the server.
+ *
+ * @param routes  the route table, as `{ id, pattern }`
+ * @param pageFor a route id to its compiled module, possibly a promise
+ */
+export function includeContext({ config, routes = [], pageFor, lookup = null }) {
+  const foreign = config.proxy
+    ? includeResolver(config.proxy, { lookup: config.proxy.lookup ?? lookup ?? null })
+    : null;
+
+  const context = {
+    resolve: foreign?.resolve ?? null,
+
+    route: async (path, id, ctx, options) => {
+      const memo = options?.includeMemo;
+      const key = `${path}#${id}`;
+      if (memo?.has(key)) return memo.get(key);
+
+      const answer = render(path, id, ctx, options);
+      // The promise is held, not the result: two includes of one route that
+      // start together should still be one render.
+      memo?.set(key, answer);
+      return answer;
+    },
+  };
+
+  /**
+   * Rendered rather than fetched. It is the same process, so a request to
+   * ourselves would run the whole middleware stack to answer something we can
+   * answer directly, and would need a URL the server may not know behind a
+   * proxy.
+   *
+   * The host's `ctx` goes through, cookies and all. That is what makes reading
+   * contagious: a page including a route whose loader reads a cookie is personal
+   * too, and the cache has to know.
+   */
+  async function render(path, id, ctx, options) {
+    const url = new URL(path, ctx.url);
+
+    let found = null;
+    for (const route of routes) {
+      const params = paramsFor(route, url.pathname);
+      if (params) {
+        found = { route, params };
+        break;
+      }
+    }
+    if (!found) throw new Error(`[transclude] no route answers ${path}`);
+
+    const page = await pageFor(found.route.id);
+    if (!page) throw new Error(`[transclude] ${path} is not a page`);
+
+    const html = await renderFragment(
+      page,
+      {
+        ...ctx,
+        url: url.href,
+        params: found.params,
+        route: { id: found.route.id, pattern: found.route.pattern, path: url.pathname },
+        // The include is not the request. A loader that branches on the region
+        // asked for should see this render for what it is.
+        fragment: null,
+        action: null,
+      },
+      { region: id, ...options, include: context },
+    );
+
+    // A loader answering for itself has nothing to say here: the including page
+    // asked for markup, not for a redirect.
+    return html instanceof Response ? null : html;
+  }
+
+  return context;
+}

package/src/lookup.js

@@ -0,0 +1,49 @@
+// Resolving a hostname, which is the one check the core cannot make.
+//
+// `src/address.js` decides what an address means and imports nothing. This
+// turns a name into addresses, which needs the runtime: Node and Deno have a
+// resolver, and workerd has none at all.
+//
+// Not reachable from `src/app.js`. It is passed in by the servers that can
+// supply it, and left out by the one that cannot.
+
+import { promises as dns } from 'node:dns';
+
+import { blockedAddress } from './address.js';
+
+/**
+ * A `lookup` for the proxy: the reason a host may not be fetched, or null.
+ *
+ * Every address a name resolves to is checked, not just the first. A name that
+ * answers with one public address and one private one is a way in, and which of
+ * the two a connection uses is not ours to decide.
+ *
+ * This still leaves a gap that no amount of checking here closes: the name is
+ * resolved once for the check and again by the connection, and a record whose
+ * TTL expires in between can change. The allowlist is what actually holds, and
+ * this is defense behind it.
+ *
+ * @param {{ resolver?: object }} [deps] injected so a test needs no DNS
+ * @returns {(hostname: string) => Promise<string[]>} every address the name answers with
+ */
+export function nodeLookup({ resolver = dns } = {}) {
+  return async (hostname) => {
+    // A literal is already decided by `checkUrl`, which runs before this.
+    if (blockedAddress(hostname)) return blockedAddress(hostname);
+
+    let addresses;
+    try {
+      addresses = await resolver.lookup(hostname, { all: true });
+    } catch {
+      // A name that does not resolve is not this check's refusal to make. The
+      // fetch will fail on its own and say so in its own words.
+      return null;
+    }
+
+    for (const { address } of addresses) {
+      const why = blockedAddress(address);
+      if (why) return `${address}, which is ${why}`;
+    }
+    return null;
+  };
+}