@transclude/core 0.1.0

package/src/production.js

@@ -0,0 +1,159 @@
+// The Node wiring for the production app.
+//
+// `app.js` is the app, and it names no runtime. This file is the four things that
+// do: bytes off a disk, a hash from `node:crypto`, compression from `node:zlib`,
+// and Hono's Node `serveStatic` for the public directory. That last one is kept
+// separate because it does byte ranges, which an in-memory map cannot.
+//
+// A runtime with no filesystem writes its own version of this file. It is about
+// thirty lines, and `app.js` does not change.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { publicFiles } from './public-files.js';
+import { createApp } from './app.js';
+import { etagOf, loadAssets, loadStatic } from './static-cache.js';
+import { compressResponse } from './compress.js';
+import { loadProject, portOf } from './project.js';
+import { nodeLookup } from './lookup.js';
+
+const { root, config, configFile } = await loadProject();
+const dist = path.join(root, config.outDir);
+
+export const port = portOf(config, process.env.PORT);
+
+export const noBuild = !fs.existsSync(path.join(dist, 'routes.json'));
+
+/**
+ * This server reads `dist`, never the source. An edit made since the last build
+ * is not visible here, which looks the same as the edit not working, so say so
+ * rather than leave it to be found.
+ */
+function newestSource() {
+  // The app, and the framework wherever it is installed. `fileURLToPath`, not
+  // `url.pathname`: a space in the path stays percent-encoded in the latter, and
+  // `Atelier%20Dakroub` is not a directory.
+  const here = path.dirname(fileURLToPath(import.meta.url));
+  const roots = [path.join(root, config.appDir), here];
+  let newest = { time: 0, file: null };
+
+  const walk = (dir) => {
+    if (!fs.existsSync(dir)) return;
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) walk(full);
+      else {
+        const { mtimeMs } = fs.statSync(full);
+        if (mtimeMs > newest.time) newest = { time: mtimeMs, file: full };
+      }
+    }
+  };
+
+  for (const dir of roots) walk(dir);
+  if (fs.existsSync(configFile)) {
+    const { mtimeMs } = fs.statSync(configFile);
+    if (mtimeMs > newest.time) newest = { time: mtimeMs, file: configFile };
+  }
+  return newest;
+}
+
+const builtAt = noBuild ? 0 : fs.statSync(path.join(dist, 'routes.json')).mtimeMs;
+const newest = newestSource();
+const stale = !noBuild && newest.time > builtAt;
+
+const manifest = noBuild
+  ? { routes: [], dynamic: [], endpoints: [] }
+  : JSON.parse(fs.readFileSync(path.join(dist, 'routes.json'), 'utf8'));
+
+const bundle = noBuild
+  ? { pages: {}, endpoints: {}, middleware: null }
+  : await import(pathToFileURL(path.join(dist, 'server/entry.js')).href);
+
+const assets = loadAssets(path.join(dist, 'client'));
+const statics = loadStatic(path.join(dist, 'static'));
+
+/** A build artifact under `static/`, as text. Absent until a build writes it. */
+const readText = (name) => {
+  const file = path.join(dist, 'static', name);
+  return fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : null;
+};
+
+/** A page the framework reaches for rather than routes to. */
+const readPage = (name) => {
+  const file = path.join(dist, 'static', name);
+  return fs.existsSync(file) ? readEntry(file) : null;
+};
+
+const publicRoot = path.join(dist, 'public');
+
+export const app = createApp({
+  config,
+  lookup: config.proxy ? nodeLookup() : null,
+  manifest,
+  pages: bundle.pages,
+  endpoints: bundle.endpoints ?? {},
+  middleware: bundle.middleware ?? null,
+  statics,
+  assets,
+  // `serveStatic` joins `root` onto the request path, so it resolves against the
+  // working directory. This file works from its own location so it does not depend
+  // on where the process was started.
+  publicFiles: fs.existsSync(publicRoot) ? publicFiles(relativeToCwd(publicRoot)) : null,
+  notFound: readPage('404.html'),
+  errorPage: readPage('500.html'),
+  hash: etagOf,
+  compress: compressResponse,
+  precache: readText('precache.json'),
+});
+
+/**
+ * What a freshly started server prints. Kept here rather than in an adapter so
+ * three of them do not each grow their own copy of it.
+ *
+ * @param {number} port
+ * @returns {void}
+ */
+export function summary(port) {
+  const kb = (n) => `${Math.round(n / 1024)} KB`;
+
+  console.log(`http://localhost:${port}`);
+  console.log(
+    `  prerendered  ${statics.count} pages, ${kb(statics.bytes)}` +
+      (statics.onDisk ? ` (${statics.onDisk} over budget, read per request)` : ''),
+  );
+  console.log(`  assets       ${assets.count} files, ${kb(assets.bytes)}`);
+  console.log(
+    `  precompressed ${statics.encoded + assets.encoded}/${statics.count + assets.count} resources`,
+  );
+  console.log(`  on demand    ${manifest.dynamic.map((r) => r.pattern).join(', ') || 'none'}`);
+
+  if (stale) {
+    const ago = Math.round((newest.time - builtAt) / 1000);
+    console.log('');
+    const where = path.relative(root, newest.file);
+    console.log(`  ⚠  ${where} changed ${ago}s after the last build.`);
+    console.log('     This server reads dist/, so that edit is not being served.');
+    console.log('     Run `npm run build` (or `npm run preview` to do both).');
+  }
+}
+
+function relativeToCwd(absolute) {
+  const relative = path.relative(process.cwd(), absolute);
+  return relative === '' ? '.' : relative;
+}
+
+function readEntry(file) {
+  const body = fs.readFileSync(file);
+  const encodings = new Map();
+  const etag = etagOf(body);
+
+  for (const [encoding, suffix] of [['br', '.br'], ['gzip', '.gz']]) {
+    if (!fs.existsSync(`${file}${suffix}`)) continue;
+    encodings.set(encoding, {
+      body: fs.readFileSync(`${file}${suffix}`),
+      etag: `${etag.slice(0, -1)}-${encoding}"`,
+    });
+  }
+  return { body, etag, encodings, type: 'text/html; charset=utf-8' };
+}

package/src/project.js

@@ -0,0 +1,110 @@
+// Where the app is, and what it configured.
+//
+// The framework used to reach two directories up from its own file for both, and
+// import `transclude.config.js` by relative path. That is only true while it sits
+// inside the app it serves. Installed as a package it sits in node_modules, where
+// two directories up is somebody else's package.
+//
+// So the root comes from where the command was run, and the config is loaded from
+// there at run time. Nothing under `framework/` names a path in the app again.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+export const CONFIG_FILE = 'transclude.config.js';
+
+/**
+ * The port an app listens on when it does not name one.
+ *
+ * 3000 and 5173 are the two most crowded ports on a developer's machine, and a
+ * server someone else started answering yours is a bad half hour. 1960 is the
+ * year Project Xanadu began, which is where transclusion comes from.
+ */
+export const DEFAULT_PORT = 1960;
+
+/**
+ * `PORT` beats the config, so a host that assigns one is obeyed without an edit.
+ * Dev and production share this, so an app has one port rather than two.
+ *
+ * @param {object} [config]
+ * @param {string|undefined} [env] `PORT`, which wins
+ * @returns {number}
+ * @throws when the value is set but not a port
+ */
+export function portOf(config = {}, env = undefined) {
+  const asked = env ?? config.port ?? DEFAULT_PORT;
+  const port = Number(asked);
+
+  if (!Number.isInteger(port) || port < 1 || port > 65535) {
+    throw new Error(
+      `[transclude] port must be a whole number from 1 to 65535, not ${JSON.stringify(asked)}`,
+    );
+  }
+  return port;
+}
+
+/**
+ * The nearest directory at or above `from` holding the config file.
+ *
+ * npm runs a script with the package root as the working directory, so the
+ * search almost always ends at the first try. It walks up so that running a bin
+ * by hand from a subdirectory works too.
+ *
+ * @param {string} [from]
+ * @returns {string} the directory holding transclude.config.js
+ */
+export function findRoot(from = process.cwd()) {
+  let dir = path.resolve(from);
+
+  for (;;) {
+    if (fs.existsSync(path.join(dir, CONFIG_FILE))) return dir;
+    const up = path.dirname(dir);
+    if (up === dir) break;
+    dir = up;
+  }
+
+  throw new Error(
+    `[transclude] no ${CONFIG_FILE} in ${path.resolve(from)} or any directory above it`,
+  );
+}
+
+/**
+ * The root and its config together, because nothing needs one without the other.
+ *
+ * Imported by URL rather than by path: a space in the project path stays
+ * percent-encoded in a bare file path and `Atelier%20Dakroub` is not a directory.
+ *
+ * @param {string} [from]
+ * @returns {Promise<object>} the root, the config and every resolved directory
+ */
+export async function loadProject(from = process.cwd()) {
+  const root = findRoot(from);
+  const file = path.join(root, CONFIG_FILE);
+  const { default: config } = await import(pathToFileURL(file).href);
+
+  if (!config || typeof config !== 'object') {
+    throw new Error(`[transclude] ${CONFIG_FILE} must export a config object as its default`);
+  }
+  assertNoSplitDirs(config, file);
+  return { root, config, configFile: file };
+}
+
+/**
+ * `partialsDir` and `componentsDir` are one `elementsDir` now, and how an
+ * element renders is read from the file rather than from where it sits.
+ *
+ * Left alone, the old keys would be ignored and the app would look in
+ * `app/elements/`, find nothing, and every tag would render as an unknown
+ * element with no styles and nothing said.
+ */
+function assertNoSplitDirs(config, file) {
+  const old = ['partialsDir', 'componentsDir'].filter((key) => key in config);
+  if (!old.length) return;
+
+  throw new Error(
+    `[transclude] ${file} sets ${old.join(' and ')}, which nothing reads. ` +
+      `Put every element in one directory and name it with \`elementsDir\`. ` +
+      `A shadow root is opt-in per file now: \`export const shadow = true\`.`,
+  );
+}

package/src/proxy.js

@@ -0,0 +1,319 @@
+// Fetching a foreign document, and answering with one fragment of it.
+//
+// The fragment is resolved here rather than in the browser. That keeps the
+// untrusted markup on this side, keeps sanitizing somewhere it cannot be turned
+// off, and means the page receives the piece it asked for instead of a whole
+// document to sift through.
+//
+// Nothing here imports `node:`. Resolving a hostname to an address is the one
+// check that needs the runtime, so it arrives as a function. See `lookup` below.
+
+import { checkUrl } from './address.js';
+import { indexDocument, resolveFragment } from './extract.js';
+import { absolutize, baseOf, sanitize } from './rewrite.js';
+import { parse } from 'parse5';
+
+export const PROXY_PATH = '/_transclude/proxy';
+
+const DEFAULTS = {
+  allow: [],
+  maxBytes: 5 * 1024 * 1024,
+  timeout: 10_000,
+  redirects: 5,
+  sanitize: true,
+  // What to do with a `style` attribute the source wrote. `<style>` blocks and
+  // `<link>` are removed either way, because their rules reach the whole page.
+  styles: 'keep',
+  cache: 50,
+  // How long a held document is used without asking the source anything. Ten
+  // fragments off one page during a render should be one request, not ten
+  // conditional ones.
+  maxAge: 60_000,
+};
+
+const STYLE_MODES = new Set(['keep', 'strip']);
+
+/**
+ * Defaults filled in, and the one value worth checking checked. A misspelled
+ * `styles` would keep every style attribute and say nothing, which reads exactly
+ * like the setting working.
+ */
+function settings(options) {
+  const config = { ...DEFAULTS, ...options };
+  if (!STYLE_MODES.has(config.styles)) {
+    throw new Error(
+      `[transclude] proxy.styles is ${JSON.stringify(config.styles)}. It is 'keep' or 'strip'.`,
+    );
+  }
+  return config;
+}
+
+/** A refusal that is safe to show, since it repeats only what was asked for. */
+class Refused extends Error {
+  constructor(status, message) {
+    super(message);
+    this.status = status;
+  }
+}
+
+const HTML = /^(text\/html|application\/xhtml\+xml)/i;
+
+/**
+ * Follow redirects ourselves so every hop is checked.
+ *
+ * `redirect: 'follow'` would let the first response send us anywhere, and the
+ * check that ran on the URL somebody typed says nothing about where hop three
+ * landed. This is the whole reason the fetch is written out by hand.
+ */
+async function fetchChecked(url, config, deps) {
+  const { fetch: get = globalThis.fetch, lookup = null } = deps;
+  let at = url;
+
+  for (let hop = 0; hop <= config.redirects; hop += 1) {
+    const why = checkUrl(at, config);
+    if (why) throw new Refused(403, why);
+
+    if (lookup) {
+      const blocked = await lookup(new URL(at).hostname);
+      if (blocked) throw new Refused(403, `${new URL(at).hostname} resolves to ${blocked}`);
+    }
+
+    const response = await get(at, {
+      redirect: 'manual',
+      signal: AbortSignal.timeout(config.timeout),
+      headers: { accept: 'text/html' },
+    });
+
+    if (response.status >= 300 && response.status < 400 && response.headers.get('location')) {
+      at = new URL(response.headers.get('location'), at).href;
+      continue;
+    }
+    return { response, url: at };
+  }
+
+  throw new Refused(502, `more than ${config.redirects} redirects`);
+}
+
+/**
+ * The body, refused past the cap.
+ *
+ * `content-length` is a claim, not a promise, so it is used to refuse early and
+ * the bytes are counted anyway.
+ */
+async function bodyWithin(response, maxBytes) {
+  const claimed = Number(response.headers.get('content-length'));
+  if (claimed && claimed > maxBytes) throw new Refused(413, `document is larger than ${maxBytes} bytes`);
+
+  if (!response.body) return await response.text();
+
+  const reader = response.body.getReader();
+  const chunks = [];
+  let size = 0;
+
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    size += value.byteLength;
+    if (size > maxBytes) {
+      await reader.cancel();
+      throw new Refused(413, `document is larger than ${maxBytes} bytes`);
+    }
+    chunks.push(value);
+  }
+
+  const joined = new Uint8Array(size);
+  let at = 0;
+  for (const chunk of chunks) {
+    joined.set(chunk, at);
+    at += chunk.byteLength;
+  }
+  return new TextDecoder().decode(joined);
+}
+
+/**
+ * Parsed documents, by the URL they finally came from.
+ *
+ * Several fragments usually come from one page, so the parse, the cleaning and
+ * the slug table are done once and shared. Keyed by the final URL rather than
+ * the requested one, so two requests that redirect to the same place hit.
+ *
+ * @param {number} [max] how many documents to hold
+ * @returns {{ get: Function, set: Function, size: () => number }}
+ */
+export function documentStore(max = DEFAULTS.cache) {
+  const held = new Map();
+
+  return {
+    get(key) {
+      const entry = held.get(key);
+      if (!entry) return null;
+      // Least recently used goes first, so re-insert on a hit.
+      held.delete(key);
+      held.set(key, entry);
+      return entry;
+    },
+    set(key, entry) {
+      held.delete(key);
+      held.set(key, entry);
+      while (held.size > max) held.delete(held.keys().next().value);
+    },
+    get size() {
+      return held.size;
+    },
+  };
+}
+
+/**
+ * A foreign document, fetched, cleaned and indexed.
+ *
+ * The order is deliberate: read the base before `<base>` is stripped, strip
+ * before rewriting so nothing rewrites a URL on an element about to be removed,
+ * and index last so the table never names something the cleaning took out.
+ *
+ * @param {string} url
+ * @param {object} [options] the `proxy` config
+ * @param {{ fetch?: Function, store?: object, now?: Function, lookup?: Function }} [deps]
+ * @returns {Promise<object>} the indexed document, its base, and what was removed
+ */
+export async function readForeign(url, options = {}, deps = {}) {
+  const config = settings(options);
+  const store = deps.store ?? null;
+  const now = deps.now ?? (() => Date.now());
+
+  const held = store?.get(url) ?? null;
+  if (held && now() - held.at < config.maxAge) return held;
+
+  const revalidate = held
+    ? { 'if-none-match': held.etag ?? '', 'if-modified-since': held.lastModified ?? '' }
+    : {};
+
+  const get = deps.fetch ?? globalThis.fetch;
+  const wrapped = held
+    ? (at, init) =>
+        get(at, {
+          ...init,
+          headers: { ...init.headers, ...Object.fromEntries(Object.entries(revalidate).filter(([, v]) => v)) },
+        })
+    : get;
+
+  const { response, url: final } = await fetchChecked(url, config, { ...deps, fetch: wrapped });
+
+  if (response.status === 304 && held) {
+    // Still current, so the parse stands and only its age moves.
+    const refreshed = { ...held, at: now() };
+    store?.set(url, refreshed);
+    return refreshed;
+  }
+  if (!response.ok) throw new Refused(502, `the source answered ${response.status}`);
+
+  const type = response.headers.get('content-type') ?? '';
+  if (!HTML.test(type)) throw new Refused(415, `the source sent ${type || 'no content type'}`);
+
+  const html = await bodyWithin(response, config.maxBytes);
+  const base = baseOf(html, final);
+
+  const root = parse(html);
+  const removed = config.sanitize ? sanitize(root, { styles: config.styles }) : [];
+  absolutize(root, base);
+
+  const entry = {
+    doc: indexDocument(root),
+    base,
+    removed,
+    at: now(),
+    etag: response.headers.get('etag'),
+    lastModified: response.headers.get('last-modified'),
+  };
+
+  store?.set(final, entry);
+  // The requested URL is what the next request will arrive with, so it is held
+  // too when a redirect moved us.
+  if (final !== url) store?.set(url, entry);
+  return entry;
+}
+
+/**
+ * `GET /_transclude/proxy?url=…&id=…`
+ *
+ * Answers with the fragment, as markup. A page fetches this from its own origin,
+ * which is also what makes it work at all: a cross-origin fetch is refused by
+ * the default policy, which names `'self'` and nothing else.
+ */
+/**
+ * A resolver for `renderRoute`: the markup of one fragment of one URL.
+ *
+ * Shares a store with nothing else on purpose. Includes are resolved during a
+ * render, and holding the parsed document is what makes ten of them off one page
+ * cost one read.
+ *
+ * @param {object} [options]
+ * @param {object} [deps]
+ * @returns {{ resolve: (url: string, id: string) => Promise<string> }}
+ */
+export function includeResolver(options = {}, deps = {}) {
+  const config = settings(options);
+  const store = deps.store ?? documentStore(config.cache);
+
+  return {
+    resolve: async (url, id) => {
+      const entry = await readForeign(url, config, { ...deps, store });
+      return resolveFragment(entry.doc, id)?.html ?? null;
+    },
+  };
+}
+
+/**
+ * @param {object} [options]
+ * @param {object} [deps]
+ * @returns {(request: Request) => Promise<Response>}
+ */
+export function proxyHandler(options = {}, deps = {}) {
+  const config = settings(options);
+  const store = deps.store ?? documentStore(config.cache);
+
+  return async (request) => {
+    const asked = new URL(request.url);
+    const url = asked.searchParams.get('url');
+    const id = asked.searchParams.get('id');
+
+    if (!url) return text(400, 'no url');
+
+    try {
+      const entry = await readForeign(url, config, { ...deps, store });
+
+      // No id is a question about the document rather than a piece of it.
+      if (!id) {
+        const { listFragments } = await import('./extract.js');
+        return json(200, { url, fragments: listFragments(entry.doc) });
+      }
+
+      const found = resolveFragment(entry.doc, id);
+      if (!found) return text(404, `no fragment "${id}" at ${url}`);
+
+      return new Response(found.html, {
+        status: 200,
+        headers: {
+          'content-type': 'text/html; charset=utf-8',
+          // The answer depends entirely on the query, and the source may change
+          // under us, so nothing downstream should hold it on its own terms.
+          'cache-control': 'no-store',
+        },
+      });
+    } catch (error) {
+      if (error instanceof Refused) return text(error.status, error.message);
+      if (error?.name === 'TimeoutError' || error?.name === 'AbortError') {
+        return text(504, 'the source did not answer in time');
+      }
+      return text(502, 'the source could not be read');
+    }
+  };
+}
+
+const text = (status, body) =>
+  new Response(body, { status, headers: { 'content-type': 'text/plain; charset=utf-8' } });
+
+const json = (status, body) =>
+  new Response(JSON.stringify(body), {
+    status,
+    headers: { 'content-type': 'application/json; charset=utf-8' },
+  });

package/src/public-files.js

@@ -0,0 +1,77 @@
+// The author's own files, served from disk.
+//
+// Node only, and deliberately outside the portable core: `app.js` is handed a
+// handler and never builds one, which is what lets a runtime with no filesystem
+// supply its own. Nothing here is reachable from that graph.
+//
+// These are not build output. They can be large, they can be media, and media
+// needs byte ranges, which is why they go through Hono's `serveStatic` rather
+// than the in-memory cache. What that leaves out is a validator: the response
+// carried `Last-Modified` and nothing else, so a browser fell back to guessing
+// how long to hold a favicon.
+
+import fs from 'node:fs';
+import { serveStatic } from '@hono/node-server/serve-static';
+
+/** Same as the build output's. These change when the author changes them. */
+const REVALIDATE = 'public, max-age=0, must-revalidate';
+
+/**
+ * Size and modified time, not a hash of the bytes.
+ *
+ * Hashing is what the in-memory cache does, and it is the wrong trade here: a
+ * video would be read in full to answer a request that may only want the first
+ * megabyte of it. Weak on purpose, because two files can share a size and a
+ * second: it is enough to answer "has this changed", which is all a conditional
+ * request asks, and it does not claim the byte-for-byte identity a strong one
+ * does.
+ *
+ * @param {string} file
+ * @returns {string}
+ */
+function validatorFor(file) {
+  const { size, mtimeMs } = fs.statSync(file);
+  return `W/"${size.toString(36)}-${Math.floor(mtimeMs).toString(36)}"`;
+}
+
+/**
+ * A handler for `publicFiles`, with a validator and a 304.
+ *
+ * `serveStatic` sets the headers and reads no request condition, so the
+ * conditional half is done around it. The file it found is the one measured,
+ * which is the point when `precompressed` served a `.br`: different bytes are
+ * a different entity and must not share an ETag.
+ *
+ * @param {string} root relative to the working directory
+ * @returns {Function} Hono middleware
+ */
+export function publicFiles(root) {
+  const inner = serveStatic({
+    root,
+    precompressed: true,
+    onFound: (file, c) => {
+      c.header('ETag', validatorFor(file));
+      c.header('Cache-Control', REVALIDATE);
+    },
+  });
+
+  return async (c, next) => {
+    // Whatever it gives back is given back. A range is answered by *returning* a
+    // Response rather than by setting `c.res`, so swallowing this leaves the
+    // context unfinalized and every Range request becomes a 500. That is a
+    // difference no unit test here saw: it took a real server and a real
+    // `Range` header.
+    const answer = await inner(c, next);
+    const found = answer ?? c.res;
+
+    // 206 is left alone. A range was asked for and answered, and a weak
+    // validator is not one `If-Range` may be matched against.
+    if (!found || found.status !== 200) return answer;
+
+    const etag = found.headers.get('etag');
+    if (etag && c.req.header('if-none-match') === etag) {
+      return new Response(null, { status: 304, headers: found.headers });
+    }
+    return answer;
+  };
+}