html-minifier-setup 0.0.0-stage → 0.1.0

package/src/hazards.js

@@ -0,0 +1,329 @@
+// What a page’s own scripts and style sheets rely on that an option may change—found by comparing
+// output with and without the option, and matching what changed against what the page references
+
+import { parse } from 'parse5';
+
+/**
+ * `risk`: likely breaks the page; `check`: may break it, which a render check can settle;
+ * `note`: can’t be ruled out
+ * @typedef {{level: 'risk' | 'check' | 'note', message: string}} Hazard
+ */
+
+/**
+ * @typedef {object} PageContext
+ * @prop {Set<string>} scriptStrings Words in string literals of scripts and event handlers
+ * @prop {string} scriptText
+ * @prop {string[]} scriptsUnread Script URLs that couldn’t be read
+ * @prop {Set<string>} attributeRefs Attribute names that style sheets select on or scripts query
+ * @prop {{selector: string, text: string}[]} whitespaceRules Rules that keep whitespace, outside `pre` and `textarea`
+ */
+
+/** @typedef {import('parse5').DefaultTreeAdapterMap['node']} Node */
+/** @typedef {import('parse5').DefaultTreeAdapterMap['element']} Element */
+
+const MAX_PER_KIND = 3;
+
+/** @type {Record<string, string>} */
+const MODES = { 'quirks': 'quirks mode', 'limited-quirks': 'almost standards mode', 'no-quirks': 'standards mode' };
+const RE_STRING_LITERAL = /(["'`])((?:\\.|(?!\1)[^\\\n])*)\1/g;
+const RE_WORD = /[\w-]+/g;
+const RE_ATTRIBUTE_SELECTOR = /\[\s*([\w-]+)\s*(?=[~|^$*]?=|\])/g;
+const RE_CSS_COMMENT = /\/\*[\s\S]*?\*\//g;
+// At-rules without a block, like `@import`, taking quoted strings and escapes whole
+const RE_CSS_STATEMENT = /@[\w-]+(?:\\.|"(?:\\.|[^"\\\n])*"|'(?:\\.|[^'\\\n])*'|[^{};"'\\])*;/g;
+const RE_IMPORT = /@import\s+(?:url\(\s*)?(["']?)([^"')\s;]+)\1/gi;
+const RE_ATTRIBUTE_CALL = /\b(?:get|has|remove|set|toggle)Attribute(?:NS)?\(\s*(?:[^,()]*,\s*)?["'`]([\w-]+)/g;
+const RE_WHITESPACE_KEPT = /white-space(?:-collapse)?\s*:\s*(?:pre-wrap|pre-line|pre|break-spaces|preserve-breaks|preserve-spaces|preserve)(?![\w-])/i;
+const RE_READS_COMMENTS = /SHOW_COMMENT|COMMENT_NODE|nodeType\s*[!=]==?\s*8\b|\.childNodes\b[^;]*\.data\b/;
+const RE_SCRIPT_JS = /^(?:|module|(?:application|text)\/(?:x-)?(?:java|ecma)script|text\/javascript1\.[0-5]|text\/jscript|text\/livescript)$/i;
+const SELECTORS_PREFORMATTED = new Set(['pre', 'textarea']);
+
+// Conditional comments only matter to Internet Explorer 9 and older
+const RE_COMMENT_CONDITIONAL = /^\[if\b|<!\[endif\]$/i;
+/** @type {[RegExp, string][]} */
+const COMMENTS_MEANINGFUL = [
+  [/^\s*\/?ko(?:\s|$)/, 'Knockout binding'],
+  [/^\s*google(?:off|on)\s*:/i, 'search appliance directive'],
+  [/^\s*\/?esi\b|^\s*<\/?esi:/i, 'Edge Side Includes'],
+  [/^\?lit\$|^\s*lit-part\b/, 'Lit marker'],
+  [/^\s*\/?\$[!?]?\s*$|^\s*[[\]]\s*$|^\s*(?:teleport|v-if)\b/, 'hydration marker'],
+  [/^\s*\/?noindex\s*$/i, 'indexing directive'],
+  [/^\s*\/?wp:/, 'WordPress block delimiter'],
+  [/^\s*(?:Instance|Template)(?:Begin|End)\b|^\s*#(?:Begin|End)(?:Library|Template)/, 'template editor region']
+];
+
+export const OPTIONS_ATTRIBUTES = new Set(['collapseAttributeWhitespace', 'collapseBooleanAttributes', 'removeDefaultTypeAttributes', 'removeEmptyAttributes', 'removeRedundantAttributes', 'sortClassNames']);
+export const OPTIONS_WHITESPACE = new Set(['collapseInlineTagWhitespace', 'collapseNoBreakSpaces', 'collapseWhitespace', 'trimCustomFragments']);
+
+/** @param {string} text */
+const clip = (text, length = 60) => text.length > length ? `${text.slice(0, length - 1)}…` : text;
+
+/**
+ * @param {Node} node
+ * @param {(node: Node) => void} visit
+ */
+function walk(node, visit) {
+  visit(node);
+  const content = /** @type {{content?: Node}} */ (node).content;
+  for (const child of [...('childNodes' in node ? node.childNodes : []), ...(content ? [content] : [])]) {
+    walk(/** @type {Node} */ (child), visit);
+  }
+}
+
+/** @param {string} html */
+function elementsOf(html) {
+  /** @type {Element[]} */
+  const elements = [];
+  walk(parse(html), (node) => {
+    if ('tagName' in node) elements.push(node);
+  });
+  return elements;
+}
+
+/**
+ * @param {Element} element
+ * @param {string} name
+ */
+const attr = (element, name) => element.attrs.find((item) => item.name === name)?.value;
+
+/** @param {Element} element */
+const textOf = (element) => element.childNodes.map((child) => 'value' in child ? child.value : '').join('');
+
+/**
+ * Keys counted more often in `before` than in `after`
+ * @param {string[]} before
+ * @param {string[]} after
+ */
+function removed(before, after) {
+  /** @type {Map<string, number>} */
+  const counts = new Map();
+  for (const key of before) counts.set(key, (counts.get(key) ?? 0) + 1);
+  for (const key of after) counts.set(key, (counts.get(key) ?? 0) - 1);
+  return [...counts].filter(([, count]) => count > 0).map(([key]) => key);
+}
+
+/**
+ * Whether any element may match a selector, judged by the tag, classes, IDs, and attributes
+ * of the last compound of each selector in the list—erring toward a match
+ * @param {string} selector
+ * @param {Element[]} elements
+ */
+function mayMatch(selector, elements) {
+  return selector.split(',').some((part) => {
+    const compound = (part.trim().replace(/::?[\w-]+(?:\([^)]*\))?/g, '').split(/[\s>+~]+/).pop() ?? '').trim();
+    const tag = compound.match(/^[a-z][\w-]*/i)?.[0].toLowerCase();
+    const classes = [...compound.matchAll(/\.([\w-]+)/g)].map((match) => match[1] ?? '');
+    const ids = [...compound.matchAll(/#([\w-]+)/g)].map((match) => match[1] ?? '');
+    const attributes = [...compound.matchAll(/\[\s*([\w-]+)/g)].map((match) => (match[1] ?? '').toLowerCase());
+    return elements.some((element) => {
+      const own = (attr(element, 'class') ?? '').split(/\s+/);
+      return (!tag || element.tagName === tag) &&
+        classes.every((name) => own.includes(name)) &&
+        ids.every((id) => attr(element, 'id') === id) &&
+        attributes.every((name) => element.attrs.some((item) => item.name === name));
+    });
+  });
+}
+
+/** @param {string} css */
+function whitespaceRulesOf(css) {
+  /** @type {{selector: string, text: string}[]} */
+  const rules = [];
+  // Without comments and statements like `@import`, which would otherwise lead the next selector
+  for (const match of css.replace(RE_CSS_COMMENT, '').replace(RE_CSS_STATEMENT, '').matchAll(/([^{}]*)\{([^{}]*)\}/g)) {
+    const selector = (match[1] ?? '').trim();
+    const declaration = (match[2] ?? '').match(RE_WHITESPACE_KEPT)?.[0];
+    if (!declaration || selector.startsWith('@')) continue;
+    if (selector.split(',').every((part) => SELECTORS_PREFORMATTED.has(part.trim().toLowerCase()))) continue;
+    rules.push({ selector, text: `${clip(selector, 40)} { ${declaration} }` });
+  }
+  return rules;
+}
+
+/** @param {string} css */
+const selectorTokens = (css) => {
+  let selectors = css.replace(RE_CSS_COMMENT, '');
+  // Strips declaration blocks innermost first, so that nested at-rules leave their selectors
+  for (let previous = ''; previous !== selectors;) {
+    previous = selectors;
+    selectors = selectors.replace(/\{[^{}]*\}/g, ',');
+  }
+  return [...selectors.matchAll(/[.#](-?[_a-zA-Z\u00A0-\uFFFF][\w\u00A0-\uFFFF-]*)/g)].map((match) => `${match[0][0]}${match[1]}`);
+};
+
+/**
+ * Collects what a page’s scripts and style sheets reference
+ * @param {import('./pages.js').Page} page
+ * @returns {Promise<PageContext>}
+ */
+export async function analyzePage(page) {
+  /** @type {string[]} */
+  const scripts = [];
+  /** @type {string[]} */
+  const scriptsUnread = [];
+  /** @type {string[]} */
+  const styles = [];
+  /** @type {Promise<void>[]} */
+  const pending = [];
+  /** @type {Set<string>} */
+  const imported = new Set();
+
+  /**
+   * Adds a style sheet and, where local or same-origin, what it imports
+   * @param {string} css
+   * @param {string | null} base Reference of the style sheet, if not in the page
+   */
+  const addStyles = (css, base) => {
+    styles.push(css);
+    for (const [, , ref = ''] of css.replace(RE_CSS_COMMENT, '').matchAll(RE_IMPORT)) {
+      let target;
+      try {
+        target = base === null ? ref : page.resolve(ref, base);
+      } catch {
+        continue;
+      }
+      if (imported.has(target)) continue;
+      imported.add(target);
+      pending.push(page.load(target).then((sheet) => { if (sheet) addStyles(sheet.text, sheet.ref); }));
+    }
+  };
+
+  for (const element of elementsOf(page.html)) {
+    for (const { name, value } of element.attrs) {
+      if (/^on[a-z]{3,}$/.test(name)) scripts.push(value);
+      if (name === 'style') styles.push(`[style]{${value}}`);
+    }
+    if (element.tagName === 'script' && RE_SCRIPT_JS.test((attr(element, 'type') ?? '').trim())) {
+      const src = attr(element, 'src');
+      if (src === undefined) {
+        scripts.push(textOf(element));
+      } else {
+        pending.push(page.load(src).then((script) => { script === null ? scriptsUnread.push(src) : scripts.push(script.text); }));
+      }
+    } else if (element.tagName === 'style') {
+      addStyles(textOf(element), null);
+    } else if (element.tagName === 'link' && /(?:^|\s)stylesheet(?:\s|$)/i.test(attr(element, 'rel') ?? '')) {
+      const href = attr(element, 'href');
+      if (href) pending.push(page.load(href).then((sheet) => { if (sheet) addStyles(sheet.text, sheet.ref); }));
+    }
+  }
+  // Imports add to what is pending while it settles
+  while (pending.length) await Promise.all(pending.splice(0));
+
+  const scriptText = scripts.join('\n;\n');
+  const css = styles.join('\n');
+  /** @type {Set<string>} */
+  const scriptStrings = new Set();
+  /** @type {Set<string>} */
+  const attributeRefs = new Set();
+  for (const literal of scriptText.matchAll(RE_STRING_LITERAL)) {
+    const content = literal[2] ?? '';
+    for (const word of content.match(RE_WORD) ?? []) scriptStrings.add(word);
+    for (const match of content.matchAll(RE_ATTRIBUTE_SELECTOR)) attributeRefs.add((match[1] ?? '').toLowerCase());
+  }
+  for (const match of scriptText.matchAll(RE_ATTRIBUTE_CALL)) attributeRefs.add((match[1] ?? '').toLowerCase());
+  for (const match of css.matchAll(RE_ATTRIBUTE_SELECTOR)) attributeRefs.add((match[1] ?? '').toLowerCase());
+
+  return { scriptStrings, scriptText, scriptsUnread: scriptsUnread.sort(), attributeRefs, whitespaceRules: whitespaceRulesOf(css) };
+}
+
+/**
+ * What may break when an option turns `before` into `after`
+ * @param {string} name Candidate name
+ * @param {PageContext} context
+ * @param {string} before Output without the option
+ * @param {string} after Output with the option
+ * @returns {Hazard[]}
+ */
+export function findHazards(name, context, before, after) {
+  if (before === after) return [];
+  /** @type {Hazard[]} */
+  const hazards = [];
+
+  if (OPTIONS_WHITESPACE.has(name) && context.whitespaceRules.length) {
+    const elements = elementsOf(before);
+    const applying = context.whitespaceRules.filter((rule) => mayMatch(rule.selector, elements));
+    if (applying.length) {
+      hazards.push({ level: 'check', message: `Collapses whitespace that CSS keeps (${applying.slice(0, 2).map((rule) => rule.text).join('; ')})` });
+    }
+  }
+
+  if (name === 'removeComments') {
+    /** @param {string} html */
+    const comments = (html) => {
+      /** @type {string[]} */
+      const data = [];
+      walk(parse(html), (node) => { if (node.nodeName === '#comment' && 'data' in node) data.push(node.data); });
+      return data;
+    };
+    const gone = removed(comments(before), comments(after));
+    /** @type {Map<string, number>} */
+    const perLabel = new Map();
+    for (const data of gone) {
+      const conditional = RE_COMMENT_CONDITIONAL.test(data);
+      const label = conditional ? 'conditional comment' : COMMENTS_MEANINGFUL.find(([re]) => re.test(data))?.[1];
+      if (!label || (perLabel.get(label) ?? 0) >= MAX_PER_KIND) continue;
+      perLabel.set(label, (perLabel.get(label) ?? 0) + 1);
+      hazards.push(conditional
+        ? { level: 'note', message: `Removes a conditional comment, which only Internet Explorer 9 and older read: <!--${clip(data, 40)}-->` }
+        : { level: 'risk', message: `Removes a comment that looks meaningful (${label}): <!--${clip(data, 40)}-->` });
+    }
+    if (gone.length && RE_READS_COMMENTS.test(context.scriptText)) {
+      hazards.push({ level: 'risk', message: 'Removes comments, and scripts read comments' });
+    }
+  }
+
+  if (name === 'removeEmptyElements') {
+    /** @param {string} html */
+    const signatures = (html) => elementsOf(html).map((element) => JSON.stringify([element.tagName, attr(element, 'id') ?? '', attr(element, 'class') ?? '']));
+    const gone = removed(signatures(before), signatures(after));
+    let count = 0;
+    for (const signature of gone) {
+      const [tag, id, classes] = /** @type {[string, string, string]} */ (JSON.parse(signature));
+      const named = [id, ...classes.split(/\s+/)].filter((token) => token && context.scriptStrings.has(token));
+      if (named.length && count++ < MAX_PER_KIND) {
+        const shown = `<${tag}${id ? ` id="${id}"` : ''}${classes ? ` class="${clip(classes, 30)}"` : ''}>`;
+        hazards.push({ level: 'risk', message: `Removes empty ${shown}, which scripts may fill (${named.map((token) => `\`${token}\``).join(', ')})` });
+      }
+    }
+    if (gone.length && context.scriptsUnread.length) {
+      hazards.push({ level: 'note', message: `Removes empty elements that unread scripts may fill (${clip(context.scriptsUnread.join(', '))})` });
+    }
+  }
+
+  if (name === 'removeUnusedCSS') {
+    /** @param {string} html */
+    const tokens = (html) => [...new Set(elementsOf(html).filter((element) => element.tagName === 'style').flatMap((element) => selectorTokens(textOf(element))))];
+    const gone = removed(tokens(before), tokens(after));
+    const named = gone.filter((token) => context.scriptStrings.has(token.slice(1)));
+    if (named.length) {
+      hazards.push({ level: 'risk', message: `Removes rules for ${named.slice(0, MAX_PER_KIND).map((token) => `\`${token}\``).join(', ')}, which scripts name and may add` });
+    }
+    if (gone.length && context.scriptsUnread.length) {
+      hazards.push({ level: 'note', message: `Removes rules that unread scripts may need (${clip(context.scriptsUnread.join(', '))})` });
+    }
+  }
+
+  if (OPTIONS_ATTRIBUTES.has(name)) {
+    /** @param {string} html */
+    const attributes = (html) => elementsOf(html).flatMap((element) => element.attrs.map(({ name: attrName, value }) => JSON.stringify([attrName, value])));
+    const changed = new Set(removed(attributes(before), attributes(after)).map((key) => /** @type {string} */ (JSON.parse(key)[0])));
+    const referenced = [...changed].filter((attrName) => context.attributeRefs.has(attrName)).sort();
+    if (referenced.length) {
+      hazards.push({ level: 'risk', message: `Changes ${referenced.map((attrName) => `\`${attrName}\``).join(', ')} attributes, which style sheets or scripts select on` });
+    }
+  }
+
+  if (name === 'useShortDoctype') {
+    const [modeBefore = '', modeAfter = ''] = [before, after].map((html) => String(parse(html).mode));
+    if (modeBefore !== modeAfter) {
+      hazards.push({ level: 'check', message: `Switches the page from ${MODES[modeBefore] ?? modeBefore} to ${MODES[modeAfter] ?? modeAfter}, which can change layout` });
+    }
+  }
+
+  if (name === 'mergeScripts' && /\bcurrentScript\b/.test(context.scriptText)) {
+    hazards.push({ level: 'check', message: 'Scripts use `document.currentScript`, which merging changes' });
+  }
+
+  return hazards;
+}

package/src/index.js

@@ -0,0 +1,8 @@
+export { CANDIDATES, CARRIED, EXCLUDED, FALLBACKS, MODIFIERS, PRESERVED, splitConfig } from './catalog.js';
+export { loadEquivalence } from './equivalence.js';
+export { analyzePage, findHazards } from './hazards.js';
+export { measure } from './measure.js';
+export { createPage, loadPages } from './pages.js';
+export { createRenderer } from './render.js';
+export { formatReport, toConfig } from './report.js';
+export { customize } from './select.js';

package/src/measure.js

@@ -0,0 +1,60 @@
+import { promisify } from 'node:util';
+import zlib from 'node:zlib';
+
+// As in HTML Minifier’s benchmarks: common defaults for on-the-fly compression (quality 11, for
+// files precompressed at build time, takes seconds per large file)
+export const GZIP_LEVEL = 6;
+export const BROTLI_QUALITY = 6;
+
+const brotliCompress = promisify(zlib.brotliCompress);
+const gzipCompress = promisify(zlib.gzip);
+
+/** @typedef {{raw: number, gzip: number, brotli: number}} Sizes */
+/** @typedef {keyof Sizes} Metric */
+
+/** @type {Metric[]} */
+export const METRICS = ['raw', 'gzip', 'brotli'];
+
+/** @type {Sizes} */
+export const SIZES_ZERO = Object.freeze({ raw: 0, gzip: 0, brotli: 0 });
+
+/**
+ * Size by one metric as servers would send it, compressed off the main thread
+ * @param {string} text
+ * @param {Metric} metric
+ * @returns {Promise<number>}
+ */
+export async function sizeOf(text, metric) {
+  const buffer = Buffer.from(text, 'utf8');
+  if (metric === 'raw') return buffer.length;
+  if (metric === 'gzip') return (await gzipCompress(buffer, { level: GZIP_LEVEL })).length;
+  return (await brotliCompress(buffer, {
+    params: {
+      [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT,
+      [zlib.constants.BROTLI_PARAM_QUALITY]: BROTLI_QUALITY,
+      [zlib.constants.BROTLI_PARAM_SIZE_HINT]: buffer.length
+    }
+  })).length;
+}
+
+/**
+ * @param {string} text
+ * @returns {Promise<Sizes>}
+ */
+export async function measure(text) {
+  const [raw, gzip, brotli] = await Promise.all([sizeOf(text, 'raw'), sizeOf(text, 'gzip'), sizeOf(text, 'brotli')]);
+  return { raw, gzip, brotli };
+}
+
+/**
+ * @param {Sizes} a
+ * @param {Sizes} b
+ * @returns {Sizes}
+ */
+export const addSizes = (a, b) => ({ raw: a.raw + b.raw, gzip: a.gzip + b.gzip, brotli: a.brotli + b.brotli });
+
+/**
+ * @param {Sizes[]} list
+ * @returns {Sizes}
+ */
+export const sumSizes = (list) => list.reduce(addSizes, SIZES_ZERO);

package/src/pages.js

@@ -0,0 +1,230 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+/**
+ * @typedef {object} Page
+ * @prop {string} id How the page is named in reports
+ * @prop {string} html
+ * @prop {string | null} url Where the page lives online, if read from there
+ * @prop {string | null} file
+ * @prop {string | null} root Directory that root-relative references resolve against
+ * @prop {(ref: string) => Promise<Resource | null>} load Reads a resource the page references, where local or same-origin
+ * @prop {(ref: string, base: string) => string} resolve Turns a reference made by a resource the page references into one made by the page
+ */
+
+/** @typedef {{text: string, ref: string}} Resource `ref` tells where the resource ended up after redirects */
+
+// As HTML Minifier’s
+export const EXTENSIONS_DEFAULT = ['html', 'htm', 'shtml', 'shtm'];
+const MAX_REDIRECTS = 5;
+const RE_SCHEME = /^[a-z][\w+.-]*:|^\/\//i;
+const RE_URL_INPUT = /^https?:\/\//i;
+
+/**
+ * Whether a path lies within the root (or is the root)
+ * @param {string} root
+ * @param {string} target
+ */
+export function isInside(root, target) {
+  const relative = path.relative(root, target);
+  return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
+}
+
+/**
+ * @param {{id: string, html: string, url?: string | null, file?: string | null, root?: string | null, resources?: Record<string, string>}} spec
+ * @returns {Page}
+ */
+export function createPage({ id, html, url = null, file = null, root = null, resources }) {
+  /** @type {Map<string, Promise<Resource | null>>} */
+  const cache = new Map();
+  /**
+   * @param {string} ref
+   * @returns {Promise<Resource | null>}
+   */
+  const read = async (ref) => {
+    if (url) return fetchSameOrigin(url, ref);
+    const text = resources ? resources[ref] ?? null : file && root ? await readLocal(file, root, ref) : null;
+    return text === null ? null : { text, ref };
+  };
+  // Local pages resolve as if served from their root
+  const location = url ?? `http://page.invalid/${id.split('/').map(encodeURIComponent).join('/')}`;
+  return {
+    id,
+    html,
+    url,
+    file,
+    root,
+    resolve(ref, base) {
+      const target = new URL(ref, new URL(base, location));
+      return url || target.origin !== new URL(location).origin ? target.href : `${target.pathname}${target.search}`;
+    },
+    load(ref) {
+      let pending = cache.get(ref);
+      if (!pending) {
+        pending = read(ref).catch(() => null);
+        cache.set(ref, pending);
+      }
+      return pending;
+    }
+  };
+}
+
+/**
+ * @param {string} file
+ * @param {string} root
+ * @param {string} ref
+ */
+async function readLocal(file, root, ref) {
+  if (RE_SCHEME.test(ref)) return null;
+  const clean = decodeURI(ref.replace(/[?#].*$/s, ''));
+  if (!clean) return null;
+  const target = clean.startsWith('/') ? path.join(root, clean) : path.resolve(path.dirname(file), clean);
+  if (!isInside(root, target)) return null;
+  return fs.promises.readFile(target, 'utf8');
+}
+
+/**
+ * @param {string} pageUrl
+ * @param {string} ref
+ * @returns {Promise<Resource | null>}
+ */
+async function fetchSameOrigin(pageUrl, ref) {
+  const { origin } = new URL(pageUrl);
+  let target = new URL(ref, pageUrl);
+  // Redirects are followed one by one, so that none leads to another origin
+  for (let redirects = 0; redirects <= MAX_REDIRECTS && target.origin === origin; redirects++) {
+    const response = await fetch(target, { redirect: 'manual', signal: AbortSignal.timeout(15000) });
+    const location = response.headers.get('location');
+    if (response.status < 300 || response.status > 399 || !location) return response.ok ? { text: await response.text(), ref: target.href } : null;
+    await response.body?.cancel();
+    target = new URL(location, target);
+  }
+  return null;
+}
+
+/**
+ * Picks pages round-robin across directories, so that a sample covers a site’s sections (and
+ * so, likely, its templates) rather than the first directory’s pages
+ * @param {string[]} paths
+ * @param {number} count
+ * @returns {string[]}
+ */
+export function samplePaths(paths, count) {
+  if (paths.length <= count) return paths;
+  /** @type {Map<string, string[]>} */
+  const byDir = new Map();
+  for (const p of [...paths].sort()) {
+    const dir = path.dirname(p);
+    const queue = byDir.get(dir) ?? [];
+    queue.push(p);
+    byDir.set(dir, queue);
+  }
+  // Within a directory, a golden-ratio order spreads picks rather than taking its first pages
+  const queues = [...byDir.values()].map((queue) => queue
+    .map((item, index) => ({ item, key: (index * 0.6180339887) % 1 }))
+    .sort((a, b) => a.key - b.key)
+    .map(({ item }) => item));
+  /** @type {string[]} */
+  const sample = [];
+  for (let round = 0; sample.length < count; round++) {
+    for (const queue of queues) {
+      const item = queue[round];
+      if (item !== undefined && sample.length < count) sample.push(item);
+    }
+  }
+  return sample;
+}
+
+/**
+ * @param {string} dir
+ * @param {string[]} extensions
+ * @param {string[]} ignore
+ * @returns {Promise<string[]>}
+ */
+async function walk(dir, extensions, ignore) {
+  /** @type {string[]} */
+  const files = [];
+  for (const entry of await fs.promises.readdir(dir, { withFileTypes: true })) {
+    if (entry.name.startsWith('.') || ignore.includes(entry.name)) continue;
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...await walk(full, extensions, ignore));
+    } else if (entry.isFile() && (extensions.includes('*') || extensions.includes(path.extname(entry.name).slice(1).toLowerCase()))) {
+      files.push(full);
+    }
+  }
+  return files;
+}
+
+/**
+ * Symbolic links under the root that lead outside it, which the render check follows only where
+ * allowed, as a page’s scripts could send on what they lead to
+ * @param {string} root
+ * @param {string[]} [ignore]
+ * @returns {Promise<{link: string, target: string}[]>}
+ */
+export async function findOutsideLinks(root, ignore = ['node_modules']) {
+  const rootReal = await fs.promises.realpath(root);
+  /** @type {{link: string, target: string}[]} */
+  const links = [];
+  /** @param {string} dir */
+  const visit = async (dir) => {
+    for (const entry of await fs.promises.readdir(dir, { withFileTypes: true }).catch(() => [])) {
+      if (entry.name.startsWith('.') || ignore.includes(entry.name)) continue;
+      const link = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        await visit(link);
+      } else if (entry.isSymbolicLink()) {
+        const target = await fs.promises.realpath(link).catch(() => null);
+        if (target && !isInside(rootReal, target)) links.push({ link, target });
+      }
+    }
+  };
+  await visit(root);
+  return links;
+}
+
+/**
+ * Reads pages from files, directories, and URLs
+ * @param {string[]} inputs
+ * @param {{extensions?: string[], ignore?: string[], sample?: number}} [settings]
+ * @returns {Promise<Page[]>}
+ */
+export async function loadPages(inputs, { extensions = EXTENSIONS_DEFAULT, ignore = ['node_modules'], sample = 20 } = {}) {
+  // As HTML Minifier reads them, with `*` for all files
+  const extensionsRead = extensions.map((ext) => ext.trim().replace(/^\.+/, '').toLowerCase());
+  /** @type {{file: string, root: string}[]} */
+  const files = [];
+  /** @type {string[]} */
+  const urls = [];
+  for (const input of inputs) {
+    if (RE_URL_INPUT.test(input)) {
+      urls.push(input);
+      continue;
+    }
+    const resolved = path.resolve(input);
+    const stat = await fs.promises.stat(resolved).catch(() => null);
+    if (!stat) throw new Error(`Not found: ${input}`);
+    if (stat.isDirectory()) {
+      for (const file of await walk(resolved, extensionsRead, ignore)) files.push({ file, root: resolved });
+    } else {
+      files.push({ file: resolved, root: path.dirname(resolved) });
+    }
+  }
+
+  const keys = samplePaths([...files.map(({ file }) => file), ...urls], sample);
+  return Promise.all(keys.map(async (key) => {
+    if (RE_URL_INPUT.test(key)) {
+      const response = await fetch(key, { signal: AbortSignal.timeout(30000) });
+      if (!response.ok) throw new Error(`Couldn’t fetch ${key}: ${response.status}`);
+      return createPage({ id: key, html: await response.text(), url: response.url });
+    }
+    const { root } = /** @type {{file: string, root: string}} */ (files.find(({ file }) => file === key));
+    return createPage({
+      id: path.relative(root, key).split(path.sep).join('/'),
+      html: await fs.promises.readFile(key, 'utf8'),
+      file: key,
+      root
+    });
+  }));
+}