altalandi 0.4.0 → 0.6.0

package/web.mjs

@@ -0,0 +1,558 @@
+// Web pages for `altalandi <url>`: fetch politely, keep the readable part, and
+// turn it into Markdown that the folder pipeline then refines like any other
+// document.
+//
+//   readPage(url)        one page → { url, title, markdown, links }
+//   readSite(url)        up to maxPages same-origin pages under the URL's path,
+//                        from /sitemap.xml or, without one, by following links
+//   htmlToMarkdown()     main/article content (else body) without navigation,
+//                        footers, scripts, styles, forms or hidden elements
+//
+// Every request names altalandi in its User-Agent, and robots.txt is obeyed: a
+// Disallow for altalandi or for * stops the fetch, and Crawl-delay is honoured.
+// Pages are fetched one at a time.
+//
+// Zero dependencies, like the rest of the CLI.
+
+import { decodeEntities, tagAttributes, tokenizeHtml, VOID } from "./segment.mjs";
+
+export const ROBOTS_AGENT = "altalandi";
+export const userAgent = (version) => `altalandi-cli/${version} (+https://altalandi.is)`;
+
+const MAX_PAGE_BYTES = 5 * 1024 * 1024;
+const PAGE_TIMEOUT_MS = 30_000;
+const MAX_CRAWL_DELAY_MS = 30_000;
+const MAX_CHILD_SITEMAPS = 10;
+const NOT_A_PAGE = /\.(?:pdf|png|jpe?g|gif|svg|webp|avif|ico|bmp|tiff?|zip|gz|tgz|rar|7z|tar|mp3|mp4|m4a|wav|ogg|webm|mov|avi|css|js|mjs|json|xml|rss|atom|txt|csv|xlsx?|docx?|pptx?|odt|woff2?|ttf|otf|eot|exe|dmg|apk|wasm)$/i;
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+function decodeBody(buf, contentType) {
+  let label = /charset\s*=\s*["']?([\w-]+)/i.exec(contentType || "")?.[1];
+  if (!label) {
+    const head = buf.subarray(0, 4096).toString("latin1");
+    label = /<meta[^>]+charset\s*=\s*["']?\s*([\w-]+)/i.exec(head)?.[1];
+  }
+  try {
+    return new TextDecoder(label || "utf-8").decode(buf);
+  } catch {
+    return new TextDecoder("utf-8").decode(buf);
+  }
+}
+
+/** GET with our User-Agent, a timeout and a size limit; the body is decoded
+ *  with the page's own charset. */
+async function get(url, io, { timeoutMs = PAGE_TIMEOUT_MS } = {}) {
+  const fetchImpl = io.fetch || globalThis.fetch;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetchImpl(url, {
+      headers: {
+        "user-agent": io.userAgent || userAgent("0"),
+        accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      },
+      redirect: "follow",
+      signal: controller.signal,
+    });
+    const type = res.headers?.get?.("content-type") || "";
+    if (Number(res.headers?.get?.("content-length") || 0) > MAX_PAGE_BYTES) throw new Error("larger than 5 MB");
+    const buf = typeof res.arrayBuffer === "function"
+      ? Buffer.from(await res.arrayBuffer())
+      : Buffer.from(String((await res.text()) ?? ""), "utf8");
+    if (buf.length > MAX_PAGE_BYTES) throw new Error("larger than 5 MB");
+    return { status: res.status, ok: res.ok ?? (res.status >= 200 && res.status < 300), url: res.url || url, type, body: decodeBody(buf, type) };
+  } catch (e) {
+    if (controller.signal.aborted) throw new Error(`timed out after ${timeoutMs / 1000} s`);
+    throw e;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+// ── robots.txt ──────────────────────────────────────────────────────────────
+
+/** Groups of a robots.txt: { agents, rules: [{ allow, path }], crawlDelay }. */
+export function parseRobots(text) {
+  const groups = [];
+  let group = null;
+  let agentLine = false;
+  for (const raw of String(text).split(/\r?\n/)) {
+    const line = raw.replace(/#.*$/, "").trim();
+    const m = /^([A-Za-z-]+)\s*:\s*(.*)$/.exec(line);
+    if (!m) continue;
+    const field = m[1].toLowerCase();
+    const value = m[2].trim();
+    if (field === "user-agent") {
+      if (!group || !agentLine) groups.push(group = { agents: [], rules: [], crawlDelay: null });
+      group.agents.push(value.toLowerCase());
+      agentLine = true;
+      continue;
+    }
+    agentLine = false;
+    if (!group) continue;
+    if (field === "disallow" && value) group.rules.push({ allow: false, path: value });
+    else if (field === "allow" && value) group.rules.push({ allow: true, path: value });
+    else if (field === "crawl-delay" && Number.isFinite(Number(value))) group.crawlDelay = Number(value);
+  }
+  return groups;
+}
+
+const ours = (agent) => agent !== "*" && (agent.startsWith(ROBOTS_AGENT) || agent === `${ROBOTS_AGENT}-cli`);
+
+function ruleMatches(rule, target) {
+  const anchored = rule.path.endsWith("$");
+  const body = anchored ? rule.path.slice(0, -1) : rule.path;
+  const pattern = body.split("*").map((s) => s.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join(".*");
+  const m = new RegExp(`^${pattern}${anchored ? "$" : ""}`).exec(target);
+  return m ? m[0].length : -1;
+}
+
+function groupAllows(groups, target) {
+  // The longest matching rule wins; on a tie, Allow wins (RFC 9309).
+  let best = { length: -1, allow: true };
+  for (const g of groups) {
+    for (const rule of g.rules) {
+      const length = ruleMatches(rule, target);
+      if (length > best.length || (length === best.length && length >= 0 && rule.allow)) best = { length, allow: rule.allow };
+    }
+  }
+  return best.allow;
+}
+
+/** May altalandi fetch this URL? Both the altalandi group and the * group must
+ *  allow it. */
+export function robotsAllows(groups, url) {
+  const u = new URL(url);
+  const target = u.pathname + u.search;
+  const mine = groups.filter((g) => g.agents.some(ours));
+  const star = groups.filter((g) => g.agents.includes("*"));
+  return (!mine.length || groupAllows(mine, target)) && (!star.length || groupAllows(star, target));
+}
+
+export function crawlDelayMs(groups) {
+  const delays = groups
+    .filter((g) => g.agents.some((a) => a === "*" || ours(a)))
+    .map((g) => g.crawlDelay)
+    .filter((d) => d != null);
+  return delays.length ? Math.min(MAX_CRAWL_DELAY_MS, Math.max(...delays) * 1000) : 0;
+}
+
+async function robotsFor(url, io, cache) {
+  const origin = new URL(url).origin;
+  if (!cache.has(origin)) {
+    let groups;
+    try {
+      const res = await get(`${origin}/robots.txt`, io, { timeoutMs: 15_000 });
+      if (res.status >= 200 && res.status < 300) groups = parseRobots(res.body);
+      else if (res.status >= 400 && res.status < 500) groups = []; // no robots.txt: nothing is disallowed
+      else groups = { error: `HTTP ${res.status}` };
+    } catch (e) {
+      groups = { error: e.message };
+    }
+    cache.set(origin, groups);
+  }
+  const groups = cache.get(origin);
+  // A robots.txt that exists but cannot be read is treated as "disallow all".
+  if (!Array.isArray(groups)) throw new Error(`could not read ${origin}/robots.txt (${groups.error}), so nothing was fetched from ${origin}`);
+  return groups;
+}
+
+// ── sitemap ─────────────────────────────────────────────────────────────────
+
+/** Page URLs and child sitemaps listed in a sitemap or sitemap index. */
+export function parseSitemap(xml) {
+  const locs = (tag) => [...String(xml).matchAll(new RegExp(`<(?:\\w+:)?${tag}\\b[^>]*>([\\s\\S]*?)<\\/(?:\\w+:)?${tag}>`, "gi"))]
+    .map((m) => /<(?:\w+:)?loc>\s*(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?\s*<\/(?:\w+:)?loc>/i.exec(m[1])?.[1])
+    .filter(Boolean)
+    .map((s) => decodeEntities(s.trim()));
+  return { pages: locs("url"), sitemaps: locs("sitemap") };
+}
+
+async function readSitemap(origin, io) {
+  let res;
+  try { res = await get(`${origin}/sitemap.xml`, io, { timeoutMs: 15_000 }); } catch { return null; }
+  if (!res.ok || !/<(?:\w+:)?(?:urlset|sitemapindex)\b/i.test(res.body)) return null;
+  const { pages, sitemaps } = parseSitemap(res.body);
+  const sameOrigin = (u) => { try { return new URL(u).origin === origin; } catch { return false; } };
+  for (const child of sitemaps.filter((u) => sameOrigin(u) && !/\.gz$/i.test(u)).slice(0, MAX_CHILD_SITEMAPS)) {
+    try {
+      const sub = await get(child, io, { timeoutMs: 15_000 });
+      if (sub.ok) pages.push(...parseSitemap(sub.body).pages);
+    } catch { /* one unreadable child sitemap does not end the run */ }
+  }
+  return pages.filter(sameOrigin);
+}
+
+// ── HTML → tree ─────────────────────────────────────────────────────────────
+
+// Elements whose start closes an open <p> (the HTML parsing rules, simplified).
+const CLOSES_P = new Set([
+  "address", "article", "aside", "blockquote", "details", "dialog", "div", "dl", "fieldset", "figcaption",
+  "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "main", "menu",
+  "nav", "ol", "p", "pre", "section", "table", "ul",
+]);
+const IMPLIED = {
+  li: [["li"], ["ul", "ol"]],
+  dt: [["dt", "dd"], ["dl"]],
+  dd: [["dt", "dd"], ["dl"]],
+  tr: [["tr", "td", "th"], ["table", "thead", "tbody", "tfoot"]],
+  td: [["td", "th"], ["tr", "table"]],
+  th: [["td", "th"], ["tr", "table"]],
+  thead: [["thead", "tbody", "tfoot", "tr", "td", "th"], ["table"]],
+  tbody: [["thead", "tbody", "tfoot", "tr", "td", "th"], ["table"]],
+  tfoot: [["thead", "tbody", "tfoot", "tr", "td", "th"], ["table"]],
+  option: [["option"], ["select", "datalist", "optgroup"]],
+};
+
+function closeOpen(stack, names, scope) {
+  for (let k = stack.length - 1; k > 0; k--) {
+    if (scope.includes(stack[k].name)) return;
+    if (names.includes(stack[k].name)) { stack.length = k; return; }
+  }
+}
+
+/** A forgiving tree: { name, attrs, children }, text as { name: "#text", text }. */
+export function parseHtml(src) {
+  const root = { name: "#document", attrs: {}, children: [] };
+  const stack = [root];
+  const top = () => stack[stack.length - 1];
+  for (const t of tokenizeHtml(src)) {
+    if (t.type === "text") top().children.push({ name: "#text", text: decodeEntities(src.slice(t.start, t.end)) });
+    else if (t.type === "rawtext" && (t.parent === "title" || t.parent === "textarea")) {
+      top().children.push({ name: "#text", text: decodeEntities(src.slice(t.start, t.end)) });
+    } else if (t.type === "tag") {
+      if (t.closing) {
+        for (let k = stack.length - 1; k > 0; k--) if (stack[k].name === t.name) { stack.length = k; break; }
+        continue;
+      }
+      if (CLOSES_P.has(t.name)) closeOpen(stack, ["p"], ["button", "td", "th", "table", "li"]);
+      if (IMPLIED[t.name]) closeOpen(stack, ...IMPLIED[t.name]);
+      const el = { name: t.name, attrs: tagAttributes(t.raw), children: [] };
+      top().children.push(el);
+      if (!VOID.has(t.name) && !t.selfClosing) stack.push(el);
+    }
+  }
+  return root;
+}
+
+function find(node, pred, { first = false, nested = true } = {}) {
+  const out = [];
+  const walk = (n) => {
+    for (const c of n.children || []) {
+      if (c.name[0] === "#") continue;
+      if (pred(c)) {
+        out.push(c);
+        if (first) return true;
+        if (!nested) continue;
+      }
+      if (walk(c) && first) return true;
+    }
+    return false;
+  };
+  walk(node);
+  return out;
+}
+
+const textOf = (node) => (node.name === "#text" ? node.text : (node.children || []).map(textOf).join(""));
+
+// ── HTML → Markdown ─────────────────────────────────────────────────────────
+
+const ALWAYS_DROP = new Set([
+  "script", "style", "noscript", "template", "svg", "canvas", "iframe", "object", "embed", "nav", "footer",
+  "form", "button", "select", "input", "textarea", "dialog", "head", "link", "meta", "map", "audio", "video",
+  "source", "track",
+]);
+const CHROME = /(?:^|[\s_-])(?:cookie|consent|breadcrumbs?|skip-link|sr-only|visually-hidden|screen-reader-text)(?:$|[\s_-])/i;
+const INLINE = new Set([
+  "a", "abbr", "b", "bdi", "bdo", "big", "cite", "code", "data", "del", "dfn", "em", "font", "i", "img", "ins",
+  "kbd", "label", "mark", "nobr", "q", "s", "samp", "small", "span", "strike", "strong", "sub", "sup", "time",
+  "tt", "u", "var", "wbr", "br",
+]);
+const BR = "\u0000br\u0000";
+
+function dropped(el, ctx) {
+  const role = (el.attrs.role || "").toLowerCase();
+  return ALWAYS_DROP.has(el.name)
+    || el.attrs.hidden !== undefined
+    || el.attrs["aria-hidden"] === "true"
+    || /^(?:navigation|contentinfo|search)$/.test(role)
+    || CHROME.test(`${el.attrs.class || ""} ${el.attrs.id || ""}`)
+    // Without a <main> or <article>, the page's own header and sidebars are
+    // site chrome, not content.
+    || (ctx.fallback && (el.name === "header" || el.name === "aside" || /^(?:banner|complementary)$/.test(role)));
+}
+
+// Text of an element without what dropped() removes (a "Copy" button in a <pre>).
+const visibleText = (node, ctx) => (node.name === "#text" ? node.text
+  : node.name[0] === "#" || dropped(node, ctx) ? "" : (node.children || []).map((c) => visibleText(c, ctx)).join(""));
+
+// Characters that would turn plain text into Markdown syntax.
+const escapeText = (s) => s
+  .replace(/[\\`*[\]<]/g, (c) => `\\${c}`)
+  .replace(/(?<![A-Za-z0-9])_|_(?![A-Za-z0-9])/g, "\\_");
+
+function escapeLineStart(line) {
+  if (/^(?:#{1,6}(?:\s|$)|[-+](?:\s|$)|>|={3,}\s*$)/.test(line)) return `\\${line}`;
+  const num = /^(\d{1,9})([.)])(?:\s|$)/.exec(line);
+  return num ? `${num[1]}\\${line.slice(num[1].length)}` : line;
+}
+
+function absolute(href, ctx) {
+  try {
+    return new URL(href, ctx.base).href.replace(/\(/g, "%28").replace(/\)/g, "%29");
+  } catch {
+    return null;
+  }
+}
+
+const wrapWith = (s, mark) => {
+  const m = /^(\s*)([\s\S]*?)(\s*)$/.exec(s);
+  return m[2] ? `${m[1]}${mark}${m[2]}${mark}${m[3]}` : s;
+};
+
+function codeSpan(text) {
+  const s = text.replace(/\s+/g, " ").trim();
+  if (!s) return "";
+  const longest = Math.max(0, ...(s.match(/`+/g) || []).map((r) => r.length));
+  const ticks = "`".repeat(longest + 1);
+  const pad = s.startsWith("`") || s.endsWith("`") ? " " : "";
+  return `${ticks}${pad}${s}${pad}${ticks}`;
+}
+
+function inline(node, ctx) {
+  if (node.name === "#text") return escapeText(node.text);
+  if (node.name[0] === "#" || dropped(node, ctx)) return "";
+  const inner = () => node.children.map((c) => inline(c, ctx)).join("");
+  switch (node.name) {
+    case "br": return BR;
+    case "img": {
+      const src = node.attrs.src ? absolute(node.attrs.src, ctx) : null;
+      return src ? `![${escapeText((node.attrs.alt || "").trim())}](${src})` : "";
+    }
+    case "a": {
+      const text = inner().trim();
+      const href = node.attrs.href;
+      if (!text) return "";
+      if (!href || href.startsWith("#") || /^\s*javascript:/i.test(href)) return text;
+      const target = absolute(href, ctx);
+      return target ? `[${text}](${target})` : text;
+    }
+    case "strong": case "b": return wrapWith(inner(), "**");
+    case "em": case "i": return wrapWith(inner(), "*");
+    case "del": case "s": case "strike": return wrapWith(inner(), "~~");
+    case "code": case "kbd": case "samp": case "var": case "tt": return codeSpan(visibleText(node, ctx));
+    default: return inner();
+  }
+}
+
+function paragraph(raw) {
+  const s = raw.replace(/\s+/g, " ")
+    .replace(new RegExp(`\\s*${BR}\\s*`, "g"), BR)
+    .replace(new RegExp(`^(?:${BR}|\\s)+|(?:${BR}|\\s)+$`, "g"), "");
+  return s ? s.split(BR).map((line) => escapeLineStart(line.trim())).join("\\\n") : "";
+}
+
+function blocks(node, ctx) {
+  const out = [];
+  let run = [];
+  const flush = () => {
+    const p = paragraph(run.join(""));
+    if (p) out.push(p);
+    run = [];
+  };
+  for (const child of node.children || []) {
+    if (child.name === "#text") { run.push(escapeText(child.text)); continue; }
+    if (child.name[0] === "#" || dropped(child, ctx)) continue;
+    if (INLINE.has(child.name)) { run.push(inline(child, ctx)); continue; }
+    flush();
+    out.push(...block(child, ctx));
+  }
+  flush();
+  return out;
+}
+
+const indentBlock = (s, pad) => s.split("\n").map((l) => (l ? pad + l : l)).join("\n");
+const isList = (s) => /^(?:- |\d+\. )/.test(s);
+
+function block(el, ctx) {
+  const h = /^h([1-6])$/.exec(el.name);
+  if (h) {
+    const text = paragraph(el.children.map((c) => inline(c, ctx)).join("")).replace(/\\\n/g, " ");
+    return text ? [`${"#".repeat(Number(h[1]))} ${text}`] : [];
+  }
+  switch (el.name) {
+    case "p": {
+      const p = paragraph(el.children.map((c) => inline(c, ctx)).join(""));
+      return p ? [p] : [];
+    }
+    case "ul":
+    case "ol": {
+      let n = Number.parseInt(el.attrs.start, 10) || 1;
+      const items = el.children.filter((c) => c.name === "li").map((li) => {
+        const marker = el.name === "ol" ? `${n++}. ` : "- ";
+        const body = blocks(li, ctx);
+        if (!body.length) return null;
+        const pad = " ".repeat(marker.length);
+        let s = marker + indentBlock(body[0], pad).slice(pad.length);
+        for (const b of body.slice(1)) s += (isList(b) ? "\n" : "\n\n") + indentBlock(b, pad);
+        return s;
+      }).filter(Boolean);
+      return items.length ? [items.join("\n")] : [];
+    }
+    case "blockquote": {
+      const inner = blocks(el, ctx).join("\n\n");
+      return inner ? [inner.split("\n").map((l) => (l ? `> ${l}` : ">")).join("\n")] : [];
+    }
+    case "pre": {
+      const code = visibleText(el, ctx).replace(/^\n/, "").replace(/\s+$/, "");
+      if (!code) return [];
+      const lang = /(?:^|\s)(?:language|lang)-([\w+-]+)/.exec(`${el.attrs.class || ""} ${find(el, (c) => c.name === "code", { first: true })[0]?.attrs.class || ""}`)?.[1] || "";
+      const longest = Math.max(2, ...(code.match(/`{3,}/g) || []).map((r) => r.length));
+      const fence = "`".repeat(longest + 1);
+      return [`${fence}${lang}\n${code}\n${fence}`];
+    }
+    case "table": {
+      const rows = find(el, (c) => c.name === "tr", { nested: false }).map((tr) => tr.children
+        .filter((c) => c.name === "td" || c.name === "th")
+        .map((cell) => paragraph(cell.children.map((c) => inline(c, ctx)).join("")).replace(/\\\n/g, " ").replace(/\|/g, "\\|")));
+      const width = Math.max(0, ...rows.map((r) => r.length));
+      if (!width) return [];
+      const line = (cells) => `| ${[...cells, ...Array(width - cells.length).fill("")].join(" | ")} |`;
+      return [[line(rows[0]), line(Array(width).fill("---")), ...rows.slice(1).map(line)].join("\n")];
+    }
+    case "hr": return ["---"];
+    case "dl": return el.children
+      .filter((c) => c.name === "dt" || c.name === "dd")
+      .flatMap((c) => (c.name === "dt"
+        ? [wrapWith(paragraph(c.children.map((x) => inline(x, ctx)).join("")), "**")].filter(Boolean)
+        : blocks(c, ctx)));
+    default: return blocks(el, ctx); // div, section, article, li outside a list, figure …
+  }
+}
+
+/** The readable part of a page as Markdown, with the page title. */
+export function htmlToMarkdown(html, pageUrl) {
+  const doc = parseHtml(html);
+  const baseHref = find(doc, (el) => el.name === "base" && el.attrs.href, { first: true })[0]?.attrs.href;
+  let base = pageUrl;
+  try { if (baseHref) base = new URL(baseHref, pageUrl).href; } catch { /* keep the page URL */ }
+  const title = textOf(find(doc, (el) => el.name === "title", { first: true })[0] || { name: "#text", text: "" }).replace(/\s+/g, " ").trim();
+
+  const main = find(doc, (el) => el.name === "main" || el.attrs.role === "main", { first: true })[0];
+  const articles = main ? [] : find(doc, (el) => el.name === "article", { nested: false });
+  const body = find(doc, (el) => el.name === "body", { first: true })[0] || doc;
+  const ctx = { base, fallback: !main && !articles.length };
+  const root = main || (articles.length === 1 ? articles[0] : articles.length ? { name: "#group", attrs: {}, children: articles } : body);
+
+  const parts = root.name === "#group" ? root.children.flatMap((a) => blocks(a, ctx)) : blocks(root, ctx);
+  if (title && !parts.some((p) => /^# /.test(p))) parts.unshift(`# ${escapeText(title)}`);
+  return { title, markdown: parts.join("\n\n") };
+}
+
+/** Links on a page, absolute, without fragments; rel=nofollow is respected. */
+export function pageLinks(html, pageUrl) {
+  const doc = parseHtml(html);
+  const baseHref = find(doc, (el) => el.name === "base" && el.attrs.href, { first: true })[0]?.attrs.href;
+  let base = pageUrl;
+  try { if (baseHref) base = new URL(baseHref, pageUrl).href; } catch { /* keep the page URL */ }
+  const links = [];
+  for (const a of find(doc, (el) => el.name === "a" && el.attrs.href)) {
+    if (/(?:^|\s)nofollow(?:\s|$)/i.test(a.attrs.rel || "")) continue;
+    try {
+      const u = new URL(a.attrs.href, base);
+      if (u.protocol !== "http:" && u.protocol !== "https:") continue;
+      u.hash = "";
+      links.push(u.href);
+    } catch { /* not a URL */ }
+  }
+  return links;
+}
+
+// ── pages and sites ─────────────────────────────────────────────────────────
+
+const withoutHash = (url) => { const u = new URL(url); u.hash = ""; return u.href; };
+
+/** Fetch one page if robots.txt allows it. Throws with a readable reason. */
+export async function readPage(url, io = {}, robots = new Map()) {
+  if (!robotsAllows(await robotsFor(url, io, robots), url)) throw new Error(`robots.txt does not allow altalandi to read ${url}`);
+  const res = await get(url, io);
+  if (!res.ok) throw new Error(`HTTP ${res.status} from ${url}`);
+  if (res.url !== url && !robotsAllows(await robotsFor(res.url, io, robots), res.url)) {
+    throw new Error(`robots.txt does not allow altalandi to read ${res.url}`);
+  }
+  if (res.type && !/html|xml/i.test(res.type)) throw new Error(`${url} is not an HTML page (${res.type.split(";")[0]})`);
+  const { title, markdown } = htmlToMarkdown(res.body, res.url);
+  return { url: res.url, title, markdown, links: pageLinks(res.body, res.url) };
+}
+
+/** Pages of a site, starting with `url`: same origin, under the URL's folder
+ *  (https://x.is/docs/intro covers /docs/…), from /sitemap.xml or, without
+ *  one, by following links. Returns { pages, skipped, source }. */
+export async function readSite(url, { maxPages = 20, io = {}, onPage } = {}) {
+  const robots = new Map();
+  const wait = io.sleep || sleep;
+  const first = await readPage(url, io, robots);
+  onPage?.(first);
+  const start = new URL(first.url);
+  const scope = start.pathname.slice(0, start.pathname.lastIndexOf("/") + 1) || "/";
+  const inScope = (link) => {
+    try {
+      const u = new URL(link);
+      return u.origin === start.origin && u.pathname.startsWith(scope) && !NOT_A_PAGE.test(u.pathname);
+    } catch {
+      return false;
+    }
+  };
+  const pages = [first];
+  const skipped = [];
+  const seen = new Set([withoutHash(url), withoutHash(first.url)]);
+  const listed = await readSitemap(start.origin, io);
+  const source = listed && listed.length ? "sitemap" : "links";
+  const queue = (source === "sitemap" ? listed : first.links).filter(inScope);
+  const delay = crawlDelayMs(await robotsFor(first.url, io, robots));
+  while (queue.length && pages.length < maxPages) {
+    const next = withoutHash(queue.shift());
+    if (seen.has(next)) continue;
+    seen.add(next);
+    if (!robotsAllows(await robotsFor(next, io, robots), next)) { skipped.push({ url: next, reason: "robots.txt" }); continue; }
+    if (delay) await wait(delay);
+    let page;
+    try {
+      page = await readPage(next, io, robots);
+    } catch (e) {
+      skipped.push({ url: next, reason: e.message });
+      continue;
+    }
+    const landed = withoutHash(page.url);
+    if (landed !== next && seen.has(landed)) continue;
+    seen.add(landed);
+    if (!inScope(page.url)) { skipped.push({ url: next, reason: `redirected to ${page.url}` }); continue; }
+    pages.push(page);
+    onPage?.(page);
+    if (source === "links") for (const link of page.links) if (inScope(link) && !seen.has(withoutHash(link))) queue.push(link);
+  }
+  return { pages, skipped, source };
+}
+
+/** A file name for a page: /docs/intro → docs/intro.md, / → index.md. */
+export function pagePath(url) {
+  const u = new URL(url);
+  const decode = (s) => { try { return decodeURIComponent(s.replace(/\+/g, " ")); } catch { return s; } };
+  let path = decode(u.pathname).replace(/^\/+/, "");
+  if (!path || path.endsWith("/")) path += "index";
+  path = path.replace(/\.(?:html?|php|aspx?)$/i, "");
+  const segments = path.split("/").map(safeSegment).filter((s) => s && s !== "." && s !== "..");
+  let rel = segments.join("/") || "index";
+  if (u.search) rel += `-${safeSegment(decode(u.search.slice(1)).replace(/[=&;]/g, "-")).slice(0, 40)}`;
+  return `${rel}.md`;
+}
+
+/** One path segment that is safe on Windows, macOS and Linux. */
+export function safeSegment(s) {
+  const clean = String(s)
+    .replace(/[<>:"/\\|?*\u0000-\u001f\s]+/g, "-")
+    .replace(/-{2,}/g, "-")
+    .replace(/^[-.]+|[-.]+$/g, "");
+  return /^(?:con|prn|aux|nul|com\d|lpt\d)$/i.test(clean) ? `${clean}-` : clean;
+}