@nurkamol/seo-audit 1.31.0

package/src/compare.mjs

@@ -0,0 +1,100 @@
+// The same page, asked for twice by two different readers.
+//
+// A site that serves one thing to Googlebot and another to a browser is either
+// cloaking or misconfiguring its bot protection, and both are invisible to an
+// audit that fetches once. This asks a sample of pages again as somebody else
+// and reports what changed.
+//
+// Deliberately not a byte comparison. A nonce, a timestamp, a cart count and a
+// session id all differ between two fetches of the same page by the same
+// client, and reporting those would drown the one case that matters. What is
+// compared is what a search engine reads: the status, the title, the canonical,
+// the indexing directives, and how much content and how many links there are.
+import { parseHtml } from './parse.mjs';
+import { Fetcher, mapLimit } from './http.mjs';
+
+const f = (level, id, title, detail, url) => ({ level, id, title, detail, url });
+
+/** Roughly, since a byte count would fire on a timestamp. Two pages within a
+ *  tenth of each other are the same page as far as this is concerned. */
+const materiallyDifferent = (a, b) => Math.abs(a - b) > Math.max(a, b) * 0.1;
+
+/** What one reader was given that the other was not. */
+function differences(mine, theirs) {
+  const out = [];
+  if (mine.status !== theirs.status) out.push(`HTTP ${mine.status} became HTTP ${theirs.status}`);
+  if (!mine.doc || !theirs.doc) return out;
+
+  if ((mine.doc.title ?? '') !== (theirs.doc.title ?? '')) {
+    out.push(`the title changed from "${mine.doc.title ?? '(none)'}" to "${theirs.doc.title ?? '(none)'}"`);
+  }
+  const canonical = (d) => d.canonical?.[0] ?? '(none)';
+  if (canonical(mine.doc) !== canonical(theirs.doc)) {
+    out.push(`the canonical changed from ${canonical(mine.doc)} to ${canonical(theirs.doc)}`);
+  }
+  const robots = (d) => (d.robots ?? '(none)').toLowerCase();
+  if (robots(mine.doc) !== robots(theirs.doc)) {
+    out.push(`the robots meta changed from "${robots(mine.doc)}" to "${robots(theirs.doc)}"`);
+  }
+  if (materiallyDifferent(mine.doc.words, theirs.doc.words)) {
+    out.push(`the page went from ${mine.doc.words} words to ${theirs.doc.words}`);
+  }
+  const links = (d) => d.links.internal.length;
+  if (materiallyDifferent(links(mine.doc), links(theirs.doc))) {
+    out.push(`the internal links went from ${links(mine.doc)} to ${links(theirs.doc)}`);
+  }
+  return out;
+}
+
+/** Re-fetch a sample of crawled pages as somebody else.
+ *
+ *  A sample rather than the whole crawl, because this doubles the request cost
+ *  of every page it looks at, and because a site that cloaks does it to the
+ *  template rather than to page 400 alone. The report says how many were
+ *  compared so a sample never reads as a clean bill of health for the site. */
+export async function compareAgents(pages, { agent, label, sample = 10, onProgress } = {}) {
+  const live = pages.filter((p) => p.doc && p.res.ok);
+  if (!live.length || !agent) return [];
+
+  // Spread across the crawl rather than taking the first ten, which on a
+  // sitemap ordered by date would be ten pages of one month.
+  const step = Math.max(1, Math.floor(live.length / sample));
+  const chosen = live.filter((_, i) => i % step === 0).slice(0, sample);
+
+  const other = new Fetcher({ concurrency: 3, userAgent: agent });
+  const results = await mapLimit(chosen, 3, async (page) => {
+    const res = await other.get(page.url);
+    onProgress?.({ phase: 'compare', status: res.status, ms: res.ms, url: page.url });
+    const isHtml = /text\/html/i.test(res.headers.get('content-type') ?? '');
+    return {
+      page,
+      theirs: { status: res.status, doc: res.ok && isHtml ? parseHtml(res.body, page.url) : null },
+    };
+  });
+
+  const out = [];
+  const changed = [];
+  for (const { page, theirs } of results) {
+    const diffs = differences({ status: page.res.status, doc: page.doc }, theirs);
+    if (diffs.length) changed.push({ url: page.url, diffs });
+  }
+
+  for (const { url, diffs } of changed.slice(0, 5)) {
+    out.push(f('warn', 'serves-differently', `This page is different when ${label} asks for it`,
+      `${diffs.join('; ')}. A page that changes with the reader is either cloaking or bot protection ` +
+        'misfiring, and Google indexes the version it was given, not the one a person sees.', url));
+  }
+
+  out.push(f('info', 'compare-sampled',
+    changed.length
+      ? `${changed.length} of ${chosen.length} sampled pages differ when ${label} asks`
+      : `${chosen.length} pages were identical when ${label} asked`,
+    changed.length
+      ? `Compared ${chosen.length} pages spread across the crawl, not all ${live.length}, because this ` +
+        'doubles the cost of every page it looks at. The rest were not compared.'
+      : `Compared ${chosen.length} pages spread across the crawl. Status, title, canonical, robots meta, ` +
+        'word count and link count all matched, which is what a site that is not cloaking looks like.',
+    live[0].url));
+
+  return out;
+}

package/src/config.mjs

@@ -0,0 +1,156 @@
+// Configuration: what this site has decided to live with.
+//
+// Every real site has findings that are true and deliberate — a contact page
+// is meant to be short, a privacy policy has no business carrying editorial
+// links. Without a way to say so, the report fills with noise nobody reads,
+// and the one new finding that matters is lost in it.
+//
+// Looked for in the working directory, or passed with --config:
+//
+//   seo-audit.config.json
+//   {
+//     "limit": 200,
+//     "failOn": "error",
+//     "ignore": [
+//       "img-srcset",                                  // everywhere
+//       { "id": "thin-content", "urls": ["/contact/", "/*/legal/**"] },
+//       { "id": "no-editorial-links", "urls": ["**/privacy-policy/"] }
+//     ],
+//     "expect": [
+//       { "urls": ["/journal/*/"], "types": ["BlogPosting"] },
+//       { "urls": ["/"], "types": ["LocalBusiness"] }
+//     ]
+//   }
+import { readFileSync, existsSync } from 'node:fs';
+
+const FILENAMES = ['seo-audit.config.json', '.seo-audit.json'];
+
+/** Glob over URL paths: `*` stops at a slash, `**` does not. */
+export function matchGlob(pattern, path) {
+  const rx = pattern
+    .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+    // A placeholder while `*` is translated, written as an escape rather than
+    // as the byte itself: a raw NUL in the source makes this file binary to
+    // git and invisible to grep, which is how it went unnoticed that the glob
+    // matcher lived here at all.
+    .replace(/\*\*/g, '\u0000')
+    .replace(/\*/g, '[^/]*')
+    .replace(/\u0000/g, '.*');
+  return new RegExp(`^${rx}$`).test(path);
+}
+
+/** A rule matches a finding when the id matches and, if the rule names URLs,
+ *  one of them matches the finding's path. */
+function ruleMatches(rule, finding) {
+  const id = typeof rule === 'string' ? rule : rule.id;
+  if (id !== finding.id) return false;
+  const urls = typeof rule === 'string' ? null : rule.urls;
+  if (!urls?.length) return true;
+  if (!finding.url) return false;
+  let path;
+  try {
+    path = new URL(finding.url).pathname;
+  } catch {
+    path = finding.url;
+  }
+  return urls.some((u) => matchGlob(u, path));
+}
+
+export function loadConfig(explicitPath) {
+  const path = explicitPath ?? FILENAMES.find((f) => existsSync(f));
+  if (!path) return { source: null };
+  if (!existsSync(path)) throw new Error(`Config not found: ${path}`);
+  try {
+    return { ...JSON.parse(readFileSync(path, 'utf8')), source: path };
+  } catch (err) {
+    throw new Error(`Config is not valid JSON (${path}): ${err.message}`);
+  }
+}
+
+/** The sites to audit, each with whatever it overrides.
+ *
+ *  A portfolio is not a list of interchangeable sites: one has a deliberately
+ *  short contact page, another has no journal to expect BlogPosting on. So a
+ *  `sites` entry may be a bare URL or an object carrying its own settings,
+ *  which are merged over the shared config rather than replacing it.
+ *
+ *    "sites": [
+ *      "https://one.example",
+ *      { "url": "https://two.example", "ignore": ["thin-content"], "limit": 50 }
+ *    ]
+ *
+ *  URLs given on the command line win over the config file entirely, because
+ *  naming sites explicitly is how you audit a subset of a portfolio. */
+export function resolveSites(positional = [], file = {}) {
+  const list = positional.length ? positional : (file.sites ?? []);
+  return list
+    .map((entry) => (typeof entry === 'string' ? { url: entry } : { ...entry }))
+    .filter((entry) => entry.url)
+    .map(({ url, ...overrides }) => ({
+      url: /^https?:\/\//i.test(url) ? url : `https://${url}`,
+      overrides,
+    }));
+}
+
+/** Shared options with one site's overrides on top. `ignore` accumulates,
+ *  since a portfolio rule and a site rule are both meant to apply. */
+export function optionsForSite(shared, overrides = {}) {
+  return {
+    ...shared,
+    ...overrides,
+    ignore: [...(shared.ignore ?? []), ...(overrides.ignore ?? [])],
+  };
+}
+
+/** Drop findings the site has accepted. Returns [kept, ignoredCount]. */
+export function applyIgnores(findings, ignore = []) {
+  if (!ignore.length) return [findings, 0];
+  const kept = findings.filter((f) => !ignore.some((rule) => ruleMatches(rule, f)));
+  return [kept, findings.length - kept.length];
+}
+
+/** Structured-data expectations, as findings. `expect` says which schema types
+ *  a group of pages must carry — the difference between "the JSON parses" and
+ *  "this article is actually marked up as an article". */
+export function expectationChecks(pages, expect = []) {
+  const out = [];
+  if (!expect.length) return out;
+
+  for (const page of pages) {
+    if (!page.doc) continue;
+    let path;
+    try {
+      path = new URL(page.url).pathname;
+    } catch {
+      continue;
+    }
+    const rules = expect.filter((rule) => rule.urls.some((u) => matchGlob(u, path)));
+    if (!rules.length) continue;
+
+    // Every @type on the page, including those nested in a @graph.
+    const types = new Set();
+    const collect = (node) => {
+      if (!node || typeof node !== 'object') return;
+      if (Array.isArray(node)) return node.forEach(collect);
+      for (const t of [node['@type']].flat().filter(Boolean)) types.add(t);
+      if (node['@graph']) collect(node['@graph']);
+    };
+    for (const block of page.doc.jsonld) if (block.ok) collect(block.data);
+
+    for (const rule of rules) {
+      const missing = rule.types.filter((t) => !types.has(t));
+      if (missing.length) {
+        out.push({
+          level: 'error',
+          id: 'schema-expected',
+          title: `Missing expected structured data: ${missing.join(', ')}`,
+          detail: types.size
+            ? `Page declares ${[...types].join(', ')}.`
+            : 'Page has no structured data at all.',
+          url: page.url,
+        });
+      }
+    }
+  }
+  return out;
+}

package/src/console.mjs

@@ -0,0 +1,146 @@
+// Search Console: what the pages in this crawl actually do in Google.
+//
+// Every ordering in this report so far is derived from the site's own markup —
+// how many links point at a page, how far it is from the homepage. Those are
+// good proxies. Impressions are not a proxy: a broken canonical on a page with
+// four thousand impressions a month is a different sentence from the same
+// canonical on a page nobody has ever been shown.
+//
+// Opt-in, and the only thing in this tool that needs an account. Credentials
+// are read the way the PageSpeed key is — the environment first, then
+// ~/.config/seo-audit/.env — and never from the repository.
+import { readFileSync, existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+const TOKEN_URL = 'https://oauth2.googleapis.com/token';
+const API = 'https://searchconsole.googleapis.com/webmasters/v3/sites';
+
+const f = (level, id, title, detail, url) => ({ level, id, title, detail, url });
+
+/** One credential, environment first. */
+export function findCredential(name, env = process.env, read = readFileSync) {
+  if (env[name]) return env[name];
+  const dotfile = join(homedir(), '.config', 'seo-audit', '.env');
+  if (!existsSync(dotfile)) return null;
+  const match = read(dotfile, 'utf8').match(new RegExp(`^\\\\s*${name}\\\\s*=\\\\s*(\\\\S+)`, 'm'));
+  return match?.[1] ?? null;
+}
+
+/** All three, or a sentence saying which is missing. */
+export function credentials(env = process.env) {
+  const found = {
+    clientId: findCredential('GSC_CLIENT_ID', env),
+    clientSecret: findCredential('GSC_CLIENT_SECRET', env),
+    refreshToken: findCredential('GSC_REFRESH_TOKEN', env),
+  };
+  const missing = Object.entries({
+    GSC_CLIENT_ID: found.clientId,
+    GSC_CLIENT_SECRET: found.clientSecret,
+    GSC_REFRESH_TOKEN: found.refreshToken,
+  })
+    .filter(([, value]) => !value)
+    .map(([name]) => name);
+  return missing.length ? { missing } : found;
+}
+
+/** A refresh token is the long-lived half; this trades it for the short one. */
+async function accessToken({ clientId, clientSecret, refreshToken }, fetcher = fetch) {
+  const res = await fetcher(TOKEN_URL, {
+    method: 'POST',
+    headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams({
+      client_id: clientId,
+      client_secret: clientSecret,
+      refresh_token: refreshToken,
+      grant_type: 'refresh_token',
+    }),
+  });
+  const data = await res.json();
+  if (!res.ok || !data.access_token) {
+    throw new Error(data.error_description ?? data.error ?? `HTTP ${res.status} from Google's token endpoint`);
+  }
+  return data.access_token;
+}
+
+const isoDaysAgo = (days, now) => new Date(now - days * 86400000).toISOString().slice(0, 10);
+
+/** Impressions and clicks per page for the last 28 days.
+ *
+ *  Search Console reports the last three days incompletely, so the window ends
+ *  three days back: a page that looks like it lost all its impressions
+ *  yesterday has usually just not been counted yet. */
+export async function pageTraffic(siteUrl, creds, { fetcher = fetch, now = Date.now(), rowLimit = 25000 } = {}) {
+  const token = await accessToken(creds, fetcher);
+  const res = await fetcher(`${API}/${encodeURIComponent(siteUrl)}/searchAnalytics/query`, {
+    method: 'POST',
+    headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+    body: JSON.stringify({
+      startDate: isoDaysAgo(31, now),
+      endDate: isoDaysAgo(3, now),
+      dimensions: ['page'],
+      rowLimit,
+    }),
+  });
+  const data = await res.json();
+  if (!res.ok) {
+    throw new Error(data.error?.message ?? `HTTP ${res.status} from Search Console`);
+  }
+  const traffic = new Map();
+  for (const row of data.rows ?? []) {
+    const page = row.keys?.[0];
+    if (!page) continue;
+    traffic.set(page.replace(/\/$/, ''), {
+      impressions: Math.round(row.impressions ?? 0),
+      clicks: Math.round(row.clicks ?? 0),
+    });
+  }
+  return traffic;
+}
+
+/** Attach traffic to findings, and say what was found.
+ *
+ *  A property Search Console does not have, or credentials it will not accept,
+ *  is reported as a note rather than thrown: the rest of the audit is still
+ *  worth having, and an audit that dies because an optional integration failed
+ *  is worse than one that says so. */
+export async function searchConsole(origin, findings, opts = {}) {
+  const creds = opts.credentials ?? credentials();
+  if (creds.missing) {
+    return [
+      f('info', 'search-console-unconfigured', 'Search Console was asked for but not configured',
+        `Missing ${creds.missing.join(', ')}. Set them in the environment or in ` +
+          '~/.config/seo-audit/.env — never in the repository. Without them the report is ordered by ' +
+          'how much of the site links to a page, which is a proxy for the same thing.', origin),
+    ];
+  }
+
+  let traffic;
+  try {
+    traffic = await pageTraffic(opts.siteUrl ?? `${origin}/`, creds, opts);
+  } catch (err) {
+    return [
+      f('info', 'search-console-failed', 'Search Console did not answer',
+        `${err.message}. The property has to be one this account can read, and a domain property is ` +
+          'named "sc-domain:example.com" rather than by its URL. Everything else in this report is ' +
+          'unaffected.', origin),
+    ];
+  }
+
+  let matched = 0;
+  for (const finding of findings) {
+    if (!finding.url) continue;
+    const seen = traffic.get(finding.url.replace(/\/$/, ''));
+    if (!seen) continue;
+    finding.traffic = seen;
+    matched++;
+  }
+
+  const shown = [...traffic.values()].reduce((n, t) => n + t.impressions, 0);
+  return [
+    f('info', 'search-console', `Search Console has ${traffic.size.toLocaleString()} pages for this site`,
+      `${matched.toLocaleString()} of this crawl's findings are on pages Google has shown, ` +
+        `${shown.toLocaleString()} times between them over 28 days. Findings are ordered by that where it ` +
+        'is known, and by how much of the site links to a page where it is not.', origin),
+  ];
+}

package/src/dupes.mjs

@@ -0,0 +1,164 @@
+// Pages whose content is the same page again.
+//
+// Titles and descriptions have been compared since early on; bodies never were,
+// and that is the axis that matters most — a hundred product pages that differ
+// by one word are a hundred pages competing with each other for one result,
+// and they burn the crawl budget that would have gone to the pages that differ.
+//
+// It costs no requests. The text was read, measured for `words`, and thrown
+// away; what is kept now is a sketch of it, a few hundred bytes a page.
+//
+// **MinHash**, rather than comparing text to text. Comparing every page with
+// every other page is quadratic, and at five thousand pages that is twelve
+// million comparisons of documents. A sketch of 64 numbers estimates how much
+// two pages overlap by how many of those numbers agree, and banding the sketch
+// puts likely pairs in the same bucket so most pairs are never compared at all.
+// The estimate is unbiased and its error at 64 samples is about 6% — which is
+// why the threshold below is nowhere near the edge.
+
+/** Words, lowercased, with everything that is not a letter or a number gone.
+ *  Punctuation and case differences are not content differences. */
+function normalise(text) {
+  return text
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]+/gu, ' ')
+    .split(/\s+/)
+    .filter(Boolean);
+}
+
+/** FNV-1a, 32-bit. Small, fast, well spread, and four lines. */
+function hash(text) {
+  let value = 0x811c9dc5;
+  for (let i = 0; i < text.length; i++) {
+    value ^= text.charCodeAt(i);
+    value = Math.imul(value, 0x01000193) >>> 0;
+  }
+  return value;
+}
+
+const SKETCH = 64;
+const SHINGLE = 5; // Five-word runs: long enough that a shared sentence is
+                   // meaningful, short enough that reordering does not hide a copy.
+
+/** The 64 permutations, fixed so two runs of this tool agree. Odd multipliers
+ *  so each is a bijection on 32 bits and no sketch position collapses. */
+const PERMUTATIONS = Array.from({ length: SKETCH }, (_, i) => ({
+  a: (2654435761 * (i + 1) * 2 + 1) >>> 0,
+  b: (40503 * (i + 7)) >>> 0,
+}));
+
+/** A sketch of a page's content, or `null` when there is not enough of it to
+ *  compare honestly.
+ *
+ *  `null` is returned rather than a weak sketch on purpose. A page with fifty
+ *  words shares most of them with any other page of fifty words, and a check
+ *  that reports those as duplicates would be reporting the length, not the
+ *  content. */
+export function fingerprint(text, { minimumWords = 100 } = {}) {
+  const words = normalise(text);
+  if (words.length < minimumWords) return null;
+
+  const sketch = new Array(SKETCH).fill(0xffffffff);
+  let shingles = 0;
+  for (let i = 0; i + SHINGLE <= words.length; i++) {
+    const value = hash(words.slice(i, i + SHINGLE).join(' '));
+    shingles++;
+    for (let k = 0; k < SKETCH; k++) {
+      const permuted = (Math.imul(value, PERMUTATIONS[k].a) + PERMUTATIONS[k].b) >>> 0;
+      if (permuted < sketch[k]) sketch[k] = permuted;
+    }
+  }
+  return shingles ? { sketch, words: words.length } : null;
+}
+
+/** How much two pages overlap, 0 to 1. The share of sketch positions that
+ *  agree estimates the Jaccard similarity of their five-word runs. */
+export function similarity(a, b) {
+  if (!a || !b) return 0;
+  let same = 0;
+  for (let i = 0; i < SKETCH; i++) if (a.sketch[i] === b.sketch[i]) same++;
+  return same / SKETCH;
+}
+
+const BAND = 4;
+
+/** Groups of pages whose content is near-identical.
+ *
+ *  `pages` is `[{ url, fingerprint, ...anything }]`. Anything without a
+ *  fingerprint is ignored rather than grouped with everything else.
+ *
+ *  Banding first: pages whose sketches agree across a whole band of four are
+ *  candidates, and only candidates are scored. Two pages at the threshold below
+ *  share a band with probability ~99%, so almost nothing real is missed, and
+ *  the pairs that are never compared are the ones that were never close. */
+export function cluster(pages, { threshold = 0.9 } = {}) {
+  const usable = pages.filter((p) => p.fingerprint);
+  if (usable.length < 2) return [];
+
+  const buckets = new Map();
+  for (const page of usable) {
+    for (let band = 0; band * BAND < SKETCH; band++) {
+      const key = band + ':' + page.fingerprint.sketch.slice(band * BAND, band * BAND + BAND).join(',');
+      buckets.set(key, [...(buckets.get(key) ?? []), page]);
+    }
+  }
+
+  // Union–find over the candidate pairs, so three pages that are each near the
+  // other two come out as one group of three rather than three pairs.
+  const parent = new Map(usable.map((p) => [p.url, p.url]));
+  const find = (url) => {
+    let root = url;
+    while (parent.get(root) !== root) root = parent.get(root);
+    // Point everything along the way straight at the root, so the next lookup
+    // is one step.
+    let walk = url;
+    while (parent.get(walk) !== root) {
+      const next = parent.get(walk);
+      parent.set(walk, root);
+      walk = next;
+    }
+    return root;
+  };
+  const union = (a, b) => {
+    const [ra, rb] = [find(a), find(b)];
+    if (ra !== rb) parent.set(ra, rb);
+  };
+
+  const scored = new Map();
+  for (const candidates of buckets.values()) {
+    if (candidates.length < 2 || candidates.length > 200) continue; // A bucket of
+    // everything is a bucket of nothing: it means the sketches are degenerate,
+    // not that the site is one page repeated.
+    for (let i = 0; i < candidates.length; i++) {
+      for (let j = i + 1; j < candidates.length; j++) {
+        const pair = [candidates[i].url, candidates[j].url].sort().join('\u0000');
+        if (scored.has(pair)) continue;
+        const score = similarity(candidates[i].fingerprint, candidates[j].fingerprint);
+        scored.set(pair, score);
+        if (score >= threshold) union(candidates[i].url, candidates[j].url);
+      }
+    }
+  }
+
+  const groups = new Map();
+  for (const page of usable) {
+    const root = find(page.url);
+    groups.set(root, [...(groups.get(root) ?? []), page]);
+  }
+
+  return [...groups.values()]
+    .filter((group) => group.length > 1)
+    .map((group) => {
+      const urls = group.map((p) => p.url).sort();
+      // The lowest score inside the group, so the number reported is the one
+      // that is true of every pair in it rather than of the closest pair.
+      let lowest = 1;
+      for (let i = 0; i < group.length; i++) {
+        for (let j = i + 1; j < group.length; j++) {
+          lowest = Math.min(lowest, similarity(group[i].fingerprint, group[j].fingerprint));
+        }
+      }
+      return { urls, pages: group, similarity: lowest };
+    })
+    .sort((a, b) => b.pages.length - a.pages.length || a.urls[0].localeCompare(b.urls[0]));
+}

package/src/graph.mjs

@@ -0,0 +1,89 @@
+// The site's internal link graph, built once and read by everything that needs
+// it: the orphan check, click depth, and the ordering of the report.
+//
+// It was computed inside crossPageChecks and thrown away, which meant the two
+// checks that used it could disagree with each other and nothing else could use
+// it at all. A finding's *reach* — how many links point at the pages it affects,
+// and how far those pages are from the homepage — is the difference between a
+// list of problems and a list of work worth doing, and it was already in memory.
+
+/** How a URL is compared with another. Fragments are not pages and a trailing
+ *  slash is not a difference. */
+export const key = (url) => (url ?? '').split('#')[0].replace(/\/$/, '');
+
+const isHome = (url) => {
+  try {
+    return new URL(url).pathname.replace(/\/$/, '') === '';
+  } catch {
+    return false;
+  }
+};
+
+/** Build the graph from crawled pages.
+ *
+ *  `home` is the page to measure distance from, and may be one the crawl never
+ *  fetched — a sitemap is under no obligation to list it. Returns:
+ *
+ *    depth    key → clicks from the homepage, absent when nothing reaches it
+ *    inlinks  key → how many pages link to it, self-links not counted
+ *    from     key → the page it was first reached through, for printing a route
+ *    stranded pages with no path from the homepage at all
+ *
+ *  Everything here is a count of links that were actually read. Nothing is
+ *  weighted, scored, or guessed. */
+export function linkGraph(live, home = null) {
+  const pages = new Map(live.map((p) => [key(p.url), p]));
+  const root = live.find((p) => isHome(p.url)) ?? (home?.doc ? home : null);
+  if (root && !pages.has(key(root.url))) pages.set(key(root.url), root);
+
+  const inlinks = new Map();
+  for (const page of pages.values()) {
+    const seen = new Set();
+    for (const href of page.doc?.links?.internal ?? []) {
+      const target = key(href);
+      // A page linking to itself says nothing about how well linked it is, and
+      // a logo in the header does it on every page of the site.
+      if (target === key(page.url) || seen.has(target)) continue;
+      seen.add(target);
+      inlinks.set(target, (inlinks.get(target) ?? 0) + 1);
+    }
+  }
+
+  const depth = new Map();
+  const from = new Map();
+  if (root) {
+    depth.set(key(root.url), 0);
+    // Breadth first, so the first time a page is reached is by its shortest
+    // path — which is the number worth reporting and the route worth printing.
+    for (let frontier = [key(root.url)]; frontier.length; ) {
+      const next = [];
+      for (const at of frontier) {
+        for (const href of pages.get(at)?.doc?.links?.internal ?? []) {
+          const to = key(href);
+          if (!pages.has(to) || depth.has(to)) continue;
+          depth.set(to, depth.get(at) + 1);
+          from.set(to, at);
+          next.push(to);
+        }
+      }
+      frontier = next;
+    }
+  }
+
+  return {
+    root,
+    depth,
+    from,
+    inlinks,
+    stranded: live.filter((p) => !depth.has(key(p.url))),
+    /** What a finding on this URL can reach: how many pages link to it, and how
+     *  far it is from the homepage. Absent values stay absent rather than
+     *  becoming zero, because "nothing links here" and "this was never
+     *  measured" are different answers. */
+    reachOf(url) {
+      const at = key(url);
+      if (!pages.has(at)) return null;
+      return { inlinks: inlinks.get(at) ?? 0, depth: depth.get(at) ?? null };
+    },
+  };
+}