@nurkamol/seo-audit 1.31.0

package/src/http.mjs

@@ -0,0 +1,228 @@
+// Fetching, with the two things a crawler actually needs: a concurrency limit
+// so a small site is not hammered, and a cache so the same URL is never
+// fetched twice across checks.
+
+const DEFAULT_UA = 'seo-audit (+https://github.com/nurkamol/seo-audit)';
+
+export class Fetcher {
+  /** @param {{concurrency?: number, timeout?: number}} opts */
+  constructor({ concurrency = 6, timeout = 20000, userAgent = DEFAULT_UA } = {}) {
+    this.timeout = timeout;
+    this.userAgent = userAgent;
+    /** Consecutive timeouts. Some hosts accept the TLS handshake and then
+     *  never answer — Cloudflare's bot management does this to clients whose
+     *  TLS fingerprint is not a browser. Retrying that is 20 seconds of
+     *  nothing, per attempt, per URL. */
+    this.timeouts = 0;
+    this.reachable = false;
+    /** Set when a host answers 429. Nothing starts before it, and the
+     *  concurrency comes down with it — retrying a rate limit at the speed
+     *  that caused it just spends the budget again. */
+    this.quietUntil = 0;
+    this.rateLimited = 0;
+    this.cache = new Map();
+    this.queue = [];
+    this.active = 0;
+    this.concurrency = concurrency;
+    this.count = 0;
+  }
+
+  /** Wait out a rate limit, if one is in force. */
+  async #quiet() {
+    const wait = this.quietUntil - Date.now();
+    if (wait > 0) await new Promise((r) => setTimeout(r, wait));
+  }
+
+  /** A 429 is the server asking for a slower crawl, and it is asking about the
+   *  whole run rather than about one URL. So the pause is global, and the
+   *  concurrency halves and stays down: a short run has no time to earn back
+   *  the trust, and finishing slowly beats finishing wrong.
+   *
+   *  Retry-After is honoured where it is sent and capped where it is absurd —
+   *  a store asking for an hour is asking for a different tool. Shopify sends
+   *  none at all, which is why there is a default. */
+  #backOff(res) {
+    this.rateLimited++;
+    const asked = Number.parseInt(res.headers?.get?.('retry-after') ?? '', 10);
+    // Without a Retry-After the wait doubles to a ceiling of eight seconds.
+    // Thirty would be defensible per request and ruinous across two hundred of
+    // them: the concurrency coming down is what actually gets a run through a
+    // rate limit, and the pause only has to be long enough to let it.
+    const seconds =
+      Number.isFinite(asked) && asked > 0 ? Math.min(asked, 30) : Math.min(2 ** Math.min(this.rateLimited, 3), 8);
+    this.quietUntil = Math.max(this.quietUntil, Date.now() + seconds * 1000);
+    this.concurrency = Math.max(1, Math.floor(this.concurrency / 2));
+  }
+
+  /** Run `fn` when a slot frees up. */
+  #schedule(fn) {
+    return new Promise((resolve, reject) => {
+      const run = async () => {
+        this.active++;
+        try {
+          resolve(await fn());
+        } catch (err) {
+          reject(err);
+        } finally {
+          this.active--;
+          const next = this.queue.shift();
+          if (next) next();
+        }
+      };
+      if (this.active < this.concurrency) run();
+      else this.queue.push(run);
+    });
+  }
+
+  /**
+   * GET a URL. Redirects are NOT followed — a redirect is a finding, not a
+   * detour, and following it silently is how "page with redirect" problems go
+   * unnoticed.
+   * @returns {Promise<{url: string, status: number, ok: boolean, headers: Headers,
+   *                    body: string, location: string|null, ms: number, error?: string}>}
+   */
+  async get(url, { method = 'GET', retries = 2 } = {}) {
+    const key = `${method} ${url}`;
+    if (this.cache.has(key)) return this.cache.get(key);
+
+    const attempt = async () => {
+      await this.#quiet();
+      const started = Date.now();
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), this.timeout);
+      try {
+        const res = await fetch(url, {
+          method,
+          redirect: 'manual',
+          signal: controller.signal,
+          headers: { 'User-Agent': this.userAgent, Accept: '*/*' },
+        });
+        const type = res.headers.get('content-type') ?? '';
+        // Only read a body worth parsing; a 40MB video would stall the run.
+        const body = method === 'GET' && /text|json|xml/.test(type) ? await res.text() : '';
+        this.count++;
+        this.timeouts = 0;
+        this.reachable = true;
+        return {
+          url,
+          status: res.status,
+          ok: res.status >= 200 && res.status < 300,
+          headers: res.headers,
+          body,
+          location: res.headers.get('location'),
+          ms: Date.now() - started,
+        };
+      } catch (err) {
+        return {
+          url,
+          status: 0,
+          ok: false,
+          headers: new Headers(),
+          body: '',
+          location: null,
+          ms: Date.now() - started,
+          error: err.name === 'AbortError' ? 'timed out' : err.message,
+          // A refused connection or an unknown host is an answer, not a blip:
+          // retrying cannot change it, and doing so makes a dead site slow to
+          // report instead of fast.
+          permanent: /ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ERR_INVALID_URL/.test(
+            `${err.code ?? ''} ${err.cause?.code ?? ''} ${err.message}`,
+          ),
+        };
+      } finally {
+        clearTimeout(timer);
+      }
+    };
+
+    // One blip in a 200-page crawl should not be reported as a broken page.
+    // Transport failures, 5xx and 429 are retried; a 404 is an answer.
+    //
+    // 429 was not, until a Shopify store answered it to 70 of 200 pages at the
+    // default concurrency and every one was reported as a page that did not
+    // load. The pages were fine. Spaced eight seconds apart the same URLs
+    // answered 200, on the tool's own user agent — it was the crawl's speed,
+    // and reporting the site for it is the worst kind of false positive,
+    // because it looks exactly like a site that is broken.
+    const promise = this.#schedule(async () => {
+      let last;
+      for (let i = 0; i <= retries; i++) {
+        // Once a host has timed out repeatedly and never once answered, stop
+        // paying 20 seconds a go to confirm it. Reported as unreachable.
+        if (this.timeouts >= 3 && !this.reachable) {
+          return {
+            url, status: 0, ok: false, headers: new Headers(), body: '',
+            location: null, ms: 0, error: 'host is not answering', permanent: true,
+          };
+        }
+        last = await attempt();
+        if (last.error === 'timed out') this.timeouts++;
+        if (last.permanent) return last;
+        if (last.status === 429) {
+          this.#backOff(last);
+          // A host that has said no twenty times is not going to say yes
+          // because it was asked again. Let each page report the rate limit
+          // and let the run finish, rather than spending an hour proving it.
+          if (this.rateLimited > 20) return last;
+          continue;
+        }
+        if (last.status !== 0 && last.status < 500) return last;
+        if (i < retries) await new Promise((r) => setTimeout(r, 300 * (i + 1)));
+      }
+      return last;
+    });
+
+    this.cache.set(key, promise);
+    return promise;
+  }
+
+  /**
+   * Wait until a URL serves the same bytes several times in a row.
+   *
+   * A CDN rolls a deploy out unevenly: for a minute or two one edge answers
+   * with the new page and another with the old, and a crawl during that window
+   * produces a snapshot that is wrong in a way nobody can reproduce later.
+   */
+  async settle(url, seconds) {
+    const deadline = Date.now() + seconds * 1000;
+    const wanted = 3;
+    let previous = null;
+    let same = 0;
+    while (Date.now() < deadline) {
+      this.cache.delete(`GET ${url}`);
+      const res = await this.get(url, { retries: 0 });
+      const fingerprint = `${res.status}:${res.body.length}:${res.headers.get('etag') ?? ''}`;
+      same = fingerprint === previous ? same + 1 : 0;
+      previous = fingerprint;
+      if (same >= wanted - 1) return true;
+      await new Promise((r) => setTimeout(r, 3000));
+    }
+    return false;
+  }
+
+  /** Follow a chain by hand so the number of hops can be reported. */
+  async chain(url, max = 5) {
+    const hops = [];
+    let current = url;
+    for (let i = 0; i < max; i++) {
+      const res = await this.get(current);
+      hops.push({ url: current, status: res.status });
+      if (res.status < 300 || res.status >= 400 || !res.location) return { hops, final: res };
+      current = new URL(res.location, current).toString();
+    }
+    return { hops, final: await this.get(current) };
+  }
+}
+
+/** Run tasks with a cap, preserving input order in the results. */
+export async function mapLimit(items, limit, fn) {
+  const results = new Array(items.length);
+  let cursor = 0;
+  const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
+    while (cursor < items.length) {
+      const i = cursor++;
+      results[i] = await fn(items[i], i);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}

package/src/options.mjs

@@ -0,0 +1,77 @@
+// Every flag, and whether the window can reach it.
+//
+// The command line grew thirty-two flags and the macOS window reached ten of
+// them, and nothing anywhere said whether that was a decision or an oversight.
+// It was both, in different places, and there was no way to tell which from
+// outside — the same failure this project refuses in its reports, where a
+// missing finding reads exactly like a passing one.
+//
+// So: one table, and a test that fails when a flag is added without an answer.
+// `app: true` means the window reaches it. A **string** means it does not, and
+// the string is the reason — "not yet" is a perfectly good reason, written
+// down, and it stops being invisible.
+//
+// This is not a parser. `bin/seo-audit.mjs` still owns argument handling and
+// `src/audit.mjs` still owns defaults; duplicating either here would create the
+// second source of truth this file exists to prevent.
+
+/**
+ * `flag`   what the command line calls it
+ * `query`  the /stream parameter the window sends on a run, when it sends one
+ * `via`    how the window reaches it when it is not a run parameter
+ * `app`    true, or the reason it is not reachable
+ */
+export const OPTIONS = [
+  // --- what a run does ----------------------------------------------------
+  { flag: '--limit', query: 'limit', app: true },
+  { flag: '--concurrency', query: 'concurrency', app: true },
+  { flag: '--check-external', query: 'external', app: true },
+  { flag: '--sitemap', query: 'sitemap', app: true },
+  { flag: '--browser', query: 'browser', app: true },
+  { flag: '--os', query: 'os', app: true },
+  { flag: '--user-agent', query: 'userAgent', app: true },
+  { flag: '--write-sitemap', query: 'sitemap-out', app: true, via: 'the Export menu' },
+  { flag: '--ignore', query: 'ignore', app: true, via: 'right-clicking a finding, and the Settings list' },
+  { flag: '--psi', query: 'psi', app: true, via: 'Settings → Performance' },
+  { flag: '--psi-sample', query: 'psi-sample', app: true, via: 'Settings → Performance' },
+  { flag: '--psi-strategy', query: 'psi-strategy', app: true, via: 'Settings → Performance' },
+  { flag: '--since', query: null, app: 'not yet — it needs a date picker and a sense of when the last run was, which the window has in the library and does not offer yet' },
+  { flag: '--exclude', query: null, app: 'not yet — a list of patterns needs somewhere to live in Settings, and one text field would be worse than nothing' },
+
+  // --- reached another way ------------------------------------------------
+  { flag: '--dry-run', query: null, app: true, via: 'the Preview button' },
+  { flag: '--baseline', query: null, app: true, via: 'the Compare menu, over /diff' },
+  { flag: '--json', query: null, app: true, via: 'the Export menu' },
+  { flag: '--csv', query: null, app: true, via: 'the Export menu' },
+  { flag: '--md', query: null, app: true, via: 'the Export menu' },
+  { flag: '--html', query: null, app: true, via: 'the Export menu' },
+
+  // --- deliberately not in a window ---------------------------------------
+  { flag: '--help', query: null, app: 'a window has no command line to explain' },
+  { flag: '--version', query: null, app: 'the sidebar shows it, and offers every other one' },
+  { flag: '--quiet', query: null, app: 'the crawl log is on screen while it runs' },
+  { flag: '--verbose', query: null, app: 'the crawl log is on screen while it runs' },
+  { flag: '--serve', query: null, app: 'the window is what --serve serves' },
+  { flag: '--fail-on', query: null, app: 'a window has no exit code for a build to read' },
+  { flag: '--update-baseline', query: null, app: 'a baseline is a file a repository commits' },
+  { flag: '--config', query: null, app: 'a config file is a file a repository commits' },
+  { flag: '--redirects', query: null, app: 'a migration map is a file a repository commits' },
+
+
+  // --- not yet, and that is a decision rather than an oversight ------------
+  { flag: '--search-console', query: null, app: 'not yet — needs an OAuth client, and has never run against the live API' },
+  { flag: '--against', query: null, app: 'not yet — the window compares two kept runs instead of two live deployments' },
+  { flag: '--compare-as', query: null, app: 'not yet — it fetches a sample of pages a second time, and the window has no control for spending that' },
+  { flag: '--compare-sample', query: null, app: 'not yet — see --compare-as' },
+  { flag: '--settle', query: null, app: 'not yet — waiting out a deploy that is still rolling out is a CI shape, not something somebody watches a window do' },
+];
+
+/** The parameters a client should send on a run, for one that wants to build
+ *  its own controls rather than hard-code this list. */
+export const runParameters = () =>
+  OPTIONS.filter((o) => o.app === true && o.query).map(({ flag, query }) => ({ flag, query }));
+
+/** Everything the window does not reach, and why. Served so the answer is
+ *  discoverable rather than only being in a source file. */
+export const notInApp = () =>
+  OPTIONS.filter((o) => o.app !== true).map(({ flag, app }) => ({ flag, reason: app }));

package/src/parse.mjs

@@ -0,0 +1,347 @@
+// Minimal HTML extraction.
+//
+// No parser dependency on purpose: this tool should run anywhere with `npx`
+// and nothing installed. The regexes below are deliberately narrow — they read
+// well-formed markup produced by a static site generator, which is what this
+// audits. Anything ambiguous is reported as unknown rather than guessed.
+
+
+import { fingerprint } from './dupes.mjs';
+const stripTags = (s) => s.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+
+const decode = (s) =>
+  s
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    // The typographic entities a CMS emits into link text and headings. Left
+    // undecoded, "here's their page &raquo;" normalises to a phrase with the
+    // word "raquo" in it, which is nobody's anchor text.
+    .replace(/&(l|r)aquo;/g, (_, side) => (side === 'l' ? '«' : '»'))
+    .replace(/&(m|n)dash;/g, (_, kind) => (kind === 'm' ? '—' : '–'))
+    .replace(/&hellip;/g, '…')
+    .replace(/&(l|r)squo;/g, "'")
+    .replace(/&(l|r)dquo;/g, '"')
+    .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)));
+
+/** Attribute value from a tag string: attr(`<img alt="x">`, 'alt') → 'x'
+ *
+ *  The lookbehind matters, and has been widened twice by real sites:
+ *
+ *  - `\b` treats the hyphen in `data-src` as a boundary, so a plain
+ *    word-boundary match reads a lazy-loading site's `data-src` as its `src`
+ *    and reports images that are not there.
+ *  - `:` and `[` introduce a framework binding — `:src`, `v-bind:src`,
+ *    `x-bind:src`, `[src]` — whose value is a JavaScript expression, not a URL.
+ *    allbirds.com binds `:src="(cardRefs['7205190238288']?.selectedImage…)"`,
+ *    and reading those as real sources reported twenty-four of its images as
+ *    404s that do not exist. */
+export function attr(tag, name) {
+  const start = `(?<![-:\\[\\w])${name}`;
+  const m =
+    tag.match(new RegExp(`${start}\\s*=\\s*"([^"]*)"`, 'i')) ??
+    tag.match(new RegExp(`${start}\\s*=\\s*'([^']*)'`, 'i')) ??
+    // Unquoted, which HTML permits and minifiers produce: smashingmagazine.com
+    // ships `<meta name=viewport content="…">`, and reading only quoted values
+    // reported nine of its pages as having no viewport at all.
+    tag.match(new RegExp(`${start}\\s*=\\s*([^\\s"'\`=<>]+)`, 'i'));
+  if (m) return decode(m[1]);
+  // Bare boolean attribute (`<img alt>`) — present, with an empty value.
+  return new RegExp(`${start}(?=[\\s/>])`, 'i').test(tag) ? '' : null;
+}
+
+/** Blank out attribute values that contain whole tags.
+ *
+ *  Markup inside an attribute value is a code sample, not part of the page.
+ *  astro.build stores an entire Astro component in a `data-code` attribute for
+ *  its copy button, and the `<img src={product.imageUrl}>` in that string was
+ *  read as a real image with no alt — an error, on a site that has no such
+ *  problem.
+ *
+ *  Deliberately narrow: the value must contain something shaped like a tag, so
+ *  `title="a < b"` and `content="Tea & Cake"` are untouched. */
+export const stripMarkupInAttributes = (html) =>
+  html.replace(/="[^"]*<[a-z][^">]*>[^"]*"/gi, '=""');
+
+// Japanese, Chinese and Thai do not put spaces between words, so splitting on
+// whitespace counts an entire paragraph as one. The Japanese translation of a
+// React docs page counted 177 against the English original's 411 — the same
+// page, the same content — and was reported as thin.
+//
+// Counted at roughly two characters to the word, the usual working equivalence.
+// It is an approximation, and deliberately a generous one: over-counting keeps
+// a real page quiet, while under-counting calls it thin, and only one of those
+// is a finding somebody has to argue with.
+const UNSPACED_SCRIPT =
+  /[぀-ヿ㐀-䶿一-鿿豈-﫿฀-๿]/g;
+
+export function countWords(text) {
+  if (!text) return 0;
+  const unspaced = text.match(UNSPACED_SCRIPT)?.length ?? 0;
+  const spaced = text
+    .replace(UNSPACED_SCRIPT, ' ')
+    .split(/\s+/)
+    .filter((w) => /[\p{L}\p{N}]/u.test(w)).length;
+  return spaced + Math.round(unspaced / 2);
+}
+
+export function parseHtml(rawHtml, pageUrl) {
+  const html = stripMarkupInAttributes(rawHtml);
+
+  // Elements are read from markup with <script> and <style> contents removed.
+  // A script that builds HTML by concatenation — `'<li><a href="' + a.url + '">'`
+  // — is code, not links on the page, and smashingmagazine.com's offline-article
+  // list had nine of those reported as links to a page that does not exist.
+  //
+  // JSON-LD is read from `html` instead, because it lives inside a <script>.
+  const markup = html
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ');
+
+  const head = (markup.match(/<head[\s\S]*?<\/head>/i) ?? [''])[0];
+  const mainRegion = (markup.match(/<main[\s\S]*?<\/main>/i) ?? [''])[0]
+    || (markup.match(/<article[\s\S]*?<\/article>/i) ?? [''])[0]
+    || null;
+  const main = mainRegion || markup;
+
+  const metas = [...markup.matchAll(/<meta\b[^>]*>/gi)].map((m) => m[0]);
+  const metaBy = (key, value) => {
+    const tag = metas.find((t) => (attr(t, key) ?? '').toLowerCase() === value);
+    return tag ? attr(tag, 'content') : null;
+  };
+
+  const links = [...markup.matchAll(/<link\b[^>]*>/gi)].map((m) => m[0]);
+  const linkRel = (rel) => links.filter((t) => (attr(t, 'rel') ?? '').toLowerCase() === rel);
+
+  // The three rel values Google reads a favicon from, and `rel` is a token
+  // list, so the legacy `shortcut icon` is matched by the `icon` in it without
+  // needing a rule of its own.
+  const ICON_RELS = new Set(['icon', 'apple-touch-icon', 'apple-touch-icon-precomposed']);
+
+  const abs = (href) => {
+    try {
+      return new URL(href, pageUrl).toString();
+    } catch {
+      return null;
+    }
+  };
+
+  const anchors = [...markup.matchAll(/<a\b[^>]*>/gi)].map((m) => m[0]);
+  const mainAnchors = [...main.matchAll(/<a\b[^>]*>/gi)].map((m) => m[0]);
+  const hrefs = (list) =>
+    list
+      .map((t) => attr(t, 'href'))
+      .filter((h) => h && !/^(#|mailto:|tel:|javascript:|data:)/i.test(h))
+      .map(abs)
+      .filter(Boolean);
+
+  const origin = new URL(pageUrl).origin;
+  const internal = (list) => hrefs(list).filter((h) => h.startsWith(origin));
+
+  // Anchors paired with the words attached to them. Google reads those words as
+  // a description of the destination — they are the one signal a page gets from
+  // outside itself — and until now they were parsed and thrown away.
+  //
+  // The name is resolved the way a browser resolves an accessible name, in
+  // order, because each of these is a real way to label a link and reporting
+  // any of them as unlabelled would be wrong:
+  //
+  //   the text inside → an image's alt → aria-label → the anchor's own title
+  //
+  // aria-labelledby points at another element by id. It is not followed here —
+  // that means reading the rest of the document — and its mere presence counts
+  // as named, since the alternative is calling a labelled link unlabelled.
+  //
+  // A missing </a> makes the match run to the next one, which produces text
+  // where there was none. That direction is safe: it can only silence this,
+  // never invent it.
+  const namedAnchors = [...markup.matchAll(/<a\b([^>]*)>([\s\S]*?)<\/a>/gi)].map((m) => {
+    const tag = `<a${m[1]}>`;
+    const inner = m[2];
+    const img = inner.match(/<img\b[^>]*>/i)?.[0];
+    const svgTitle = inner.match(/<title\b[^>]*>([\s\S]*?)<\/title>/i)?.[1];
+    const name =
+      decode(stripTags(inner)) ||
+      (img && attr(img, 'alt')) ||
+      attr(tag, 'aria-label') ||
+      attr(tag, 'title') ||
+      (svgTitle && decode(stripTags(svgTitle))) ||
+      (img && attr(img, 'title')) ||
+      // A framework binding is a label the author supplied and this cannot
+      // read — `:alt="item.title"`, `[ariaLabel]="…"`. The same trap that made
+      // img-alt report twenty-four of allbirds.com's images as missing alt.
+      (img && /[:[]alt\b/i.test(img) ? '…' : '') ||
+      (/[:[](attr\.)?aria-?label\b/i.test(tag) ? '…' : '') ||
+      // Labelled by something elsewhere in the document, or by a child that
+      // labels itself. Not resolved, only believed.
+      (attr(tag, 'aria-labelledby') !== null || /aria-label(ledby)?=/i.test(inner) ? '…' : '') ||
+      '';
+    return { tag, href: attr(tag, 'href'), name: name.slice(0, 300) };
+  });
+
+  // Internal only, self-links dropped: a page linking to itself says nothing
+  // about anywhere, and a logo in the header does it on every page of the site.
+  const anchorTexts = namedAnchors
+    .filter((a) => a.href && !/^(#|mailto:|tel:|javascript:|data:)/i.test(a.href))
+    .map((a) => ({ ...a, href: abs(a.href) }))
+    .filter((a) => a.href?.startsWith(origin))
+    .map((a) => ({ href: a.href.split('#')[0], name: a.name }))
+    .filter((a) => a.href.replace(/\/$/, '') !== pageUrl.split('#')[0].replace(/\/$/, ''));
+
+  const jsonld = [...html.matchAll(/<script\b[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)]
+    .map((m) => {
+      try {
+        return { ok: true, data: JSON.parse(m[1]) };
+      } catch (err) {
+        return { ok: false, error: err.message };
+      }
+    });
+
+  const images = [...markup.matchAll(/<img\b[^>]*>/gi)].map((m) => {
+    const tag = m[0];
+    return {
+      tag,
+      src: attr(tag, 'src'),
+      alt: attr(tag, 'alt'),
+      width: attr(tag, 'width'),
+      height: attr(tag, 'height'),
+      srcset: attr(tag, 'srcset'),
+      loading: attr(tag, 'loading'),
+      fetchpriority: attr(tag, 'fetchpriority'),
+      role: attr(tag, 'role'),
+      // Captured for the two things it can contradict, never for its absence:
+      // an image with no title has nothing wrong with it.
+      title: attr(tag, 'title'),
+      // `:alt="item.title"` is alt text the framework fills in on render. The
+      // value cannot be read from here, but the author plainly provided one,
+      // and calling that a missing alt is guessing wrong at error level.
+      altBound: /[:[]alt\b/i.test(tag),
+      // Inside a <picture> the sibling <source> may carry the srcset instead.
+      inPicture: false,
+    };
+  });
+
+  const pictures = [...markup.matchAll(/<picture[\s\S]*?<\/picture>/gi)].map((m) => m[0]);
+  for (const img of images) {
+    if (img.src && pictures.some((p) => p.includes(img.src))) img.inPicture = true;
+  }
+
+  const headings = (level) =>
+    [...markup.matchAll(new RegExp(`<h${level}\\b[^>]*>([\\s\\S]*?)</h${level}>`, 'gi'))].map((m) =>
+      stripTags(m[1]),
+    );
+
+  // Heading levels in document order, so a skipped level is visible — read
+  // from <main> only. The footer's column headings are furniture repeated on
+  // every page, not part of this page's outline, and counting them reports a
+  // jump on exactly the pages whose content happens to have no h2.
+  const headingLevels = [...main.matchAll(/<h([1-6])\b/gi)].map((m) => Number(m[1]));
+
+  const bodyText = stripTags(
+    main
+      .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+      .replace(/<style[\s\S]*?<\/style>/gi, ' '),
+  );
+
+  return {
+    title: (markup.match(/<title[^>]*>([\s\S]*?)<\/title>/i) ?? [null, null])[1]?.trim(),
+    description: metaBy('name', 'description'),
+    robots: metaBy('name', 'robots'),
+    // <meta http-equiv="refresh" content="0;url=…"> — a redirect that is not
+    // one, and the only kind this tool can see in the markup.
+    refresh: metaBy('http-equiv', 'refresh'),
+    viewport: metaBy('name', 'viewport'),
+    lang: attr((markup.match(/<html\b[^>]*>/i) ?? [''])[0], 'lang'),
+    canonical: linkRel('canonical').map((t) => abs(attr(t, 'href'))).filter(Boolean),
+    hreflang: linkRel('alternate')
+      .filter((t) => attr(t, 'hreflang'))
+      .map((t) => ({ lang: attr(t, 'hreflang'), href: abs(attr(t, 'href')) })),
+    og: Object.fromEntries(
+      metas
+        .filter((t) => (attr(t, 'property') ?? '').startsWith('og:'))
+        .map((t) => [attr(t, 'property'), attr(t, 'content')]),
+    ),
+    twitter: Object.fromEntries(
+      metas
+        .filter((t) => (attr(t, 'name') ?? '').startsWith('twitter:'))
+        .map((t) => [attr(t, 'name'), attr(t, 'content')]),
+    ),
+    h1: headings(1),
+    h2: headings(2),
+    headingLevels,
+    charset:
+      metaBy('charset', undefined) ??
+      (metas.some((t) => attr(t, 'charset') !== null)
+        ? attr(metas.find((t) => attr(t, 'charset') !== null), 'charset')
+        : /charset=/i.test(head)
+          ? 'declared'
+          : null),
+    images,
+    // Declared favicons, in the order a search engine would prefer them: the
+    // plain `icon` first, then the iOS ones. Google looks for these on the
+    // home page and draws the result beside every listing the site owns.
+    icons: links
+      .map((tag) => ({
+        rel: (attr(tag, 'rel') ?? '').toLowerCase().split(/\s+/).filter(Boolean),
+        href: abs(attr(tag, 'href')),
+      }))
+      .filter((icon) => icon.href && icon.rel.some((r) => ICON_RELS.has(r)))
+      .sort((a, b) => Number(b.rel.includes('icon')) - Number(a.rel.includes('icon')))
+      .map((icon) => icon.href)
+      .filter((href, i, all) => all.indexOf(href) === i),
+    jsonld,
+    links: {
+      internal: [...new Set(internal(anchors))],
+      inMain: [...new Set(internal(mainAnchors))],
+      external: [...new Set(hrefs(anchors).filter((h) => !h.startsWith(origin)))],
+      // Internal links the page tells Google not to follow. `rel` carries a
+      // space-separated list, so nofollow travels with noopener and friends.
+      //
+      // Fragments are stripped and self-links dropped: WordPress marks its
+      // comment-reply links rel="nofollow" pointing at #respond on the page
+      // they are already on, and every article on a WordPress site would report
+      // a withheld path that leads nowhere new.
+      // Every internal link with the words attached to it — see above.
+      anchorTexts,
+      nofollowInternal: [
+        ...new Set(
+          internal(anchors.filter((t) => /(^|\s)nofollow(\s|$)/i.test(attr(t, 'rel') ?? '')))
+            .map((h) => h.split('#')[0])
+            .filter((h) => h.replace(/\/$/, '') !== pageUrl.split('#')[0].replace(/\/$/, '')),
+        ),
+      ],
+    },
+    words: countWords(bodyText),
+    // A sketch of the content, for finding pages that are the same page again.
+    // Only when the page marked its content region: without `<main>` or
+    // `<article>` the text above is the whole document, navigation and footer
+    // included, and every page of a small site would look like every other.
+    // Saying "not compared" is the honest answer; guessing is not.
+    fingerprint: mainRegion ? fingerprint(bodyText) : null,
+  };
+}
+
+/** URLs from a sitemap or sitemap index. Returns {urls, sitemaps, entries}.
+ *
+ *  `entries` pairs each <loc> with its own <lastmod>, read from inside the
+ *  <url> block so a date cannot drift onto a neighbouring URL. It is additional
+ *  rather than a replacement: `urls` stays a plain list of strings, because
+ *  every caller wants exactly that and changing it would ripple through
+ *  discovery for no gain. */
+export function parseSitemap(xml) {
+  const locs = [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)].map((m) => decode(m[1]));
+  const isIndex = /<sitemapindex/i.test(xml);
+  if (isIndex) return { urls: [], sitemaps: locs, entries: [] };
+
+  const entries = [...xml.matchAll(/<url\b[^>]*>([\s\S]*?)<\/url>/gi)]
+    .map((m) => ({
+      loc: decode(m[1].match(/<loc>\s*([^<\s]+)\s*<\/loc>/i)?.[1] ?? ''),
+      lastmod: m[1].match(/<lastmod>\s*([^<\s]+)\s*<\/lastmod>/i)?.[1] ?? null,
+    }))
+    .filter((entry) => entry.loc);
+
+  return { urls: locs, sitemaps: [], entries };
+}