@goflag/cli 0.2.13 → 0.2.15

package/dist/{chunk-57NVXEJS.js → chunk-KHIN5TNX.js}

@@ -25,6 +25,25 @@ function fingerprint(category, ...parts) {
   return `${category}-${digest}`;
 }
 
+// src/lib/core/canonicalise.ts
+function canonicaliseUrl(href, base, options = {}) {
+  let url;
+  try {
+    url = new URL(href, base);
+  } catch {
+    return null;
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+  url.hash = "";
+  if (!options.preserveTrailingSlash && url.pathname.length > 1 && url.pathname.endsWith("/")) {
+    url.pathname = url.pathname.slice(0, -1);
+  }
+  return url.toString();
+}
+
+// src/lib/core/inspect.ts
+import * as cheerio3 from "cheerio";
+
 // src/lib/core/extract/static.ts
 import * as cheerio from "cheerio";
 
@@ -105,7 +124,9 @@ function decodeHtmlEntities(s) {
 
 // src/lib/core/extract/static.ts
 function extractStatic(html, options) {
-  const $ = cheerio.load(html);
+  return extractStaticFrom(cheerio.load(html), options);
+}
+function extractStaticFrom($, options) {
   const base = resolveBase($, options.baseUrl);
   const raw = extractRaw($);
   const meta = parseGenericMeta(raw, base);
@@ -233,8 +254,8 @@ function parseGenericMeta(raw, base) {
   for (const l of raw.links) {
     const rel = l.rel?.toLowerCase();
     if (rel === "canonical" && l.href) {
-      const resolved = safeResolve(l.href, base);
-      meta.canonical = sourced(resolved, { kind: "link", rel: "canonical" }, l.href);
+      const resolved2 = safeResolve(l.href, base);
+      meta.canonical = sourced(resolved2, { kind: "link", rel: "canonical" }, l.href);
     }
   }
   return meta;
@@ -952,12 +973,98 @@ function countEntries(body) {
   return url + sm;
 }
 
+// src/lib/core/links/extract.ts
+import * as cheerio2 from "cheerio";
+var ANCHOR_SPEC = { selector: "a[href]", attr: "href", source: "a" };
+var ASSET_SPECS = [
+  { selector: "img[src]", attr: "src", source: "img" },
+  { selector: "script[src]", attr: "src", source: "script" },
+  { selector: "link[href]", attr: "href", source: "link" },
+  { selector: "iframe[src]", attr: "src", source: "iframe" }
+];
+function extractLinks(html, options) {
+  return extractLinksFrom(cheerio2.load(html), options);
+}
+function extractLinksFrom($, options) {
+  const base = resolveBase2($, options.baseUrl);
+  const origin = safeOrigin(options.baseUrl);
+  const specs = options.includeAssets ? [ANCHOR_SPEC, ...ASSET_SPECS] : [ANCHOR_SPEC];
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const spec of specs) {
+    $(spec.selector).each((_, el) => {
+      const raw = $(el).attr(spec.attr);
+      if (raw === void 0) return;
+      const rawHref = raw.trim();
+      if (!rawHref) return;
+      const ref = buildRef($, el, rawHref, spec, base, origin);
+      if (!ref) return;
+      const dedupeKey = `${ref.source}|${ref.url}`;
+      if (seen.has(dedupeKey)) return;
+      seen.add(dedupeKey);
+      out.push(ref);
+    });
+  }
+  return out;
+}
+function buildRef($, el, rawHref, spec, base, origin) {
+  const fragment = extractFragment(rawHref);
+  const canonical = canonicaliseUrl(rawHref, base, { preserveTrailingSlash: true });
+  const url = canonical ?? rawHref;
+  const kind = canonical ? classify(canonical, origin) : "external";
+  const ref = {
+    rawHref,
+    url,
+    kind,
+    source: spec.source,
+    rel: spec.source === "a" || spec.source === "link" ? parseRel($(el).attr("rel")) : []
+  };
+  if (spec.source === "a") {
+    const text = $(el).text().trim();
+    if (text) ref.anchorText = text;
+  }
+  if (fragment) ref.fragment = fragment;
+  return ref;
+}
+function extractFragment(rawHref) {
+  const hash = rawHref.indexOf("#");
+  if (hash === -1) return void 0;
+  const frag = rawHref.slice(hash);
+  return frag.length > 1 ? frag : void 0;
+}
+function parseRel(rel) {
+  if (!rel) return [];
+  return rel.split(/\s+/).map((t) => t.trim().toLowerCase()).filter(Boolean);
+}
+function classify(url, origin) {
+  if (!origin) return "external";
+  return safeOrigin(url) === origin ? "internal" : "external";
+}
+function safeOrigin(url) {
+  try {
+    return new URL(url).origin;
+  } catch {
+    return null;
+  }
+}
+function resolveBase2($, pageUrl) {
+  const declared = $("base[href]").first().attr("href")?.trim();
+  if (declared) {
+    try {
+      return new URL(declared, pageUrl).toString();
+    } catch {
+    }
+  }
+  return pageUrl;
+}
+
 // src/lib/core/inspect.ts
 async function inspect(url, options = {}) {
   const {
     probes: enableProbes = true,
     mode = "auto",
     headless: headlessOpts,
+    linkRefs: wantLinkRefs = false,
     ...fetchOptions
   } = options;
   const fetchedAt = (/* @__PURE__ */ new Date()).toISOString();
@@ -971,7 +1078,8 @@ async function inspect(url, options = {}) {
     });
   }
   const { meta: staticMeta, body: staticBody } = await fetchStatic(url, fetchOptions);
-  const staticParsed = extractStatic(staticBody, { baseUrl: staticMeta.finalUrl });
+  const { staticParsed, linkRefs } = parseStatic(staticBody, staticMeta.finalUrl, wantLinkRefs);
+  const withLinkRefs = linkRefs ? { linkRefs } : {};
   if (mode === "static") {
     const probes2 = await runProbes({
       enabled: enableProbes,
@@ -985,7 +1093,8 @@ async function inspect(url, options = {}) {
       fetch: staticMeta,
       extractor: { mode: "static", escalated: false },
       html: { static: staticBody },
-      probes: probes2
+      probes: probes2,
+      ...withLinkRefs
     };
   }
   const heuristic = looksClientRendered(staticParsed);
@@ -1002,7 +1111,8 @@ async function inspect(url, options = {}) {
       fetch: staticMeta,
       extractor: { mode: "static", escalated: false },
       html: { static: staticBody },
-      probes: probes2
+      probes: probes2,
+      ...withLinkRefs
     };
   }
   let renderedBody;
@@ -1034,7 +1144,8 @@ async function inspect(url, options = {}) {
         escalationBlocked: err.message
       },
       html: { static: staticBody },
-      probes: probes2
+      probes: probes2,
+      ...withLinkRefs
     };
   }
   const renderedParsed = extractStatic(renderedBody, { baseUrl: renderedFetch.finalUrl });
@@ -1056,9 +1167,16 @@ async function inspect(url, options = {}) {
     },
     html: { static: staticBody, rendered: renderedBody },
     hydration,
-    probes
+    probes,
+    ...withLinkRefs
   };
 }
+function parseStatic(html, baseUrl, wantLinkRefs) {
+  const $ = cheerio3.load(html);
+  const staticParsed = extractStaticFrom($, { baseUrl });
+  if (!wantLinkRefs) return { staticParsed };
+  return { staticParsed, linkRefs: extractLinksFrom($, { baseUrl }) };
+}
 async function inspectHeadlessOnly(args) {
   const { url, fetchedAt, enableProbes, fetchOptions, headlessOpts } = args;
   const result = await extractHeadless(url, {
@@ -1209,60 +1327,74 @@ async function crawl(options) {
   const recovered = [];
   let truncated = false;
   const seenFinal = /* @__PURE__ */ new Set();
-  while (queue.length > 0) {
+  const inspectItem = async (item) => {
+    try {
+      const page = await inspect(item.url, options.inspectOptions);
+      return { kind: "ok", item, page };
+    } catch (err) {
+      return { kind: "err", item, error: err instanceof Error ? err.message : String(err) };
+    }
+  };
+  const pending = [];
+  const maxPending = concurrency * 4;
+  let active = 0;
+  let requestDone = () => void 0;
+  const start = (item) => {
+    active += 1;
+    pending.push(
+      inspectItem(item).finally(() => {
+        active -= 1;
+        requestDone();
+      })
+    );
+  };
+  while (queue.length > 0 || pending.length > 0) {
     if (options.inspectOptions?.signal?.aborted) break;
-    const wave = queue.splice(0, concurrency);
-    const inspections = wave.map(async (item) => {
-      try {
-        const page = await inspect(item.url, options.inspectOptions);
-        return { kind: "ok", item, page };
-      } catch (err) {
-        return {
-          kind: "err",
-          item,
-          error: err instanceof Error ? err.message : String(err)
-        };
-      }
+    while (active < concurrency && pending.length < maxPending && queue.length > 0) {
+      start(queue.shift());
+    }
+    const oldest = pending[0];
+    const finished = new Promise((resolve3) => {
+      requestDone = resolve3;
     });
-    const results = await Promise.all(inspections);
-    for (const result of results) {
-      if (result.kind === "err") {
-        const attempt = (result.item.attempt ?? 0) + 1;
-        if (attempt < MAX_ATTEMPTS) {
-          queue.push({ ...result.item, attempt });
-          continue;
-        }
-        errors.push({ url: result.item.url, message: result.error });
+    const oldestReady = await Promise.race([oldest.then(() => true), finished.then(() => false)]);
+    if (!oldestReady) continue;
+    const result = await pending.shift();
+    if (result.kind === "err") {
+      const attempt = (result.item.attempt ?? 0) + 1;
+      if (attempt < MAX_ATTEMPTS) {
+        queue.push({ ...result.item, attempt });
         continue;
       }
-      if (result.item.attempt) recovered.push(result.item.url);
-      const finalUrl = canonicaliseUrl(result.page.fetch.finalUrl) ?? result.page.fetch.finalUrl;
-      if (seenFinal.has(finalUrl)) continue;
-      seenFinal.add(finalUrl);
-      pages.push(result.page);
-      if (options.onPage) {
-        options.onPage(result.page, { visited: pages.length, queued: queue.length });
-      }
-      if (pages.length >= maxPages) {
-        truncated = true;
-        break;
-      }
-      if (result.item.depth >= depthLimit) continue;
-      const candidates = extractCandidateLinks(result.page);
-      for (const candidate of candidates) {
-        const normalised = canonicaliseUrl(candidate.href, result.page.fetch.finalUrl);
-        if (!normalised) continue;
-        if (visitedSet.has(normalised)) continue;
-        const parsed = new URL(normalised);
-        if (parsed.origin !== origin) continue;
-        const passesIncludeExclude = filterAllows(parsed.pathname, include, exclude);
-        const isHreflang = candidate.source === "hreflang";
-        if (!passesIncludeExclude && !(followHreflang && isHreflang)) continue;
-        visitedSet.add(normalised);
-        queue.push({ url: normalised, depth: result.item.depth + 1 });
-      }
+      errors.push({ url: result.item.url, message: result.error });
+      continue;
+    }
+    if (result.item.attempt) recovered.push(result.item.url);
+    const finalUrl = canonicaliseUrl(result.page.fetch.finalUrl) ?? result.page.fetch.finalUrl;
+    if (seenFinal.has(finalUrl)) continue;
+    seenFinal.add(finalUrl);
+    pages.push(result.page);
+    if (options.onPage) {
+      options.onPage(result.page, { visited: pages.length, queued: queue.length });
+    }
+    if (pages.length >= maxPages) {
+      truncated = true;
+      break;
+    }
+    if (result.item.depth >= depthLimit) continue;
+    const candidates = extractCandidateLinks(result.page);
+    for (const candidate of candidates) {
+      const normalised = canonicaliseUrl(candidate.href, result.page.fetch.finalUrl);
+      if (!normalised) continue;
+      if (visitedSet.has(normalised)) continue;
+      const parsed = new URL(normalised);
+      if (parsed.origin !== origin) continue;
+      const passesIncludeExclude = filterAllows(parsed.pathname, include, exclude);
+      const isHreflang = candidate.source === "hreflang";
+      if (!passesIncludeExclude && !(followHreflang && isHreflang)) continue;
+      visitedSet.add(normalised);
+      queue.push({ url: normalised, depth: result.item.depth + 1 });
     }
-    if (truncated) break;
   }
   return {
     pages,
@@ -1277,20 +1409,6 @@ function filterAllows(pathname, include, exclude) {
   if (include.length === 0) return true;
   return matchesAny(pathname, include);
 }
-function canonicaliseUrl(href, base, options = {}) {
-  let url;
-  try {
-    url = new URL(href, base);
-  } catch {
-    return null;
-  }
-  if (url.protocol !== "http:" && url.protocol !== "https:") return null;
-  url.hash = "";
-  if (!options.preserveTrailingSlash && url.pathname.length > 1 && url.pathname.endsWith("/")) {
-    url.pathname = url.pathname.slice(0, -1);
-  }
-  return url.toString();
-}
 
 // src/lib/rules/evaluate.ts
 function bandFor(value, bands) {
@@ -1435,14 +1553,14 @@ function normalise(value) {
   return value.trim().toLowerCase().split(/\s+/).sort().join(" ");
 }
 function sameAsset(href, src) {
-  const pathOf2 = (value) => {
+  const pathOf3 = (value) => {
     try {
       return new URL(value, "https://goflag.invalid").pathname;
     } catch {
       return value.split("?")[0] ?? value;
     }
   };
-  return pathOf2(href) === pathOf2(src);
+  return pathOf3(href) === pathOf3(src);
 }
 function hasOpenGraph(ex) {
   return Boolean(
@@ -2765,6 +2883,41 @@ function sortLocales(locales) {
   return [...xDefault, ...rest];
 }
 
+// src/lib/core/probes/not-found.ts
+import { randomBytes } from "crypto";
+async function probeNotFound(origin, options = {}) {
+  const nonce = options.nonce ?? randomBytes(16).toString("hex");
+  const base = `/goflag-probe-${nonce}`;
+  return Promise.all([
+    ask(new URL(base, origin).toString(), "bare", options),
+    ask(new URL(`${base}.txt`, origin).toString(), "dotted", options)
+  ]);
+}
+async function ask(url, shape, options) {
+  const { signal, cleanup } = combineSignals(options.signal, options.timeoutMs ?? 5e3);
+  try {
+    let res = await fetch(url, { method: "HEAD", signal, redirect: "follow" });
+    if (!isNotFound(res.status)) {
+      res = await fetch(url, { signal, redirect: "follow" });
+      await res.body?.cancel().catch(() => void 0);
+    }
+    return {
+      url,
+      shape,
+      status: res.status,
+      finalUrl: res.url || url,
+      contentType: res.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() || void 0
+    };
+  } catch {
+    return { url, shape, status: 0, finalUrl: url };
+  } finally {
+    cleanup();
+  }
+}
+function isNotFound(status) {
+  return status === 404 || status === 410;
+}
+
 // src/lib/core/robots/match.ts
 function robotsAllows(groups, path, userAgent = "*") {
   const group = selectGroup(groups, userAgent);
@@ -2829,6 +2982,180 @@ function escapeRegExp(value) {
   return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
 
+// src/lib/rules/site-shared.ts
+var SAMPLE = 5;
+function sample(locs) {
+  const shown = locs.slice(0, SAMPLE);
+  const rest = locs.length - shown.length;
+  return `${shown.map((l) => `\`${l}\``).join(", ")}${rest > 0 ? `, and ${rest} more` : ""}`;
+}
+function isDead(status) {
+  return status === 0 || status >= 400;
+}
+
+// src/lib/rules/llms-txt.ts
+var TEXT_TYPES = /* @__PURE__ */ new Set(["text/plain", "text/markdown", "text/x-markdown"]);
+function readable(site) {
+  return site.llmsTxt?.found === true && !site.llmsTxt.html;
+}
+function explainedByCatchAll(site, status) {
+  const dotted = site.notFound?.find((probe) => probe.shape === "dotted");
+  if (!dotted) return false;
+  const klass = (code) => Math.floor(code / 100);
+  return (klass(status) === 2 || klass(status) === 5) && klass(dotted.status) === klass(status);
+}
+function sameOriginLinks(site) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const link of site.llmsTxt?.links ?? []) {
+    if (seen.has(link.url) || !sameOrigin2(link.url, site.origin)) continue;
+    seen.add(link.url);
+    out.push({ url: link.url, line: link.line });
+  }
+  return out;
+}
+function sameOrigin2(url, origin) {
+  try {
+    return new URL(url).origin === new URL(origin).origin;
+  } catch {
+    return false;
+  }
+}
+function fileUrl(site) {
+  return site.llmsTxt?.url ?? `${site.origin}/llms.txt`;
+}
+var llmstxtUnreachable = {
+  id: "llmstxt.unreachable",
+  severity: "warning",
+  summary: "`/llms.txt` must answer with the file or with 404, not with a server error",
+  rigor: "guideline",
+  sources: ["lighthouse-llms-txt", "llmstxt"],
+  // A 5xx only. No answer at all is not evidence about the route, and every
+  // 4xx — a 404 above all — is the site saying it has no file.
+  appliesTo: (site) => site.llmsTxt !== void 0 && site.llmsTxt.status >= 500 && !explainedByCatchAll(site, site.llmsTxt.status),
+  check: ({ site, issue }) => issue({
+    pageUrl: fileUrl(site),
+    message: `\`/llms.txt\` answered ${site.llmsTxt.status}. Having no llms.txt is fine \u2014 a 404 says so \u2014 but a server error says something is here and failed, and the agent that asked for it gets nothing it can use. Lighthouse fails its llms.txt audit on exactly this.`,
+    origin: { kind: "computed" }
+  })
+};
+var llmstxtContentType = {
+  id: "llmstxt.content-type",
+  severity: "warning",
+  summary: "`/llms.txt` must be served as text \u2014 `text/plain` or `text/markdown` \u2014 not as a page",
+  rigor: "guideline",
+  sources: ["llmstxt", "ietf-rfc9110"],
+  appliesTo: (site) => {
+    const probe = site.llmsTxt;
+    if (probe?.found !== true) return false;
+    if (probe.html) return !explainedByCatchAll(site, probe.status);
+    return !TEXT_TYPES.has(probe.contentType ?? "");
+  },
+  check: ({ site, issue }) => {
+    const probe = site.llmsTxt;
+    const served = probe.contentType ? `\`${probe.contentType}\`` : "no content type";
+    const message = probe.html ? `\`/llms.txt\` answered ${probe.status} with an HTML page (${served}), not a markdown file. An agent that asks for the site's llms.txt is handed a page instead. If the site has no llms.txt the path should answer 404; if it has one, this is not the route serving it.` : `\`/llms.txt\` is served with ${served}. The file is markdown, read by agents and by people: \`text/plain\` or \`text/markdown\` say so, while anything else tells a client to treat it as something it is not \u2014 \`text/html\` renders it as one run-on paragraph, and a download type does not render it at all.`;
+    return issue({ pageUrl: fileUrl(site), message, origin: { kind: "computed" } });
+  }
+};
+var llmstxtH1Missing = {
+  id: "llmstxt.h1.missing",
+  severity: "warning",
+  summary: "An llms.txt must have an H1 naming the site",
+  rigor: "guideline",
+  sources: ["llmstxt"],
+  appliesTo: (site) => readable(site) && !site.llmsTxt.h1,
+  check: ({ site, issue }) => issue({
+    pageUrl: fileUrl(site),
+    message: "`/llms.txt` has no H1 (`# Name`, with a space after the `#`). The proposal calls it the only required section \u2014 without it the file is not an llms.txt by its own definition, and a reader following the format has no name for what the rest describes.",
+    origin: { kind: "computed" }
+  })
+};
+function listedProbes(site) {
+  const byUrl = site.llmsTxtLinks?.byUrl;
+  if (!byUrl) return [];
+  return sameOriginLinks(site).flatMap(({ url }) => byUrl.get(url) ?? []);
+}
+var llmstxtLinkUnreachable = {
+  id: "llmstxt.link.unreachable",
+  severity: "warning",
+  summary: "Every same-origin URL an llms.txt lists must answer",
+  rigor: "guideline",
+  sources: ["llmstxt"],
+  appliesTo: (site) => readable(site) && listedProbes(site).some((probe) => isDead(probe.status)),
+  check: ({ site, issue }) => {
+    const dead = listedProbes(site).filter((probe) => isDead(probe.status));
+    const unprobed = site.llmsTxtLinks?.unprobed ?? 0;
+    const floor = unprobed > 0 ? ` ${unprobed} further URL${unprobed === 1 ? " was" : "s were"} not checked \u2014 this count is a floor, not a total.` : "";
+    return issue({
+      pageUrl: fileUrl(site),
+      message: `${dead.length} URL${dead.length === 1 ? "" : "s"} \`/llms.txt\` lists ${dead.length === 1 ? "does" : "do"} not answer: ${sample(dead.map((probe) => `${probe.url} (${probe.status === 0 ? "no response" : `HTTP ${probe.status}`})`))}. The file exists to send agents to these pages, so each dead one is a lookup that ends in an error instead of an answer.${floor}`,
+      origin: { kind: "computed" }
+    });
+  }
+};
+var llmstxtLinkBlockedByRobots = {
+  id: "llmstxt.link.blocked-by-robots",
+  severity: "warning",
+  summary: "An llms.txt must not list URLs that robots.txt forbids fetching",
+  rigor: "guideline",
+  sources: ["llmstxt", "ietf-rfc9309", "anthropic-crawlers"],
+  appliesTo: (site) => readable(site) && site.robots?.found === true && robotsAllows(site.robots.groups, "/").allowed,
+  check: ({ site, issue }) => {
+    const robots = site.robots;
+    const file = fileUrl(site);
+    const blocked = [];
+    for (const url of [file, ...sameOriginLinks(site).map((link) => link.url)]) {
+      let path;
+      try {
+        const parsed = new URL(url);
+        path = `${parsed.pathname}${parsed.search}`;
+      } catch {
+        continue;
+      }
+      const decision = robotsAllows(robots.groups, path);
+      if (decision.allowed || !decision.rule) continue;
+      const self = url === file ? ", the file itself" : "";
+      blocked.push(
+        `${url} (line ${decision.rule.line}: Disallow: ${decision.rule.pattern}${self})`
+      );
+    }
+    if (blocked.length === 0) return [];
+    return issue({
+      pageUrl: file,
+      message: `${blocked.length} URL${blocked.length === 1 ? "" : "s"} \`/llms.txt\` points agents to ${blocked.length === 1 ? "is" : "are"} disallowed by \`robots.txt\` for \`User-agent: *\`: ${sample(blocked)}. The file offers them to agents, and an agent that honours robots.txt \u2014 Anthropic documents that Claude-User does \u2014 never fetches them.`,
+      origin: { kind: "computed" }
+    });
+  }
+};
+var LLMS_TXT_RULES = [
+  llmstxtContentType,
+  llmstxtH1Missing,
+  llmstxtLinkBlockedByRobots,
+  llmstxtLinkUnreachable,
+  llmstxtUnreachable
+];
+
+// src/lib/rules/sitemap-inventory.ts
+function sitemapReadInFull(site) {
+  const discovery = site.discovery;
+  if (discovery?.diagnostics.found !== true) return false;
+  if (discovery.truncated || discovery.diagnostics.childSitemapErrors > 0) return false;
+  const read = new Set(discovery.documents.map((doc) => resolved(doc.url)));
+  const base = site.robots?.url ?? `${site.origin}/robots.txt`;
+  return (site.robots?.sitemaps ?? []).every((declared) => {
+    const url = resolved(declared.value, base);
+    return url === void 0 || read.has(url);
+  });
+}
+function resolved(value, base) {
+  try {
+    return new URL(value, base).href;
+  } catch {
+    return void 0;
+  }
+}
+
 // src/lib/rules/site-rules.ts
 function isMultilingual(site) {
   return site.localeAxis.multilingual;
@@ -3037,6 +3364,69 @@ var iconsIcoMissing = {
     });
   }
 };
+var httpNotFound = {
+  id: "http.not-found",
+  severity: "error",
+  summary: "A URL that does not exist must answer 404 or 410",
+  rigor: "vendor-spec",
+  sources: ["ietf-rfc9110", "google-soft-404", "google-http-status"],
+  appliesTo: (site) => (site.notFound ?? []).some((probe) => notFoundFailure(probe.status)),
+  check: ({ site, issue }) => {
+    const probes = site.notFound ?? [];
+    const failed = probes.filter((probe) => notFoundFailure(probe.status));
+    const held = probes.filter((probe) => isNotFound(probe.status));
+    const answers = failed.map((probe) => {
+      const path = pathOf(probe.url);
+      const via = probe.finalUrl !== probe.url ? ` after redirecting to \`${probe.finalUrl}\`` : "";
+      const type = probe.contentType ? ` \`${probe.contentType}\`` : "";
+      return `\`${path}\` answered ${probe.status}${type}${via}`;
+    });
+    const contrast = held.length ? ` (${held.map((probe) => `\`${pathOf(probe.url)}\` answered ${probe.status}`).join(", ")}, so only some unknown paths are affected)` : "";
+    const soft = failed.some((probe) => notFoundFailure(probe.status) === "soft");
+    const crash = failed.some((probe) => notFoundFailure(probe.status) === "error");
+    const consequences = [
+      soft ? "A 2xx for a page that does not exist is a soft 404: every invented URL becomes a page to crawl, and a client asking for a file the site does not have \u2014 `/llms.txt`, `/feed.xml` \u2014 gets a page it may take for that file." : "",
+      crash ? "A 5xx says the server failed, not that the page is missing: every mistyped link becomes a server error, and Google slows its crawl of the whole site in proportion to the URLs that return one." : ""
+    ].filter(Boolean);
+    return issue({
+      pageUrl: site.origin,
+      message: `Asked for a path that cannot exist, the origin did not answer 404 or 410: ${answers.join("; ")}${contrast}. ${consequences.join(" ")}`,
+      origin: { kind: "computed" },
+      fix: {
+        title: "Make the route that catches unknown paths call notFound()",
+        snippet: [
+          "// A dynamic first segment matches everything, `/anything.txt` included,",
+          "// whenever a middleware matcher skips dotted paths. Validate it wherever",
+          "// it is read \u2014 the layout, every page, every generateMetadata:",
+          "const { locale } = await params;",
+          "if (!routing.locales.includes(locale)) notFound();",
+          "//",
+          "// The layout alone is not enough in a production build: the page and its",
+          "// metadata render alongside it, and whichever throws first sets the",
+          "// status \u2014 a 500 from a page that formatted a date in locale `llms.txt`.",
+          "//",
+          "// A catch-all `[...slug]` must do the same for slugs it does not know,",
+          "// and a static host's SPA fallback (`/* /index.html 200`) should be a",
+          "// 404 page instead."
+        ].join("\n"),
+        language: "ts"
+      }
+    });
+  }
+};
+function notFoundFailure(status) {
+  if (status >= 200 && status < 300) return "soft";
+  if (status >= 500) return "error";
+  return void 0;
+}
+function pathOf(url) {
+  try {
+    const parsed = new URL(url);
+    return `${parsed.pathname}${parsed.search}`;
+  } catch {
+    return url;
+  }
+}
 var ROBOTS_BYTE_LIMIT = 500 * 1024;
 var robotstxtUnreachable = {
   id: "robotstxt.unreachable",
@@ -3172,12 +3562,6 @@ var robotsBlocksPage = {
   }
 };
 var CHANGEFREQ = /* @__PURE__ */ new Set(["always", "hourly", "daily", "weekly", "monthly", "yearly", "never"]);
-var SAMPLE = 5;
-function sample(locs) {
-  const shown = locs.slice(0, SAMPLE);
-  const rest = locs.length - shown.length;
-  return `${shown.map((l) => `\`${l}\``).join(", ")}${rest > 0 ? `, and ${rest} more` : ""}`;
-}
 var sitemapMissing = {
   id: "sitemap.missing",
   severity: "warning",
@@ -3494,16 +3878,43 @@ function sameUrlKey(url) {
 }
 function pagesByUrl(site) {
   const byUrl = /* @__PURE__ */ new Map();
-  for (const page of site.pages) {
+  for (const page of [...site.pages, ...site.variants ?? []]) {
     const key = sameUrlKey(page.fetch.finalUrl);
     if (key) byUrl.set(key, page);
   }
   return byUrl;
 }
+var NOINDEX = /* @__PURE__ */ new Set(["noindex", "none"]);
+var VALUED_DIRECTIVES = /* @__PURE__ */ new Set([
+  "max-snippet",
+  "max-image-preview",
+  "max-video-preview",
+  "unavailable_after"
+]);
+function headerDirectiveNames(value) {
+  if (!value) return [];
+  return value.split(",").map((token) => {
+    let directive = token.trim().toLowerCase();
+    const colon = directive.indexOf(":");
+    if (colon !== -1 && !VALUED_DIRECTIVES.has(directive.slice(0, colon).trim())) {
+      directive = directive.slice(colon + 1);
+    }
+    return directive.split(":")[0].trim();
+  });
+}
 function saysNoindex(page) {
-  if (metaRobotsTokens(page).has("noindex")) return true;
-  const header = page.fetch.headers["x-robots-tag"] ?? "";
-  return header.split(",").map((token) => token.trim().toLowerCase()).includes("noindex");
+  const metas = [page.meta.robots?.value, page.meta.googlebot?.value];
+  if (metas.some((content) => (content ?? "").split(",").some(isNoindex))) return true;
+  return headerDirectiveNames(page.fetch.headers["x-robots-tag"]).some(isNoindex);
+}
+function isNoindex(directive) {
+  return NOINDEX.has(directive.trim().toLowerCase());
+}
+function canonicalElsewhere(page, self = sameUrlKey(page.fetch.finalUrl)) {
+  const canonical = page.links.canonical;
+  if (!canonical) return false;
+  const target = sameUrlKey(canonical);
+  return target !== void 0 && target !== self;
 }
 var sitemapEntryBlockedByRobots = {
   id: "sitemap.entry.blocked-by-robots",
@@ -3574,17 +3985,14 @@ var sitemapEntryNonCanonical = {
     for (const entry of site.discovery.urls) {
       const key = sameUrlKey(entry.loc);
       const page = key ? byUrl.get(key) : void 0;
-      const canonical = page?.links.canonical;
-      if (!page || !canonical) continue;
-      const canonicalKey = sameUrlKey(canonical);
-      if (canonicalKey && canonicalKey !== key) {
-        wrong.push(`${entry.loc} \u2192 ${canonical}`);
+      if (page && canonicalElsewhere(page)) {
+        wrong.push(`${entry.loc} \u2192 ${page.links.canonical}`);
       }
     }
     if (wrong.length === 0) return [];
     return issue({
       pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
-      message: `${wrong.length} sitemap entr${wrong.length === 1 ? "y names a page whose canonical" : "ies name pages whose canonicals"} point elsewhere: ${sample(wrong)}. The sitemap is a list of what to index, so it should name the URL the site itself prefers.`,
+      message: `${wrong.length} sitemap entr${wrong.length === 1 ? "y names a page whose canonical points" : "ies name pages whose canonicals point"} elsewhere: ${sample(wrong)}. The sitemap is a list of what to index, so it should name the URL the site itself prefers.`,
       origin: { kind: "computed" }
     });
   }
@@ -3592,26 +4000,96 @@ var sitemapEntryNonCanonical = {
 var sitemapOrphans = {
   id: "sitemap.orphans",
   severity: "warning",
-  summary: "Indexable pages the crawl found should be listed in the sitemap",
+  summary: "Indexable pages the crawl found, and the canonical URLs they name, should be listed in the sitemap",
   rigor: "guideline",
   sources: ["sitemaps-protocol", "google-sitemaps"],
   // One finding with a count and a sample, not one per page — the same shape
   // translation holes take, and for the same reason: forty repeats of one
   // omission is noise, and the omission is a property of the sitemap.
-  appliesTo: (site) => site.discovery?.diagnostics.found === true && site.discovery.urls.length > 0,
+  //
+  // Judged only against an inventory goflag read in full — every sitemap the
+  // site declares, none of them cut short (`./sitemap-inventory.ts`). A page
+  // listed where goflag could not read looks exactly like one the site forgot.
+  // What stopped the read is its own finding where one exists
+  // (`sitemap.index.child-error`); this one waits until the list is whole.
+  appliesTo: (site) => sitemapReadInFull(site) && site.discovery.urls.length > 0,
   check: ({ site, issue }) => {
-    const listed = new Set(
-      site.discovery.urls.map((entry) => sameUrlKey(entry.loc)).filter(Boolean)
-    );
-    const orphans = site.pages.filter((page) => !saysNoindex(page)).map((page) => ({ page, key: sameUrlKey(page.fetch.finalUrl) })).filter(({ key }) => key !== void 0 && !listed.has(key)).map(({ page }) => page.fetch.finalUrl);
-    if (orphans.length === 0) return [];
+    const { listed, fetched, unfollowed } = sitemapReach(site);
+    const orphans = [];
+    const unaudited = /* @__PURE__ */ new Map();
+    for (const page of site.pages) {
+      if (saysNoindex(page)) continue;
+      const key = sameUrlKey(page.fetch.finalUrl);
+      if (key === void 0) continue;
+      if (!canonicalElsewhere(page, key)) {
+        if (!listed.has(key)) orphans.push(page.fetch.finalUrl);
+        continue;
+      }
+      const target = page.links.canonical;
+      const targetKey2 = sameUrlKey(target);
+      if (sameOrigin3(target, site.origin) && !listed.has(targetKey2) && !fetched.has(targetKey2) && !unaudited.has(targetKey2)) {
+        unaudited.set(targetKey2, { url: target, namedBy: page.fetch.finalUrl });
+      }
+    }
+    const named = [...unaudited.values()];
+    const total = orphans.length + named.length;
+    if (total === 0) return [];
+    const said = [];
+    if (orphans.length > 0) {
+      said.push(
+        `${orphans.length} crawled page${orphans.length === 1 ? "" : "s"} ask${orphans.length === 1 ? "s" : ""} to be indexed and ${orphans.length === 1 ? "is" : "are"} absent from the sitemap: ${sample(orphans)}.`
+      );
+    }
+    if (named.length > 0) {
+      said.push(
+        `${named.length} URL${named.length === 1 ? "" : "s"} that crawled pages name as their canonical ${named.length === 1 ? "is" : "are"} absent from ${orphans.length > 0 ? "it too" : "the sitemap"}, and ${named.length === 1 ? "was" : "were"} not audited: ${sample(named.map((t) => `${t.url} (canonical of ${t.namedBy})`))}.`
+      );
+    }
     return issue({
       pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
-      message: `${orphans.length} crawled page${orphans.length === 1 ? "" : "s"} ask${orphans.length === 1 ? "s" : ""} to be indexed and ${orphans.length === 1 ? "is" : "are"} absent from the sitemap: ${sample(orphans)}. A consumer that reads the sitemap rather than following links will never see ${orphans.length === 1 ? "it" : "them"}.`,
+      message: `${said.join(" ")} A consumer that reads the sitemap rather than following links will never see ${total === 1 ? "it" : "them"}.${ceilingNote(unfollowed)}`,
       origin: { kind: "computed" }
     });
   }
 };
+function sameOrigin3(a, b) {
+  try {
+    return new URL(a).origin === new URL(b).origin;
+  } catch {
+    return false;
+  }
+}
+function sitemapReach(site) {
+  const fetched = /* @__PURE__ */ new Set();
+  const landingOf = /* @__PURE__ */ new Map();
+  for (const page of [...site.pages, ...site.variants ?? []]) {
+    const final = sameUrlKey(page.fetch.finalUrl);
+    const requested = sameUrlKey(page.fetch.requestedUrl);
+    if (!final) continue;
+    fetched.add(final);
+    if (requested && requested !== final) landingOf.set(requested, final);
+  }
+  for (const probe of entryProbes(site)) {
+    const from = sameUrlKey(probe.url);
+    const to = probe.redirected ? sameUrlKey(probe.finalUrl) : void 0;
+    if (from && to) landingOf.set(from, to);
+  }
+  const listed = /* @__PURE__ */ new Set();
+  let unfollowed = 0;
+  for (const entry of site.discovery.urls) {
+    const key = sameUrlKey(entry.loc);
+    if (!key) continue;
+    listed.add(key);
+    const landing = landingOf.get(key);
+    if (landing) listed.add(landing);
+    else if (!fetched.has(key) && !site.sitemapEntries?.byUrl.has(entry.loc)) unfollowed += 1;
+  }
+  return { listed, fetched, unfollowed };
+}
+function ceilingNote(unfollowed) {
+  if (unfollowed === 0) return "";
+  return ` ${unfollowed} sitemap entr${unfollowed === 1 ? "y was" : "ies were"} not fetched, and a page one of them redirects to would be counted here \u2014 this count is a ceiling, not a total.`;
+}
 function entryProbes(site) {
   return [...site.sitemapEntries?.byUrl.values() ?? []];
 }
@@ -3652,9 +4130,6 @@ var sitemapEntryRedirects = {
     });
   }
 };
-function isDead(status) {
-  return status === 0 || status >= 400;
-}
 function isRobotsFailure(status) {
   return status === 0 || status >= 500;
 }
@@ -3669,7 +4144,9 @@ function isAbsolute(value) {
 var SITE_RULES = [
   hreflangMissing,
   hreflangClusterIncomplete,
+  httpNotFound,
   iconsIcoMissing,
+  ...LLMS_TXT_RULES,
   robotsBlocksPage,
   robotsBlocksSite,
   robotstxtCrossOrigin,
@@ -3824,7 +4301,7 @@ function suggestedLocales(axis) {
 import { gunzipSync } from "zlib";
 
 // src/lib/core/sitemap/parse.ts
-import * as cheerio2 from "cheerio";
+import * as cheerio4 from "cheerio";
 function alternatesOf(node) {
   const out = [];
   const seen = /* @__PURE__ */ new Set();
@@ -3852,7 +4329,7 @@ function parseSitemap(body) {
   }
   let $;
   try {
-    $ = cheerio2.load(trimmed, { xmlMode: true });
+    $ = cheerio4.load(trimmed, { xmlMode: true });
   } catch {
     return { kind: "unknown", wellFormed: false, urls: [], sitemaps: [] };
   }
@@ -4249,6 +4726,10 @@ function normaliseContentType(raw) {
   const semi = raw.indexOf(";");
   return (semi === -1 ? raw : raw.slice(0, semi)).trim().toLowerCase() || void 0;
 }
+function capText(text, maxBytes) {
+  if (Buffer.byteLength(text, "utf8") <= maxBytes) return text;
+  return Buffer.from(text.slice(0, maxBytes), "utf8").subarray(0, maxBytes).toString("utf8");
+}
 async function readCappedText(res, maxBytes) {
   const reader = res.body?.getReader();
   if (!reader) {
@@ -4381,6 +4862,34 @@ async function checkLink(url, options = {}) {
     durationMs: Date.now() - started
   };
 }
+function checkFromAnswer(url, answer, options = {}) {
+  if (answer.status < 200 || answer.status >= 300) return void 0;
+  const outcome = {
+    status: answer.status,
+    finalUrl: url,
+    redirectChain: [],
+    loop: false,
+    body: capText(answer.body, options.softBodyBytes ?? DEFAULTS2.softBodyBytes),
+    method: "GET"
+  };
+  const softNotFound = isSoftNotFound(outcome);
+  return {
+    url,
+    finalUrl: url,
+    status: outcome.status,
+    verdict: classifyLink({
+      status: outcome.status,
+      redirected: false,
+      softNotFound,
+      loop: false
+    }),
+    method: "GET",
+    redirectChain: [],
+    reason: describeReason(outcome, softNotFound),
+    checkedAt: answer.checkedAt,
+    durationMs: answer.durationMs
+  };
+}
 async function probeWithRetry(url, options) {
   const retries = options.retries ?? DEFAULTS2.retries;
   const sleep = options.sleep ?? defaultSleep;
@@ -4498,7 +5007,7 @@ function schemeOf(url) {
   }
 }
 function defaultSleep(ms) {
-  return new Promise((resolve2) => setTimeout(resolve2, ms));
+  return new Promise((resolve3) => setTimeout(resolve3, ms));
 }
 
 // src/lib/core/sitemap/entries.ts
@@ -4919,6 +5428,86 @@ function isIconType(contentType) {
   return contentType.startsWith("image/") || contentType === "application/ico" || contentType === "application/x-ico";
 }
 
+// src/lib/core/llmstxt/parse.ts
+var H1 = /^ {0,3}#(?:[ \t]+(.*?))?[ \t]*#*[ \t]*$/;
+var FENCE = /^ {0,3}(`{3,}|~{3,})/;
+var LINK = /(!?)\[([^\]]*)\]\(\s*<?([^)\s>]+)>?(?:\s+(?:"[^"]*"|'[^']*'))?\s*\)/g;
+function parseLlmsTxt(body, fileUrl2) {
+  const lines = body.replace(/^\uFEFF/, "").split(/\r\n|\r|\n/);
+  const links = [];
+  let h1;
+  let fence;
+  for (const [index, raw] of lines.entries()) {
+    const line = index + 1;
+    const fenceMatch = FENCE.exec(raw);
+    if (fenceMatch) {
+      const marker = fenceMatch[1];
+      if (fence === void 0) fence = marker;
+      else if (marker[0] === fence[0] && marker.length >= fence.length) fence = void 0;
+      continue;
+    }
+    if (fence !== void 0) continue;
+    if (!h1) {
+      const heading = H1.exec(raw);
+      const text = heading?.[1]?.trim();
+      if (text) h1 = { text, line };
+    }
+    const prose = raw.replace(/`[^`]*`/g, "");
+    for (const match of prose.matchAll(LINK)) {
+      if (match[1] === "!") continue;
+      const url = resolve(match[3], fileUrl2);
+      if (url) links.push({ name: match[2].trim(), url, line });
+    }
+  }
+  return { ...h1 ? { h1 } : {}, links };
+}
+function looksLikeHtml(body) {
+  return /^(?:<!doctype\s+html|<html[\s>])/i.test(body.replace(/^\uFEFF/, "").trimStart());
+}
+function resolve(href, base) {
+  try {
+    const url = new URL(href, base);
+    if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
+    url.hash = "";
+    return url.toString();
+  } catch {
+    return void 0;
+  }
+}
+
+// src/lib/core/probes/llms-txt.ts
+async function probeLlmsTxt(origin, options = {}) {
+  const url = new URL("/llms.txt", origin).toString();
+  const { signal, cleanup } = combineSignals(options.signal, options.timeoutMs ?? 5e3);
+  const nothing = { found: false, html: false, links: [] };
+  try {
+    const res = await fetch(url, { signal, redirect: "follow" });
+    const finalUrl = res.url || url;
+    const contentType = res.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() || void 0;
+    if (!res.ok) {
+      await res.body?.cancel().catch(() => void 0);
+      return { url, status: res.status, finalUrl, contentType, ...nothing };
+    }
+    const body = await res.text();
+    if (looksLikeHtml(body)) {
+      return { url, status: res.status, finalUrl, contentType, found: true, html: true, links: [] };
+    }
+    return {
+      url,
+      status: res.status,
+      finalUrl,
+      contentType,
+      found: true,
+      html: false,
+      ...parseLlmsTxt(body, finalUrl)
+    };
+  } catch {
+    return { url, status: 0, finalUrl: url, ...nothing };
+  } finally {
+    cleanup();
+  }
+}
+
 // src/lib/rules/advisory.ts
 function resolveEvidencePath(extraction, path) {
   let current = extraction;
@@ -5087,7 +5676,12 @@ var hreflangSitemapMismatch = {
   summary: "`<head>` alternates advertise a locale the sitemap does not list",
   prose: "Your `<head>` advertises a translation your sitemap does not list. Is the sitemap missing an entry, or is the alternate pointing at a page you did not mean to publish?",
   why: "The two are derived from one intent by different code paths, so a disagreement means one of them is wrong. No specification says which: the declaration methods are equivalent, and a page can legitimately be cross-linked and kept out of the sitemap.",
-  appliesTo: (site) => site.localeAxis.multilingual && (site.discovery?.urls.length ?? 0) > 0,
+  // A question about what the sitemap *omits*, so it waits for a sitemap read in
+  // full (`./sitemap-inventory.ts`): a translation listed in a child goflag
+  // could not read, or in a second declared sitemap it never opened, would be
+  // asked about as missing — a question with a false premise wastes an agent's
+  // turn as surely as a false finding wastes a human's.
+  appliesTo: (site) => site.localeAxis.multilingual && (site.discovery?.urls.length ?? 0) > 0 && sitemapReadInFull(site),
   ask: (site) => {
     const axis = new Set(site.localeAxis.locales.map((l) => l.toLowerCase()));
     const bySitemap = /* @__PURE__ */ new Map();
@@ -5164,89 +5758,6 @@ function collectSiteAdvisories(site, rules = SITE_PROSE_RULES) {
   return out;
 }
 
-// src/lib/core/links/extract.ts
-import * as cheerio3 from "cheerio";
-var ANCHOR_SPEC = { selector: "a[href]", attr: "href", source: "a" };
-var ASSET_SPECS = [
-  { selector: "img[src]", attr: "src", source: "img" },
-  { selector: "script[src]", attr: "src", source: "script" },
-  { selector: "link[href]", attr: "href", source: "link" },
-  { selector: "iframe[src]", attr: "src", source: "iframe" }
-];
-function extractLinks(html, options) {
-  const $ = cheerio3.load(html);
-  const base = resolveBase2($, options.baseUrl);
-  const origin = safeOrigin(options.baseUrl);
-  const specs = options.includeAssets ? [ANCHOR_SPEC, ...ASSET_SPECS] : [ANCHOR_SPEC];
-  const out = [];
-  const seen = /* @__PURE__ */ new Set();
-  for (const spec of specs) {
-    $(spec.selector).each((_, el) => {
-      const raw = $(el).attr(spec.attr);
-      if (raw === void 0) return;
-      const rawHref = raw.trim();
-      if (!rawHref) return;
-      const ref = buildRef($, el, rawHref, spec, base, origin);
-      if (!ref) return;
-      const dedupeKey = `${ref.source}|${ref.url}`;
-      if (seen.has(dedupeKey)) return;
-      seen.add(dedupeKey);
-      out.push(ref);
-    });
-  }
-  return out;
-}
-function buildRef($, el, rawHref, spec, base, origin) {
-  const fragment = extractFragment(rawHref);
-  const canonical = canonicaliseUrl(rawHref, base, { preserveTrailingSlash: true });
-  const url = canonical ?? rawHref;
-  const kind = canonical ? classify(canonical, origin) : "external";
-  const ref = {
-    rawHref,
-    url,
-    kind,
-    source: spec.source,
-    rel: spec.source === "a" || spec.source === "link" ? parseRel($(el).attr("rel")) : []
-  };
-  if (spec.source === "a") {
-    const text = $(el).text().trim();
-    if (text) ref.anchorText = text;
-  }
-  if (fragment) ref.fragment = fragment;
-  return ref;
-}
-function extractFragment(rawHref) {
-  const hash = rawHref.indexOf("#");
-  if (hash === -1) return void 0;
-  const frag = rawHref.slice(hash);
-  return frag.length > 1 ? frag : void 0;
-}
-function parseRel(rel) {
-  if (!rel) return [];
-  return rel.split(/\s+/).map((t) => t.trim().toLowerCase()).filter(Boolean);
-}
-function classify(url, origin) {
-  if (!origin) return "external";
-  return safeOrigin(url) === origin ? "internal" : "external";
-}
-function safeOrigin(url) {
-  try {
-    return new URL(url).origin;
-  } catch {
-    return null;
-  }
-}
-function resolveBase2($, pageUrl) {
-  const declared = $("base[href]").first().attr("href")?.trim();
-  if (declared) {
-    try {
-      return new URL(declared, pageUrl).toString();
-    } catch {
-    }
-  }
-  return pageUrl;
-}
-
 // src/lib/core/links/types.ts
 function emptyVerdictSummary() {
   return { ok: 0, redirect: 0, broken: 0, blocked: 0, warning: 0, skipped: 0 };
@@ -5278,8 +5789,16 @@ async function runLinkAudit(discovery, options = {}) {
   let pagesScanned = 0;
   let pagesFailed = 0;
   let scanDone = 0;
+  const maxBytes = options.maxScanBytes ?? DEFAULTS3.maxScanBytes;
+  const handedPages = [];
+  const toFetch = [];
+  for (const pageUrl of pageUrls) {
+    const handed = options.pages?.get(pageUrl);
+    if (handed) handedPages.push([pageUrl, handed]);
+    else toFetch.push(pageUrl);
+  }
   await mapWithHostCaps(
-    pageUrls,
+    toFetch,
     {
       concurrency: options.scanConcurrency ?? DEFAULTS3.scanConcurrency,
       perHost: Infinity,
@@ -5291,7 +5810,7 @@ async function runLinkAudit(discovery, options = {}) {
         timeoutMs: options.timeoutMs ?? DEFAULTS3.timeoutMs,
         allowInsecureTls: options.allowInsecureTls,
         userAgent: options.userAgent,
-        maxBytes: options.maxScanBytes ?? DEFAULTS3.maxScanBytes,
+        maxBytes,
         signal: options.signal
       });
       if (res.status === 0 || res.status >= 400 || res.body === void 0) {
@@ -5309,6 +5828,17 @@ async function runLinkAudit(discovery, options = {}) {
     },
     () => "scan"
   );
+  for (const [pageUrl, handed] of handedPages) {
+    if (options.signal?.aborted) break;
+    const refs = handed.links && !options.includeAssets && Buffer.byteLength(handed.html) <= maxBytes ? handed.links : extractLinks(capText(handed.html, maxBytes), {
+      baseUrl: pageUrl,
+      includeAssets: options.includeAssets
+    });
+    for (const ref of refs) occurrences.push({ pageUrl, ref });
+    pagesScanned += 1;
+    scanDone += 1;
+    options.onProgress?.({ phase: "scan", done: scanDone, total: pageUrls.length });
+  }
   const uniqueUrls = [];
   const seen = /* @__PURE__ */ new Set();
   for (const occ of occurrences) {
@@ -5325,8 +5855,20 @@ async function runLinkAudit(discovery, options = {}) {
   }
   const checks = {};
   let checkDone = 0;
+  const toProbe = [];
+  for (const url of uniqueUrls) {
+    const answer = options.answers?.get(url);
+    const check = answer ? checkFromAnswer(url, answer) : void 0;
+    if (!check) {
+      toProbe.push(url);
+      continue;
+    }
+    checks[url] = check;
+    checkDone += 1;
+    options.onProgress?.({ phase: "check", done: checkDone, total: uniqueUrls.length });
+  }
   await mapWithHostCaps(
-    uniqueUrls,
+    toProbe,
     {
       concurrency: options.checkConcurrency ?? DEFAULTS3.checkConcurrency,
       perHost: options.maxPerHost ?? DEFAULTS3.maxPerHost,
@@ -5400,9 +5942,9 @@ function hostOf(url) {
 function mapWithHostCaps(items, opts, worker, hostOf3) {
   const concurrency = Math.max(1, opts.concurrency);
   const perHost = opts.perHost;
-  return new Promise((resolve2) => {
+  return new Promise((resolve3) => {
     if (items.length === 0) {
-      resolve2();
+      resolve3();
       return;
     }
     const pending = items.map((item) => ({ item, host: hostOf3(item) }));
@@ -5412,7 +5954,7 @@ function mapWithHostCaps(items, opts, worker, hostOf3) {
     const finish = () => {
       if (!settled) {
         settled = true;
-        resolve2();
+        resolve3();
       }
     };
     const pump = () => {
@@ -5578,14 +6120,14 @@ async function attachManifests(pages, options) {
   }
 }
 async function probeAssets(pages, options) {
-  const sameOrigin2 = (url) => {
+  const sameOrigin4 = (url) => {
     try {
       return new URL(url).origin === new URL(options.origin).origin;
     } catch {
       return false;
     }
   };
-  const wantedUrl = (url) => /^https?:\/\//i.test(url) && (options.checkExternal || sameOrigin2(url));
+  const wantedUrl = (url) => /^https?:\/\//i.test(url) && (options.checkExternal || sameOrigin4(url));
   const wanted = /* @__PURE__ */ new Set();
   for (const page of pages) {
     for (const image of page.openGraph.images) {
@@ -5639,7 +6181,8 @@ function dropCanonicalDuplicates(pages) {
     if (target === self) return true;
     return !crawled.has(target);
   });
-  return { kept, dropped: pages.length - kept.length };
+  const keptSet = new Set(kept);
+  return { kept, variants: pages.filter((page) => !keptSet.has(page)) };
 }
 async function runAudit(inputUrl, options = {}) {
   const normalized = normalizeInputUrl(inputUrl);
@@ -5673,6 +6216,14 @@ async function runAudit(inputUrl, options = {}) {
     signal: options.signal,
     timeoutMs: options.timeoutMs
   }).catch(() => void 0);
+  const [notFound, llmsTxt] = await Promise.all([
+    probeNotFound(origin, { signal: options.signal, timeoutMs: options.timeoutMs }).catch(
+      () => void 0
+    ),
+    probeLlmsTxt(origin, { signal: options.signal, timeoutMs: options.timeoutMs }).catch(
+      () => void 0
+    )
+  ]);
   const crawlResult = await crawl({
     entryUrl: entry,
     seedUrls: sitemapUrls,
@@ -5683,6 +6234,7 @@ async function runAudit(inputUrl, options = {}) {
     inspectOptions: {
       mode: options.static ? "static" : "auto",
       probes: false,
+      linkRefs: true,
       allowInsecureTls: options.allowInsecureTls,
       timeoutMs: options.timeoutMs,
       signal: options.signal
@@ -5706,7 +6258,8 @@ async function runAudit(inputUrl, options = {}) {
   if (escalationWarning) warnings.push(escalationWarning);
   const okPages = pages.filter((p) => isOkStatus(p.fetch.status));
   const documents = okPages.filter(isHtmlPage);
-  const { kept: htmlPages, dropped: duplicatePages } = dropCanonicalDuplicates(documents);
+  const { kept: htmlPages, variants } = dropCanonicalDuplicates(documents);
+  const duplicatePages = variants.length;
   const unreachablePages = [
     ...pages.filter((p) => !isOkStatus(p.fetch.status)).map((p) => ({
       id: fingerprint("page", routeKey(p.fetch.finalUrl)),
@@ -5822,6 +6375,16 @@ async function runAudit(inputUrl, options = {}) {
     checkExternal,
     timeoutMs: options.timeoutMs,
     maxPages: effectiveMaxPages,
+    pages: new Map(
+      okPages.filter(isHtmlPage).map((page) => [
+        page.fetch.finalUrl,
+        {
+          html: page.html.static,
+          links: page.extractor.escalated ? void 0 : page.linkRefs
+        }
+      ])
+    ),
+    answers: crawlAnswers(okPages),
     signal: options.signal,
     onProgress: (p) => options.onProgress?.({
       phase: p.phase === "scan" ? "scan" : "links",
@@ -5830,18 +6393,23 @@ async function runAudit(inputUrl, options = {}) {
     })
   });
   warnings.push(...linkReport.diagnostics.warnings);
+  const answered = {
+    crawled: new Map(crawlResult.pages.map((page) => [page.fetch.finalUrl, page.fetch.status])),
+    checked: new Map(Object.values(linkReport.checks).map((check) => [check.url, check])),
+    maxProbes: effectiveMaxPages,
+    signal: options.signal,
+    timeoutMs: options.timeoutMs,
+    allowInsecureTls: options.allowInsecureTls
+  };
   const sitemapEntries = discovery ? await probeSitemapEntries(
     discovery.urls.map((entry2) => entry2.loc),
-    {
-      crawled: new Map(
-        crawlResult.pages.map((page) => [page.fetch.finalUrl, page.fetch.status])
-      ),
-      checked: new Map(Object.values(linkReport.checks).map((check) => [check.url, check])),
-      maxProbes: effectiveMaxPages,
-      signal: options.signal,
-      timeoutMs: options.timeoutMs,
-      allowInsecureTls: options.allowInsecureTls
-    }
+    answered
+  ) : void 0;
+  const llmsTxtLinks = llmsTxt?.found && !llmsTxt.html ? await probeSitemapEntries(
+    [...new Set(llmsTxt.links.map((link) => link.url))].filter(
+      (url) => sameOriginAs(url, origin)
+    ),
+    answered
   ) : void 0;
   const siteContext = {
     origin,
@@ -5851,7 +6419,14 @@ async function runAudit(inputUrl, options = {}) {
     discovery,
     robots,
     favicon: favicon2,
+    notFound,
+    llmsTxt,
+    llmsTxtLinks,
     sitemapEntries,
+    // Set aside above, and still what their URLs serve. The sitemap rules that
+    // ask what a listed URL serves need them: a listed variant is exactly what
+    // `sitemap.entry.non-canonical` exists to catch.
+    variants,
     // Same index the matrix was built from, so a rule that groups by route
     // and the grid that reports holes cannot disagree about which URLs are
     // one page.
@@ -6014,6 +6589,28 @@ function countUncrawled(sitemapUrls, pages) {
   for (const url of sitemapUrls) if (!crawled.has(url)) n += 1;
   return n;
 }
+function sameOriginAs(url, origin) {
+  try {
+    return new URL(url).origin === origin;
+  } catch {
+    return false;
+  }
+}
+function crawlAnswers(pages) {
+  const answers = /* @__PURE__ */ new Map();
+  for (const page of pages) {
+    if (!isOkStatus(page.fetch.status) || !isHtmlPage(page)) continue;
+    if (page.extractor.escalated) continue;
+    if (page.fetch.redirectCount > 0 || page.fetch.finalUrl !== page.fetch.requestedUrl) continue;
+    answers.set(page.fetch.requestedUrl, {
+      status: page.fetch.status,
+      body: page.html.static,
+      checkedAt: page.fetchedAt,
+      durationMs: page.fetch.durationMs
+    });
+  }
+  return answers;
+}
 function syntheticDiscovery(origin, baseUrl, pages, truncated) {
   return {
     origin,
@@ -6436,7 +7033,7 @@ function hostOf2(value) {
     return value;
   }
 }
-function pathOf(value) {
+function pathOf2(value) {
   try {
     const { pathname, search } = new URL(value);
     return `${pathname}${search}`;
@@ -6465,7 +7062,7 @@ function originLabel(origin) {
       return "computed";
   }
 }
-function resolve(...facts) {
+function resolve2(...facts) {
   for (const fact of facts) {
     if (fact && fact.value.trim() !== "") {
       return { value: fact.value, from: originLabel(fact.origin) };
@@ -6509,13 +7106,13 @@ function cardOf(extraction) {
   return {
     url,
     host: hostOf2(url),
-    path: pathOf(url),
-    ogTitle: resolve(og.title, doc.title),
-    ogDescription: resolve(og.description, meta.description),
-    documentTitle: resolve(doc.title),
-    metaDescription: resolve(meta.description),
-    siteName: resolve(og.siteName),
-    xTitle: resolve(twitter.title, og.title, doc.title),
+    path: pathOf2(url),
+    ogTitle: resolve2(og.title, doc.title),
+    ogDescription: resolve2(og.description, meta.description),
+    documentTitle: resolve2(doc.title),
+    metaDescription: resolve2(meta.description),
+    siteName: resolve2(og.siteName),
+    xTitle: resolve2(twitter.title, og.title, doc.title),
     image: imageOf(extraction),
     icon: safeUrl(extraction.links.icons[0]?.href),
     themeColor: meta.themeColor?.value,
@@ -7018,7 +7615,7 @@ function renderPreview(report, options = {}) {
   }
   const empty = report.extractions === void 0 ? `This report carries no extractions, so there is nothing to draw. Pass <code>extractions: true</code> to <code>runAudit</code>, or use <code>goflag preview &lt;url&gt;</code>, which does.` : `The crawl reached no HTML page it could read, so there is nothing to draw. Every page was unreachable, not HTML, or a declared duplicate of another.`;
   const rows = extractions.map((extraction, index) => {
-    const path = pathOf(extraction.http.finalUrl);
+    const path = pathOf2(extraction.http.finalUrl);
     return {
       index,
       locale: splitRoute(path).locale,
@@ -7030,7 +7627,7 @@ function renderPreview(report, options = {}) {
   });
   const byRoute = /* @__PURE__ */ new Map();
   extractions.forEach((extraction, index) => {
-    const route = splitRoute(pathOf(extraction.http.finalUrl)).route;
+    const route = splitRoute(pathOf2(extraction.http.finalUrl)).route;
     byRoute.set(route, [...byRoute.get(route) ?? [], rows[index]]);
   });
   const body = extractions.length === 0 ? `<section class="page"><p class="none">${empty}</p></section>` : extractions.map(
@@ -7038,13 +7635,13 @@ function renderPreview(report, options = {}) {
       extraction,
       issuesByPage.get(extraction.http.finalUrl) ?? [],
       index,
-      byRoute.get(splitRoute(pathOf(extraction.http.finalUrl)).route) ?? []
+      byRoute.get(splitRoute(pathOf2(extraction.http.finalUrl)).route) ?? []
     )
   ).join("\n");
   const nav = extractions.length > 1 ? `<nav class="nav">
   <p class="navnote">${extractions.length} pages, by path. A folder folds; the \u2197 beside one opens the page at that path itself.</p>
   ${routeList(
-    buildRouteTree(extractions.map((extraction) => pathOf(extraction.http.finalUrl))),
+    buildRouteTree(extractions.map((extraction) => pathOf2(extraction.http.finalUrl))),
     extractions.length <= TREE_OPEN_MAX,
     "tree"
   )}
@@ -7362,4 +7959,4 @@ export {
   summarize,
   Logger
 };
-//# sourceMappingURL=chunk-57NVXEJS.js.map
+//# sourceMappingURL=chunk-KHIN5TNX.js.map