@goflag/cli 0.2.12 → 0.2.14

package/dist/{chunk-QKXUKHXB.js → chunk-YQPL67HX.js}

@@ -233,8 +233,8 @@ function parseGenericMeta(raw, base) {
   for (const l of raw.links) {
     const rel = l.rel?.toLowerCase();
     if (rel === "canonical" && l.href) {
-      const resolved = safeResolve(l.href, base);
-      meta.canonical = sourced(resolved, { kind: "link", rel: "canonical" }, l.href);
+      const resolved2 = safeResolve(l.href, base);
+      meta.canonical = sourced(resolved2, { kind: "link", rel: "canonical" }, l.href);
     }
   }
   return meta;
@@ -1435,14 +1435,14 @@ function normalise(value) {
   return value.trim().toLowerCase().split(/\s+/).sort().join(" ");
 }
 function sameAsset(href, src) {
-  const pathOf2 = (value) => {
+  const pathOf3 = (value) => {
     try {
       return new URL(value, "https://goflag.invalid").pathname;
     } catch {
       return value.split("?")[0] ?? value;
     }
   };
-  return pathOf2(href) === pathOf2(src);
+  return pathOf3(href) === pathOf3(src);
 }
 function hasOpenGraph(ex) {
   return Boolean(
@@ -2765,6 +2765,41 @@ function sortLocales(locales) {
   return [...xDefault, ...rest];
 }
 
+// src/lib/core/probes/not-found.ts
+import { randomBytes } from "crypto";
+async function probeNotFound(origin, options = {}) {
+  const nonce = options.nonce ?? randomBytes(16).toString("hex");
+  const base = `/goflag-probe-${nonce}`;
+  return Promise.all([
+    ask(new URL(base, origin).toString(), "bare", options),
+    ask(new URL(`${base}.txt`, origin).toString(), "dotted", options)
+  ]);
+}
+async function ask(url, shape, options) {
+  const { signal, cleanup } = combineSignals(options.signal, options.timeoutMs ?? 5e3);
+  try {
+    let res = await fetch(url, { method: "HEAD", signal, redirect: "follow" });
+    if (!isNotFound(res.status)) {
+      res = await fetch(url, { signal, redirect: "follow" });
+      await res.body?.cancel().catch(() => void 0);
+    }
+    return {
+      url,
+      shape,
+      status: res.status,
+      finalUrl: res.url || url,
+      contentType: res.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() || void 0
+    };
+  } catch {
+    return { url, shape, status: 0, finalUrl: url };
+  } finally {
+    cleanup();
+  }
+}
+function isNotFound(status) {
+  return status === 404 || status === 410;
+}
+
 // src/lib/core/robots/match.ts
 function robotsAllows(groups, path, userAgent = "*") {
   const group = selectGroup(groups, userAgent);
@@ -2829,6 +2864,180 @@ function escapeRegExp(value) {
   return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
 
+// src/lib/rules/site-shared.ts
+var SAMPLE = 5;
+function sample(locs) {
+  const shown = locs.slice(0, SAMPLE);
+  const rest = locs.length - shown.length;
+  return `${shown.map((l) => `\`${l}\``).join(", ")}${rest > 0 ? `, and ${rest} more` : ""}`;
+}
+function isDead(status) {
+  return status === 0 || status >= 400;
+}
+
+// src/lib/rules/llms-txt.ts
+var TEXT_TYPES = /* @__PURE__ */ new Set(["text/plain", "text/markdown", "text/x-markdown"]);
+function readable(site) {
+  return site.llmsTxt?.found === true && !site.llmsTxt.html;
+}
+function explainedByCatchAll(site, status) {
+  const dotted = site.notFound?.find((probe) => probe.shape === "dotted");
+  if (!dotted) return false;
+  const klass = (code) => Math.floor(code / 100);
+  return (klass(status) === 2 || klass(status) === 5) && klass(dotted.status) === klass(status);
+}
+function sameOriginLinks(site) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const link of site.llmsTxt?.links ?? []) {
+    if (seen.has(link.url) || !sameOrigin2(link.url, site.origin)) continue;
+    seen.add(link.url);
+    out.push({ url: link.url, line: link.line });
+  }
+  return out;
+}
+function sameOrigin2(url, origin) {
+  try {
+    return new URL(url).origin === new URL(origin).origin;
+  } catch {
+    return false;
+  }
+}
+function fileUrl(site) {
+  return site.llmsTxt?.url ?? `${site.origin}/llms.txt`;
+}
+var llmstxtUnreachable = {
+  id: "llmstxt.unreachable",
+  severity: "warning",
+  summary: "`/llms.txt` must answer with the file or with 404, not with a server error",
+  rigor: "guideline",
+  sources: ["lighthouse-llms-txt", "llmstxt"],
+  // A 5xx only. No answer at all is not evidence about the route, and every
+  // 4xx — a 404 above all — is the site saying it has no file.
+  appliesTo: (site) => site.llmsTxt !== void 0 && site.llmsTxt.status >= 500 && !explainedByCatchAll(site, site.llmsTxt.status),
+  check: ({ site, issue }) => issue({
+    pageUrl: fileUrl(site),
+    message: `\`/llms.txt\` answered ${site.llmsTxt.status}. Having no llms.txt is fine \u2014 a 404 says so \u2014 but a server error says something is here and failed, and the agent that asked for it gets nothing it can use. Lighthouse fails its llms.txt audit on exactly this.`,
+    origin: { kind: "computed" }
+  })
+};
+var llmstxtContentType = {
+  id: "llmstxt.content-type",
+  severity: "warning",
+  summary: "`/llms.txt` must be served as text \u2014 `text/plain` or `text/markdown` \u2014 not as a page",
+  rigor: "guideline",
+  sources: ["llmstxt", "ietf-rfc9110"],
+  appliesTo: (site) => {
+    const probe = site.llmsTxt;
+    if (probe?.found !== true) return false;
+    if (probe.html) return !explainedByCatchAll(site, probe.status);
+    return !TEXT_TYPES.has(probe.contentType ?? "");
+  },
+  check: ({ site, issue }) => {
+    const probe = site.llmsTxt;
+    const served = probe.contentType ? `\`${probe.contentType}\`` : "no content type";
+    const message = probe.html ? `\`/llms.txt\` answered ${probe.status} with an HTML page (${served}), not a markdown file. An agent that asks for the site's llms.txt is handed a page instead. If the site has no llms.txt the path should answer 404; if it has one, this is not the route serving it.` : `\`/llms.txt\` is served with ${served}. The file is markdown, read by agents and by people: \`text/plain\` or \`text/markdown\` say so, while anything else tells a client to treat it as something it is not \u2014 \`text/html\` renders it as one run-on paragraph, and a download type does not render it at all.`;
+    return issue({ pageUrl: fileUrl(site), message, origin: { kind: "computed" } });
+  }
+};
+var llmstxtH1Missing = {
+  id: "llmstxt.h1.missing",
+  severity: "warning",
+  summary: "An llms.txt must have an H1 naming the site",
+  rigor: "guideline",
+  sources: ["llmstxt"],
+  appliesTo: (site) => readable(site) && !site.llmsTxt.h1,
+  check: ({ site, issue }) => issue({
+    pageUrl: fileUrl(site),
+    message: "`/llms.txt` has no H1 (`# Name`, with a space after the `#`). The proposal calls it the only required section \u2014 without it the file is not an llms.txt by its own definition, and a reader following the format has no name for what the rest describes.",
+    origin: { kind: "computed" }
+  })
+};
+function listedProbes(site) {
+  const byUrl = site.llmsTxtLinks?.byUrl;
+  if (!byUrl) return [];
+  return sameOriginLinks(site).flatMap(({ url }) => byUrl.get(url) ?? []);
+}
+var llmstxtLinkUnreachable = {
+  id: "llmstxt.link.unreachable",
+  severity: "warning",
+  summary: "Every same-origin URL an llms.txt lists must answer",
+  rigor: "guideline",
+  sources: ["llmstxt"],
+  appliesTo: (site) => readable(site) && listedProbes(site).some((probe) => isDead(probe.status)),
+  check: ({ site, issue }) => {
+    const dead = listedProbes(site).filter((probe) => isDead(probe.status));
+    const unprobed = site.llmsTxtLinks?.unprobed ?? 0;
+    const floor = unprobed > 0 ? ` ${unprobed} further URL${unprobed === 1 ? " was" : "s were"} not checked \u2014 this count is a floor, not a total.` : "";
+    return issue({
+      pageUrl: fileUrl(site),
+      message: `${dead.length} URL${dead.length === 1 ? "" : "s"} \`/llms.txt\` lists ${dead.length === 1 ? "does" : "do"} not answer: ${sample(dead.map((probe) => `${probe.url} (${probe.status === 0 ? "no response" : `HTTP ${probe.status}`})`))}. The file exists to send agents to these pages, so each dead one is a lookup that ends in an error instead of an answer.${floor}`,
+      origin: { kind: "computed" }
+    });
+  }
+};
+var llmstxtLinkBlockedByRobots = {
+  id: "llmstxt.link.blocked-by-robots",
+  severity: "warning",
+  summary: "An llms.txt must not list URLs that robots.txt forbids fetching",
+  rigor: "guideline",
+  sources: ["llmstxt", "ietf-rfc9309", "anthropic-crawlers"],
+  appliesTo: (site) => readable(site) && site.robots?.found === true && robotsAllows(site.robots.groups, "/").allowed,
+  check: ({ site, issue }) => {
+    const robots = site.robots;
+    const file = fileUrl(site);
+    const blocked = [];
+    for (const url of [file, ...sameOriginLinks(site).map((link) => link.url)]) {
+      let path;
+      try {
+        const parsed = new URL(url);
+        path = `${parsed.pathname}${parsed.search}`;
+      } catch {
+        continue;
+      }
+      const decision = robotsAllows(robots.groups, path);
+      if (decision.allowed || !decision.rule) continue;
+      const self = url === file ? ", the file itself" : "";
+      blocked.push(
+        `${url} (line ${decision.rule.line}: Disallow: ${decision.rule.pattern}${self})`
+      );
+    }
+    if (blocked.length === 0) return [];
+    return issue({
+      pageUrl: file,
+      message: `${blocked.length} URL${blocked.length === 1 ? "" : "s"} \`/llms.txt\` points agents to ${blocked.length === 1 ? "is" : "are"} disallowed by \`robots.txt\` for \`User-agent: *\`: ${sample(blocked)}. The file offers them to agents, and an agent that honours robots.txt \u2014 Anthropic documents that Claude-User does \u2014 never fetches them.`,
+      origin: { kind: "computed" }
+    });
+  }
+};
+var LLMS_TXT_RULES = [
+  llmstxtContentType,
+  llmstxtH1Missing,
+  llmstxtLinkBlockedByRobots,
+  llmstxtLinkUnreachable,
+  llmstxtUnreachable
+];
+
+// src/lib/rules/sitemap-inventory.ts
+function sitemapReadInFull(site) {
+  const discovery = site.discovery;
+  if (discovery?.diagnostics.found !== true) return false;
+  if (discovery.truncated || discovery.diagnostics.childSitemapErrors > 0) return false;
+  const read = new Set(discovery.documents.map((doc) => resolved(doc.url)));
+  const base = site.robots?.url ?? `${site.origin}/robots.txt`;
+  return (site.robots?.sitemaps ?? []).every((declared) => {
+    const url = resolved(declared.value, base);
+    return url === void 0 || read.has(url);
+  });
+}
+function resolved(value, base) {
+  try {
+    return new URL(value, base).href;
+  } catch {
+    return void 0;
+  }
+}
+
 // src/lib/rules/site-rules.ts
 function isMultilingual(site) {
   return site.localeAxis.multilingual;
@@ -3037,6 +3246,69 @@ var iconsIcoMissing = {
     });
   }
 };
+var httpNotFound = {
+  id: "http.not-found",
+  severity: "error",
+  summary: "A URL that does not exist must answer 404 or 410",
+  rigor: "vendor-spec",
+  sources: ["ietf-rfc9110", "google-soft-404", "google-http-status"],
+  appliesTo: (site) => (site.notFound ?? []).some((probe) => notFoundFailure(probe.status)),
+  check: ({ site, issue }) => {
+    const probes = site.notFound ?? [];
+    const failed = probes.filter((probe) => notFoundFailure(probe.status));
+    const held = probes.filter((probe) => isNotFound(probe.status));
+    const answers = failed.map((probe) => {
+      const path = pathOf(probe.url);
+      const via = probe.finalUrl !== probe.url ? ` after redirecting to \`${probe.finalUrl}\`` : "";
+      const type = probe.contentType ? ` \`${probe.contentType}\`` : "";
+      return `\`${path}\` answered ${probe.status}${type}${via}`;
+    });
+    const contrast = held.length ? ` (${held.map((probe) => `\`${pathOf(probe.url)}\` answered ${probe.status}`).join(", ")}, so only some unknown paths are affected)` : "";
+    const soft = failed.some((probe) => notFoundFailure(probe.status) === "soft");
+    const crash = failed.some((probe) => notFoundFailure(probe.status) === "error");
+    const consequences = [
+      soft ? "A 2xx for a page that does not exist is a soft 404: every invented URL becomes a page to crawl, and a client asking for a file the site does not have \u2014 `/llms.txt`, `/feed.xml` \u2014 gets a page it may take for that file." : "",
+      crash ? "A 5xx says the server failed, not that the page is missing: every mistyped link becomes a server error, and Google slows its crawl of the whole site in proportion to the URLs that return one." : ""
+    ].filter(Boolean);
+    return issue({
+      pageUrl: site.origin,
+      message: `Asked for a path that cannot exist, the origin did not answer 404 or 410: ${answers.join("; ")}${contrast}. ${consequences.join(" ")}`,
+      origin: { kind: "computed" },
+      fix: {
+        title: "Make the route that catches unknown paths call notFound()",
+        snippet: [
+          "// A dynamic first segment matches everything, `/anything.txt` included,",
+          "// whenever a middleware matcher skips dotted paths. Validate it wherever",
+          "// it is read \u2014 the layout, every page, every generateMetadata:",
+          "const { locale } = await params;",
+          "if (!routing.locales.includes(locale)) notFound();",
+          "//",
+          "// The layout alone is not enough in a production build: the page and its",
+          "// metadata render alongside it, and whichever throws first sets the",
+          "// status \u2014 a 500 from a page that formatted a date in locale `llms.txt`.",
+          "//",
+          "// A catch-all `[...slug]` must do the same for slugs it does not know,",
+          "// and a static host's SPA fallback (`/* /index.html 200`) should be a",
+          "// 404 page instead."
+        ].join("\n"),
+        language: "ts"
+      }
+    });
+  }
+};
+function notFoundFailure(status) {
+  if (status >= 200 && status < 300) return "soft";
+  if (status >= 500) return "error";
+  return void 0;
+}
+function pathOf(url) {
+  try {
+    const parsed = new URL(url);
+    return `${parsed.pathname}${parsed.search}`;
+  } catch {
+    return url;
+  }
+}
 var ROBOTS_BYTE_LIMIT = 500 * 1024;
 var robotstxtUnreachable = {
   id: "robotstxt.unreachable",
@@ -3172,12 +3444,6 @@ var robotsBlocksPage = {
   }
 };
 var CHANGEFREQ = /* @__PURE__ */ new Set(["always", "hourly", "daily", "weekly", "monthly", "yearly", "never"]);
-var SAMPLE = 5;
-function sample(locs) {
-  const shown = locs.slice(0, SAMPLE);
-  const rest = locs.length - shown.length;
-  return `${shown.map((l) => `\`${l}\``).join(", ")}${rest > 0 ? `, and ${rest} more` : ""}`;
-}
 var sitemapMissing = {
   id: "sitemap.missing",
   severity: "warning",
@@ -3494,16 +3760,43 @@ function sameUrlKey(url) {
 }
 function pagesByUrl(site) {
   const byUrl = /* @__PURE__ */ new Map();
-  for (const page of site.pages) {
+  for (const page of [...site.pages, ...site.variants ?? []]) {
     const key = sameUrlKey(page.fetch.finalUrl);
     if (key) byUrl.set(key, page);
   }
   return byUrl;
 }
+var NOINDEX = /* @__PURE__ */ new Set(["noindex", "none"]);
+var VALUED_DIRECTIVES = /* @__PURE__ */ new Set([
+  "max-snippet",
+  "max-image-preview",
+  "max-video-preview",
+  "unavailable_after"
+]);
+function headerDirectiveNames(value) {
+  if (!value) return [];
+  return value.split(",").map((token) => {
+    let directive = token.trim().toLowerCase();
+    const colon = directive.indexOf(":");
+    if (colon !== -1 && !VALUED_DIRECTIVES.has(directive.slice(0, colon).trim())) {
+      directive = directive.slice(colon + 1);
+    }
+    return directive.split(":")[0].trim();
+  });
+}
 function saysNoindex(page) {
-  if (metaRobotsTokens(page).has("noindex")) return true;
-  const header = page.fetch.headers["x-robots-tag"] ?? "";
-  return header.split(",").map((token) => token.trim().toLowerCase()).includes("noindex");
+  const metas = [page.meta.robots?.value, page.meta.googlebot?.value];
+  if (metas.some((content) => (content ?? "").split(",").some(isNoindex))) return true;
+  return headerDirectiveNames(page.fetch.headers["x-robots-tag"]).some(isNoindex);
+}
+function isNoindex(directive) {
+  return NOINDEX.has(directive.trim().toLowerCase());
+}
+function canonicalElsewhere(page, self = sameUrlKey(page.fetch.finalUrl)) {
+  const canonical = page.links.canonical;
+  if (!canonical) return false;
+  const target = sameUrlKey(canonical);
+  return target !== void 0 && target !== self;
 }
 var sitemapEntryBlockedByRobots = {
   id: "sitemap.entry.blocked-by-robots",
@@ -3574,17 +3867,14 @@ var sitemapEntryNonCanonical = {
     for (const entry of site.discovery.urls) {
       const key = sameUrlKey(entry.loc);
       const page = key ? byUrl.get(key) : void 0;
-      const canonical = page?.links.canonical;
-      if (!page || !canonical) continue;
-      const canonicalKey = sameUrlKey(canonical);
-      if (canonicalKey && canonicalKey !== key) {
-        wrong.push(`${entry.loc} \u2192 ${canonical}`);
+      if (page && canonicalElsewhere(page)) {
+        wrong.push(`${entry.loc} \u2192 ${page.links.canonical}`);
       }
     }
     if (wrong.length === 0) return [];
     return issue({
       pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
-      message: `${wrong.length} sitemap entr${wrong.length === 1 ? "y names a page whose canonical" : "ies name pages whose canonicals"} point elsewhere: ${sample(wrong)}. The sitemap is a list of what to index, so it should name the URL the site itself prefers.`,
+      message: `${wrong.length} sitemap entr${wrong.length === 1 ? "y names a page whose canonical points" : "ies name pages whose canonicals point"} elsewhere: ${sample(wrong)}. The sitemap is a list of what to index, so it should name the URL the site itself prefers.`,
       origin: { kind: "computed" }
     });
   }
@@ -3592,26 +3882,96 @@ var sitemapEntryNonCanonical = {
 var sitemapOrphans = {
   id: "sitemap.orphans",
   severity: "warning",
-  summary: "Indexable pages the crawl found should be listed in the sitemap",
+  summary: "Indexable pages the crawl found, and the canonical URLs they name, should be listed in the sitemap",
   rigor: "guideline",
   sources: ["sitemaps-protocol", "google-sitemaps"],
   // One finding with a count and a sample, not one per page — the same shape
   // translation holes take, and for the same reason: forty repeats of one
   // omission is noise, and the omission is a property of the sitemap.
-  appliesTo: (site) => site.discovery?.diagnostics.found === true && site.discovery.urls.length > 0,
+  //
+  // Judged only against an inventory goflag read in full — every sitemap the
+  // site declares, none of them cut short (`./sitemap-inventory.ts`). A page
+  // listed where goflag could not read looks exactly like one the site forgot.
+  // What stopped the read is its own finding where one exists
+  // (`sitemap.index.child-error`); this one waits until the list is whole.
+  appliesTo: (site) => sitemapReadInFull(site) && site.discovery.urls.length > 0,
   check: ({ site, issue }) => {
-    const listed = new Set(
-      site.discovery.urls.map((entry) => sameUrlKey(entry.loc)).filter(Boolean)
-    );
-    const orphans = site.pages.filter((page) => !saysNoindex(page)).map((page) => ({ page, key: sameUrlKey(page.fetch.finalUrl) })).filter(({ key }) => key !== void 0 && !listed.has(key)).map(({ page }) => page.fetch.finalUrl);
-    if (orphans.length === 0) return [];
+    const { listed, fetched, unfollowed } = sitemapReach(site);
+    const orphans = [];
+    const unaudited = /* @__PURE__ */ new Map();
+    for (const page of site.pages) {
+      if (saysNoindex(page)) continue;
+      const key = sameUrlKey(page.fetch.finalUrl);
+      if (key === void 0) continue;
+      if (!canonicalElsewhere(page, key)) {
+        if (!listed.has(key)) orphans.push(page.fetch.finalUrl);
+        continue;
+      }
+      const target = page.links.canonical;
+      const targetKey2 = sameUrlKey(target);
+      if (sameOrigin3(target, site.origin) && !listed.has(targetKey2) && !fetched.has(targetKey2) && !unaudited.has(targetKey2)) {
+        unaudited.set(targetKey2, { url: target, namedBy: page.fetch.finalUrl });
+      }
+    }
+    const named = [...unaudited.values()];
+    const total = orphans.length + named.length;
+    if (total === 0) return [];
+    const said = [];
+    if (orphans.length > 0) {
+      said.push(
+        `${orphans.length} crawled page${orphans.length === 1 ? "" : "s"} ask${orphans.length === 1 ? "s" : ""} to be indexed and ${orphans.length === 1 ? "is" : "are"} absent from the sitemap: ${sample(orphans)}.`
+      );
+    }
+    if (named.length > 0) {
+      said.push(
+        `${named.length} URL${named.length === 1 ? "" : "s"} that crawled pages name as their canonical ${named.length === 1 ? "is" : "are"} absent from ${orphans.length > 0 ? "it too" : "the sitemap"}, and ${named.length === 1 ? "was" : "were"} not audited: ${sample(named.map((t) => `${t.url} (canonical of ${t.namedBy})`))}.`
+      );
+    }
     return issue({
       pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
-      message: `${orphans.length} crawled page${orphans.length === 1 ? "" : "s"} ask${orphans.length === 1 ? "s" : ""} to be indexed and ${orphans.length === 1 ? "is" : "are"} absent from the sitemap: ${sample(orphans)}. A consumer that reads the sitemap rather than following links will never see ${orphans.length === 1 ? "it" : "them"}.`,
+      message: `${said.join(" ")} A consumer that reads the sitemap rather than following links will never see ${total === 1 ? "it" : "them"}.${ceilingNote(unfollowed)}`,
       origin: { kind: "computed" }
     });
   }
 };
+function sameOrigin3(a, b) {
+  try {
+    return new URL(a).origin === new URL(b).origin;
+  } catch {
+    return false;
+  }
+}
+function sitemapReach(site) {
+  const fetched = /* @__PURE__ */ new Set();
+  const landingOf = /* @__PURE__ */ new Map();
+  for (const page of [...site.pages, ...site.variants ?? []]) {
+    const final = sameUrlKey(page.fetch.finalUrl);
+    const requested = sameUrlKey(page.fetch.requestedUrl);
+    if (!final) continue;
+    fetched.add(final);
+    if (requested && requested !== final) landingOf.set(requested, final);
+  }
+  for (const probe of entryProbes(site)) {
+    const from = sameUrlKey(probe.url);
+    const to = probe.redirected ? sameUrlKey(probe.finalUrl) : void 0;
+    if (from && to) landingOf.set(from, to);
+  }
+  const listed = /* @__PURE__ */ new Set();
+  let unfollowed = 0;
+  for (const entry of site.discovery.urls) {
+    const key = sameUrlKey(entry.loc);
+    if (!key) continue;
+    listed.add(key);
+    const landing = landingOf.get(key);
+    if (landing) listed.add(landing);
+    else if (!fetched.has(key) && !site.sitemapEntries?.byUrl.has(entry.loc)) unfollowed += 1;
+  }
+  return { listed, fetched, unfollowed };
+}
+function ceilingNote(unfollowed) {
+  if (unfollowed === 0) return "";
+  return ` ${unfollowed} sitemap entr${unfollowed === 1 ? "y was" : "ies were"} not fetched, and a page one of them redirects to would be counted here \u2014 this count is a ceiling, not a total.`;
+}
 function entryProbes(site) {
   return [...site.sitemapEntries?.byUrl.values() ?? []];
 }
@@ -3652,9 +4012,6 @@ var sitemapEntryRedirects = {
     });
   }
 };
-function isDead(status) {
-  return status === 0 || status >= 400;
-}
 function isRobotsFailure(status) {
   return status === 0 || status >= 500;
 }
@@ -3669,7 +4026,9 @@ function isAbsolute(value) {
 var SITE_RULES = [
   hreflangMissing,
   hreflangClusterIncomplete,
+  httpNotFound,
   iconsIcoMissing,
+  ...LLMS_TXT_RULES,
   robotsBlocksPage,
   robotsBlocksSite,
   robotstxtCrossOrigin,
@@ -4498,7 +4857,7 @@ function schemeOf(url) {
   }
 }
 function defaultSleep(ms) {
-  return new Promise((resolve2) => setTimeout(resolve2, ms));
+  return new Promise((resolve3) => setTimeout(resolve3, ms));
 }
 
 // src/lib/core/sitemap/entries.ts
@@ -4919,6 +5278,86 @@ function isIconType(contentType) {
   return contentType.startsWith("image/") || contentType === "application/ico" || contentType === "application/x-ico";
 }
 
+// src/lib/core/llmstxt/parse.ts
+var H1 = /^ {0,3}#(?:[ \t]+(.*?))?[ \t]*#*[ \t]*$/;
+var FENCE = /^ {0,3}(`{3,}|~{3,})/;
+var LINK = /(!?)\[([^\]]*)\]\(\s*<?([^)\s>]+)>?(?:\s+(?:"[^"]*"|'[^']*'))?\s*\)/g;
+function parseLlmsTxt(body, fileUrl2) {
+  const lines = body.replace(/^\uFEFF/, "").split(/\r\n|\r|\n/);
+  const links = [];
+  let h1;
+  let fence;
+  for (const [index, raw] of lines.entries()) {
+    const line = index + 1;
+    const fenceMatch = FENCE.exec(raw);
+    if (fenceMatch) {
+      const marker = fenceMatch[1];
+      if (fence === void 0) fence = marker;
+      else if (marker[0] === fence[0] && marker.length >= fence.length) fence = void 0;
+      continue;
+    }
+    if (fence !== void 0) continue;
+    if (!h1) {
+      const heading = H1.exec(raw);
+      const text = heading?.[1]?.trim();
+      if (text) h1 = { text, line };
+    }
+    const prose = raw.replace(/`[^`]*`/g, "");
+    for (const match of prose.matchAll(LINK)) {
+      if (match[1] === "!") continue;
+      const url = resolve(match[3], fileUrl2);
+      if (url) links.push({ name: match[2].trim(), url, line });
+    }
+  }
+  return { ...h1 ? { h1 } : {}, links };
+}
+function looksLikeHtml(body) {
+  return /^(?:<!doctype\s+html|<html[\s>])/i.test(body.replace(/^\uFEFF/, "").trimStart());
+}
+function resolve(href, base) {
+  try {
+    const url = new URL(href, base);
+    if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
+    url.hash = "";
+    return url.toString();
+  } catch {
+    return void 0;
+  }
+}
+
+// src/lib/core/probes/llms-txt.ts
+async function probeLlmsTxt(origin, options = {}) {
+  const url = new URL("/llms.txt", origin).toString();
+  const { signal, cleanup } = combineSignals(options.signal, options.timeoutMs ?? 5e3);
+  const nothing = { found: false, html: false, links: [] };
+  try {
+    const res = await fetch(url, { signal, redirect: "follow" });
+    const finalUrl = res.url || url;
+    const contentType = res.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() || void 0;
+    if (!res.ok) {
+      await res.body?.cancel().catch(() => void 0);
+      return { url, status: res.status, finalUrl, contentType, ...nothing };
+    }
+    const body = await res.text();
+    if (looksLikeHtml(body)) {
+      return { url, status: res.status, finalUrl, contentType, found: true, html: true, links: [] };
+    }
+    return {
+      url,
+      status: res.status,
+      finalUrl,
+      contentType,
+      found: true,
+      html: false,
+      ...parseLlmsTxt(body, finalUrl)
+    };
+  } catch {
+    return { url, status: 0, finalUrl: url, ...nothing };
+  } finally {
+    cleanup();
+  }
+}
+
 // src/lib/rules/advisory.ts
 function resolveEvidencePath(extraction, path) {
   let current = extraction;
@@ -5087,7 +5526,12 @@ var hreflangSitemapMismatch = {
   summary: "`<head>` alternates advertise a locale the sitemap does not list",
   prose: "Your `<head>` advertises a translation your sitemap does not list. Is the sitemap missing an entry, or is the alternate pointing at a page you did not mean to publish?",
   why: "The two are derived from one intent by different code paths, so a disagreement means one of them is wrong. No specification says which: the declaration methods are equivalent, and a page can legitimately be cross-linked and kept out of the sitemap.",
-  appliesTo: (site) => site.localeAxis.multilingual && (site.discovery?.urls.length ?? 0) > 0,
+  // A question about what the sitemap *omits*, so it waits for a sitemap read in
+  // full (`./sitemap-inventory.ts`): a translation listed in a child goflag
+  // could not read, or in a second declared sitemap it never opened, would be
+  // asked about as missing — a question with a false premise wastes an agent's
+  // turn as surely as a false finding wastes a human's.
+  appliesTo: (site) => site.localeAxis.multilingual && (site.discovery?.urls.length ?? 0) > 0 && sitemapReadInFull(site),
   ask: (site) => {
     const axis = new Set(site.localeAxis.locales.map((l) => l.toLowerCase()));
     const bySitemap = /* @__PURE__ */ new Map();
@@ -5400,9 +5844,9 @@ function hostOf(url) {
 function mapWithHostCaps(items, opts, worker, hostOf3) {
   const concurrency = Math.max(1, opts.concurrency);
   const perHost = opts.perHost;
-  return new Promise((resolve2) => {
+  return new Promise((resolve3) => {
     if (items.length === 0) {
-      resolve2();
+      resolve3();
       return;
     }
     const pending = items.map((item) => ({ item, host: hostOf3(item) }));
@@ -5412,7 +5856,7 @@ function mapWithHostCaps(items, opts, worker, hostOf3) {
     const finish = () => {
       if (!settled) {
         settled = true;
-        resolve2();
+        resolve3();
       }
     };
     const pump = () => {
@@ -5578,14 +6022,14 @@ async function attachManifests(pages, options) {
   }
 }
 async function probeAssets(pages, options) {
-  const sameOrigin2 = (url) => {
+  const sameOrigin4 = (url) => {
     try {
       return new URL(url).origin === new URL(options.origin).origin;
     } catch {
       return false;
     }
   };
-  const wantedUrl = (url) => /^https?:\/\//i.test(url) && (options.checkExternal || sameOrigin2(url));
+  const wantedUrl = (url) => /^https?:\/\//i.test(url) && (options.checkExternal || sameOrigin4(url));
   const wanted = /* @__PURE__ */ new Set();
   for (const page of pages) {
     for (const image of page.openGraph.images) {
@@ -5639,7 +6083,8 @@ function dropCanonicalDuplicates(pages) {
     if (target === self) return true;
     return !crawled.has(target);
   });
-  return { kept, dropped: pages.length - kept.length };
+  const keptSet = new Set(kept);
+  return { kept, variants: pages.filter((page) => !keptSet.has(page)) };
 }
 async function runAudit(inputUrl, options = {}) {
   const normalized = normalizeInputUrl(inputUrl);
@@ -5673,6 +6118,14 @@ async function runAudit(inputUrl, options = {}) {
     signal: options.signal,
     timeoutMs: options.timeoutMs
   }).catch(() => void 0);
+  const [notFound, llmsTxt] = await Promise.all([
+    probeNotFound(origin, { signal: options.signal, timeoutMs: options.timeoutMs }).catch(
+      () => void 0
+    ),
+    probeLlmsTxt(origin, { signal: options.signal, timeoutMs: options.timeoutMs }).catch(
+      () => void 0
+    )
+  ]);
   const crawlResult = await crawl({
     entryUrl: entry,
     seedUrls: sitemapUrls,
@@ -5706,7 +6159,8 @@ async function runAudit(inputUrl, options = {}) {
   if (escalationWarning) warnings.push(escalationWarning);
   const okPages = pages.filter((p) => isOkStatus(p.fetch.status));
   const documents = okPages.filter(isHtmlPage);
-  const { kept: htmlPages, dropped: duplicatePages } = dropCanonicalDuplicates(documents);
+  const { kept: htmlPages, variants } = dropCanonicalDuplicates(documents);
+  const duplicatePages = variants.length;
   const unreachablePages = [
     ...pages.filter((p) => !isOkStatus(p.fetch.status)).map((p) => ({
       id: fingerprint("page", routeKey(p.fetch.finalUrl)),
@@ -5830,18 +6284,23 @@ async function runAudit(inputUrl, options = {}) {
     })
   });
   warnings.push(...linkReport.diagnostics.warnings);
+  const answered = {
+    crawled: new Map(crawlResult.pages.map((page) => [page.fetch.finalUrl, page.fetch.status])),
+    checked: new Map(Object.values(linkReport.checks).map((check) => [check.url, check])),
+    maxProbes: effectiveMaxPages,
+    signal: options.signal,
+    timeoutMs: options.timeoutMs,
+    allowInsecureTls: options.allowInsecureTls
+  };
   const sitemapEntries = discovery ? await probeSitemapEntries(
     discovery.urls.map((entry2) => entry2.loc),
-    {
-      crawled: new Map(
-        crawlResult.pages.map((page) => [page.fetch.finalUrl, page.fetch.status])
-      ),
-      checked: new Map(Object.values(linkReport.checks).map((check) => [check.url, check])),
-      maxProbes: effectiveMaxPages,
-      signal: options.signal,
-      timeoutMs: options.timeoutMs,
-      allowInsecureTls: options.allowInsecureTls
-    }
+    answered
+  ) : void 0;
+  const llmsTxtLinks = llmsTxt?.found && !llmsTxt.html ? await probeSitemapEntries(
+    [...new Set(llmsTxt.links.map((link) => link.url))].filter(
+      (url) => sameOriginAs(url, origin)
+    ),
+    answered
   ) : void 0;
   const siteContext = {
     origin,
@@ -5851,7 +6310,14 @@ async function runAudit(inputUrl, options = {}) {
     discovery,
     robots,
     favicon: favicon2,
+    notFound,
+    llmsTxt,
+    llmsTxtLinks,
     sitemapEntries,
+    // Set aside above, and still what their URLs serve. The sitemap rules that
+    // ask what a listed URL serves need them: a listed variant is exactly what
+    // `sitemap.entry.non-canonical` exists to catch.
+    variants,
     // Same index the matrix was built from, so a rule that groups by route
     // and the grid that reports holes cannot disagree about which URLs are
     // one page.
@@ -6014,6 +6480,13 @@ function countUncrawled(sitemapUrls, pages) {
   for (const url of sitemapUrls) if (!crawled.has(url)) n += 1;
   return n;
 }
+function sameOriginAs(url, origin) {
+  try {
+    return new URL(url).origin === origin;
+  } catch {
+    return false;
+  }
+}
 function syntheticDiscovery(origin, baseUrl, pages, truncated) {
   return {
     origin,
@@ -6436,7 +6909,7 @@ function hostOf2(value) {
     return value;
   }
 }
-function pathOf(value) {
+function pathOf2(value) {
   try {
     const { pathname, search } = new URL(value);
     return `${pathname}${search}`;
@@ -6465,7 +6938,7 @@ function originLabel(origin) {
       return "computed";
   }
 }
-function resolve(...facts) {
+function resolve2(...facts) {
   for (const fact of facts) {
     if (fact && fact.value.trim() !== "") {
       return { value: fact.value, from: originLabel(fact.origin) };
@@ -6509,13 +6982,13 @@ function cardOf(extraction) {
   return {
     url,
     host: hostOf2(url),
-    path: pathOf(url),
-    ogTitle: resolve(og.title, doc.title),
-    ogDescription: resolve(og.description, meta.description),
-    documentTitle: resolve(doc.title),
-    metaDescription: resolve(meta.description),
-    siteName: resolve(og.siteName),
-    xTitle: resolve(twitter.title, og.title, doc.title),
+    path: pathOf2(url),
+    ogTitle: resolve2(og.title, doc.title),
+    ogDescription: resolve2(og.description, meta.description),
+    documentTitle: resolve2(doc.title),
+    metaDescription: resolve2(meta.description),
+    siteName: resolve2(og.siteName),
+    xTitle: resolve2(twitter.title, og.title, doc.title),
     image: imageOf(extraction),
     icon: safeUrl(extraction.links.icons[0]?.href),
     themeColor: meta.themeColor?.value,
@@ -6768,6 +7241,57 @@ function translationsPanel(current, siblings) {
 </table></div>
 <p class="foot">Lengths are graphemes, which is what a card's size ladder counts. The ladder is the site's own \u2014 <code>@goflag/og</code> ships none \u2014 so this reports the length and not the step.</p></section>`;
 }
+function pathSegments(path) {
+  const cut = path.indexOf("?");
+  const pathname = cut === -1 ? path : path.slice(0, cut);
+  const search = cut === -1 ? "" : path.slice(cut);
+  const segments2 = pathname.split("/").filter(Boolean);
+  if (search === "") return segments2;
+  return segments2.length === 0 ? [search] : [...segments2.slice(0, -1), `${segments2.at(-1)}${search}`];
+}
+function buildRouteTree(paths) {
+  const root = { segments: [], children: /* @__PURE__ */ new Map(), count: 0 };
+  paths.forEach((path, index) => {
+    let node = root;
+    node.count++;
+    for (const segment of pathSegments(path)) {
+      const child = node.children.get(segment) ?? {
+        segments: [segment],
+        children: /* @__PURE__ */ new Map(),
+        count: 0
+      };
+      node.children.set(segment, child);
+      child.count++;
+      node = child;
+    }
+    node.index ??= index;
+  });
+  return { ...root, children: foldChildren(root.children) };
+}
+function foldChildren(children) {
+  return new Map([...children].map(([key, node]) => [key, foldChain(node)]));
+}
+function foldChain(node) {
+  let current = node;
+  while (current.index === void 0 && current.children.size === 1) {
+    const only = [...current.children.values()][0];
+    current = { ...only, segments: [...current.segments, ...only.segments] };
+  }
+  return { ...current, children: foldChildren(current.children) };
+}
+function routeList(node, open, className = "") {
+  const items = [...node.children.values()].map((child) => {
+    const label = esc(child.segments.map((segment) => `/${segment}`).join(""));
+    if (child.children.size === 0) {
+      return child.index === void 0 ? `<li class="leaf"><span>${label}</span></li>` : `<li class="leaf"><a href="#page-${child.index}">${label}</a></li>`;
+    }
+    const self = child.index === void 0 ? `<span class="go"></span>` : `<a class="go" href="#page-${child.index}" title="the page at this path">\u2197</a>`;
+    return `<li><details${open ? " open" : ""}><summary><span class="seg">${label}</span><span class="n">${child.count}</span>${self}</summary>
+${routeList(child, open)}</details></li>`;
+  });
+  return `<ul${className ? ` class="${className}"` : ""}>${items.join("\n")}</ul>`;
+}
+var TREE_OPEN_MAX = 20;
 function tagLabel(tag) {
   const key = tag.property ?? tag.name ?? (tag.httpEquiv ? `http-equiv=${tag.httpEquiv}` : void 0);
   if (key) return `${key}${tag.content ? ` = ${clip(tag.content, 120)}` : ""}`;
@@ -6837,9 +7361,21 @@ h1,h2,h3,.badge,.chip,.rid,.src,.from,.mono,code,pre,dt,.pin,.nav{font-family:va
 h1{font-size:22px;letter-spacing:-.02em;margin:0 0 6px}
 .lede{color:var(--ink2);margin:0;max-width:70ch}
 header.doc{padding:44px 0 24px;border-bottom:1px solid var(--line)}
-.nav{display:flex;flex-wrap:wrap;gap:8px;margin:20px 0 0;padding:0;list-style:none;font-size:12px}
-.nav a{display:inline-block;padding:4px 9px;border:1px solid var(--line2);border-radius:3px;color:var(--ink2);text-decoration:none}
-.nav a:hover,.nav a:focus-visible{border-color:var(--brand);color:var(--brand)}
+.nav{margin:22px 0 0;max-width:620px}
+.navnote{margin:0 0 8px;font-family:var(--ui);font-size:12px;color:var(--ink3)}
+.tree,.tree ul{list-style:none;margin:0;padding:0;font-size:12px}
+.tree ul{margin-left:6px;padding-left:12px;border-left:1px solid var(--line)}
+.tree summary{display:flex;align-items:center;gap:10px;padding:3px 7px;border-radius:3px;color:var(--ink2);cursor:pointer;list-style:none}
+.tree summary::-webkit-details-marker{display:none}
+.tree summary::before{content:"\u25B8";flex:none;width:1ch;font-size:9px;color:var(--ink3)}
+.tree details[open]>summary::before{content:"\u25BE"}
+.tree summary:hover,.tree summary:focus-visible{background:var(--wash);color:var(--ink)}
+.tree .seg{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
+.tree .n{margin-left:auto;flex:none;font-size:10px;color:var(--ink3);font-variant-numeric:tabular-nums}
+.tree .go{flex:none;width:1.2em;text-align:right;font-size:11px;text-decoration:none;opacity:.55}
+.tree .go:hover,.tree .go:focus-visible{opacity:1}
+.tree .leaf a,.tree .leaf span{display:block;padding:3px 7px 3px 24px;color:var(--ink2);text-decoration:none;border-radius:3px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
+.tree .leaf a:hover,.tree .leaf a:focus-visible{background:var(--wash);color:var(--brand)}
 .page{padding-top:52px}
 .pagehead h2{font-size:16px;margin:0 0 4px}
 .pagehead .url{margin:0;font-size:13px;color:var(--ink2);word-break:break-all}
@@ -6955,7 +7491,7 @@ function renderPreview(report, options = {}) {
   }
   const empty = report.extractions === void 0 ? `This report carries no extractions, so there is nothing to draw. Pass <code>extractions: true</code> to <code>runAudit</code>, or use <code>goflag preview &lt;url&gt;</code>, which does.` : `The crawl reached no HTML page it could read, so there is nothing to draw. Every page was unreachable, not HTML, or a declared duplicate of another.`;
   const rows = extractions.map((extraction, index) => {
-    const path = pathOf(extraction.http.finalUrl);
+    const path = pathOf2(extraction.http.finalUrl);
     return {
       index,
       locale: splitRoute(path).locale,
@@ -6967,7 +7503,7 @@ function renderPreview(report, options = {}) {
   });
   const byRoute = /* @__PURE__ */ new Map();
   extractions.forEach((extraction, index) => {
-    const route = splitRoute(pathOf(extraction.http.finalUrl)).route;
+    const route = splitRoute(pathOf2(extraction.http.finalUrl)).route;
     byRoute.set(route, [...byRoute.get(route) ?? [], rows[index]]);
   });
   const body = extractions.length === 0 ? `<section class="page"><p class="none">${empty}</p></section>` : extractions.map(
@@ -6975,12 +7511,17 @@ function renderPreview(report, options = {}) {
       extraction,
       issuesByPage.get(extraction.http.finalUrl) ?? [],
       index,
-      byRoute.get(splitRoute(pathOf(extraction.http.finalUrl)).route) ?? []
+      byRoute.get(splitRoute(pathOf2(extraction.http.finalUrl)).route) ?? []
     )
   ).join("\n");
-  const nav = extractions.length > 1 ? `<ul class="nav">${extractions.map(
-    (extraction, index) => `<li><a href="#page-${index}">${esc(pathOf(extraction.http.finalUrl))}</a></li>`
-  ).join("")}</ul>` : "";
+  const nav = extractions.length > 1 ? `<nav class="nav">
+  <p class="navnote">${extractions.length} pages, by path. A folder folds; the \u2197 beside one opens the page at that path itself.</p>
+  ${routeList(
+    buildRouteTree(extractions.map((extraction) => pathOf2(extraction.http.finalUrl))),
+    extractions.length <= TREE_OPEN_MAX,
+    "tree"
+  )}
+</nav>` : "";
   return `<!doctype html>
 <html lang="en">
 <head>
@@ -7294,4 +7835,4 @@ export {
   summarize,
   Logger
 };
-//# sourceMappingURL=chunk-QKXUKHXB.js.map
+//# sourceMappingURL=chunk-YQPL67HX.js.map