@goflag/cli 0.2.9 → 0.2.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -816,53 +816,111 @@ async function probeManifest(href, options = {}) {
816
816
  }
817
817
  }
818
818
 
819
+ // src/lib/core/robots/parse.ts
820
+ var RULE_DIRECTIVES = /* @__PURE__ */ new Set(["allow", "disallow"]);
821
+ var KNOWN_EXTENSIONS = /* @__PURE__ */ new Set(["crawl-delay", "host", "clean-param", "request-rate", "noindex"]);
822
+ function parseRobots(body) {
823
+ const groups = [];
824
+ const sitemaps = [];
825
+ const invalidLines = [];
826
+ const unknownDirectives = [];
827
+ let current;
828
+ let currentTookRule = false;
829
+ const lines = body.split(/\r\n|\r|\n/);
830
+ for (const [index, rawLine] of lines.entries()) {
831
+ const line = index + 1;
832
+ const withoutComment = rawLine.replace(/#.*$/, "");
833
+ const trimmed = withoutComment.trim();
834
+ if (trimmed.length === 0) continue;
835
+ const separator = trimmed.indexOf(":");
836
+ if (separator === -1) {
837
+ invalidLines.push({ line, raw: rawLine, reason: "no `:` separator" });
838
+ continue;
839
+ }
840
+ const name = trimmed.slice(0, separator).trim().toLowerCase();
841
+ const value = trimmed.slice(separator + 1).trim();
842
+ if (name === "user-agent") {
843
+ if (value.length === 0) {
844
+ invalidLines.push({ line, raw: rawLine, reason: "empty `User-agent`" });
845
+ continue;
846
+ }
847
+ if (!current || currentTookRule) {
848
+ current = { userAgents: [], rules: [] };
849
+ groups.push(current);
850
+ currentTookRule = false;
851
+ }
852
+ current.userAgents.push({ value: value.toLowerCase(), line });
853
+ continue;
854
+ }
855
+ if (name === "sitemap") {
856
+ if (value.length === 0) {
857
+ invalidLines.push({ line, raw: rawLine, reason: "empty `Sitemap`" });
858
+ continue;
859
+ }
860
+ sitemaps.push({ value, line });
861
+ continue;
862
+ }
863
+ if (RULE_DIRECTIVES.has(name)) {
864
+ if (!current) {
865
+ invalidLines.push({
866
+ line,
867
+ raw: rawLine,
868
+ reason: `\`${name}\` before any \`User-agent\` group`
869
+ });
870
+ continue;
871
+ }
872
+ current.rules.push({ kind: name, pattern: value, line });
873
+ currentTookRule = true;
874
+ continue;
875
+ }
876
+ if (KNOWN_EXTENSIONS.has(name)) {
877
+ unknownDirectives.push({ name, line });
878
+ continue;
879
+ }
880
+ invalidLines.push({ line, raw: rawLine, reason: `unknown directive \`${name}\`` });
881
+ }
882
+ return { groups, sitemaps, invalidLines, unknownDirectives };
883
+ }
884
+
819
885
  // src/lib/core/probes/robots.ts
820
886
  async function probeRobots(origin, options = {}) {
821
887
  const url = new URL("/robots.txt", origin).toString();
822
888
  const { signal, cleanup } = combineSignals(options.signal, options.timeoutMs ?? 5e3);
889
+ const empty = { groups: [], sitemaps: [], invalidLines: [], unknownDirectives: [] };
890
+ const noRedirect = { count: 0, finalUrl: url, crossOrigin: false };
823
891
  try {
824
892
  const res = await fetch(url, { signal, redirect: "follow" });
893
+ const redirected = res.url !== url;
894
+ const redirects = {
895
+ count: redirected ? 1 : 0,
896
+ finalUrl: res.url || url,
897
+ crossOrigin: redirected && !sameOrigin(res.url, url)
898
+ };
825
899
  if (!res.ok) {
826
- return { url, status: res.status, found: false, sitemaps: [], blocksAll: false };
900
+ return { url, status: res.status, found: false, byteLength: 0, redirects, ...empty };
827
901
  }
828
- const body = await res.text();
902
+ const raw = await res.text();
829
903
  return {
830
904
  url,
831
905
  status: res.status,
832
906
  found: true,
833
- raw: body,
834
- sitemaps: extractSitemaps(body),
835
- blocksAll: blocksAllUserAgents(body)
907
+ raw,
908
+ byteLength: Buffer.byteLength(raw, "utf8"),
909
+ redirects,
910
+ ...parseRobots(raw)
836
911
  };
837
912
  } catch {
838
- return { url, status: 0, found: false, sitemaps: [], blocksAll: false };
913
+ return { url, status: 0, found: false, byteLength: 0, redirects: noRedirect, ...empty };
839
914
  } finally {
840
915
  cleanup();
841
916
  }
842
917
  }
843
- function extractSitemaps(body) {
844
- const out = [];
845
- for (const line of body.split(/\r?\n/)) {
846
- const m = /^\s*Sitemap:\s*(\S+)\s*$/i.exec(line);
847
- if (m?.[1]) out.push(m[1]);
848
- }
849
- return out;
850
- }
851
- function blocksAllUserAgents(body) {
852
- let inWildcard = false;
853
- for (const rawLine of body.split(/\r?\n/)) {
854
- const line = rawLine.replace(/#.*$/, "").trim();
855
- if (line.length === 0) continue;
856
- const ua = /^User-agent:\s*(.+)$/i.exec(line);
857
- if (ua) {
858
- inWildcard = ua[1].trim() === "*";
859
- continue;
860
- }
861
- if (!inWildcard) continue;
862
- const dis = /^Disallow:\s*(.*)$/i.exec(line);
863
- if (dis && dis[1].trim() === "/") return true;
918
+ function sameOrigin(a, b) {
919
+ try {
920
+ return new URL(a).origin === new URL(b).origin;
921
+ } catch {
922
+ return true;
864
923
  }
865
- return false;
866
924
  }
867
925
 
868
926
  // src/lib/core/probes/sitemap.ts
@@ -1293,7 +1351,14 @@ function findingsToIssues(result, rules) {
1293
1351
  severity: finding.severity ?? "info",
1294
1352
  message: finding.message ?? `Expected ${finding.expected}.`,
1295
1353
  origin: finding.origin,
1296
- fix: byId.get(finding.ruleId)?.fix
1354
+ fix: byId.get(finding.ruleId)?.fix,
1355
+ // The finding has known all four since it was evaluated; this is where
1356
+ // they used to stop. A report that drops the rigor asks every consumer
1357
+ // to re-derive it from a registry it does not have.
1358
+ rigor: finding.rigor,
1359
+ sources: finding.sources,
1360
+ observed: finding.observed,
1361
+ expected: finding.expected
1297
1362
  });
1298
1363
  }
1299
1364
  for (const crash of result.crashes) {
@@ -2594,6 +2659,70 @@ function sortLocales(locales) {
2594
2659
  return [...xDefault, ...rest];
2595
2660
  }
2596
2661
 
2662
+ // src/lib/core/robots/match.ts
2663
+ function robotsAllows(groups, path, userAgent = "*") {
2664
+ const group = selectGroup(groups, userAgent);
2665
+ if (!group) return { allowed: true };
2666
+ const token = group.userAgents[0]?.value ?? "*";
2667
+ let best;
2668
+ let bestLength = -1;
2669
+ for (const rule of group.rules) {
2670
+ if (rule.pattern.length === 0) continue;
2671
+ if (!patternMatches(rule.pattern, path)) continue;
2672
+ const length = octetLength(rule.pattern);
2673
+ if (length > bestLength) {
2674
+ best = rule;
2675
+ bestLength = length;
2676
+ continue;
2677
+ }
2678
+ if (length === bestLength && rule.kind === "allow") best = rule;
2679
+ }
2680
+ if (!best) return { allowed: true, group: token };
2681
+ return { allowed: best.kind === "allow", rule: best, group: token };
2682
+ }
2683
+ function selectGroup(groups, userAgent) {
2684
+ const wanted = userAgent.trim().toLowerCase();
2685
+ let bestToken;
2686
+ for (const group of groups) {
2687
+ for (const agent of group.userAgents) {
2688
+ if (agent.value === "*") continue;
2689
+ if (!wanted.startsWith(agent.value)) continue;
2690
+ if (bestToken === void 0 || agent.value.length > bestToken.length) {
2691
+ bestToken = agent.value;
2692
+ }
2693
+ }
2694
+ }
2695
+ const token = bestToken ?? "*";
2696
+ const matching = groups.filter(
2697
+ (group) => group.userAgents.some((agent) => agent.value === token)
2698
+ );
2699
+ if (matching.length === 0) return void 0;
2700
+ if (matching.length === 1) return matching[0];
2701
+ return {
2702
+ userAgents: matching.flatMap((group) => group.userAgents),
2703
+ rules: matching.flatMap((group) => group.rules)
2704
+ };
2705
+ }
2706
+ function patternMatches(pattern, path) {
2707
+ const anchored = pattern.endsWith("$");
2708
+ const body = anchored ? pattern.slice(0, -1) : pattern;
2709
+ const source = body.split("*").map((literal) => escapeRegExp(normalizeOctets(literal))).join(".*");
2710
+ return new RegExp(`^${source}${anchored ? "$" : ""}`).test(normalizeOctets(path));
2711
+ }
2712
+ function normalizeOctets(value) {
2713
+ try {
2714
+ return decodeURIComponent(value);
2715
+ } catch {
2716
+ return value;
2717
+ }
2718
+ }
2719
+ function octetLength(pattern) {
2720
+ return Buffer.byteLength(normalizeOctets(pattern), "utf8");
2721
+ }
2722
+ function escapeRegExp(value) {
2723
+ return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
2724
+ }
2725
+
2597
2726
  // src/lib/rules/site-rules.ts
2598
2727
  function isMultilingual(site) {
2599
2728
  return site.localeAxis.multilingual;
@@ -2612,6 +2741,7 @@ function rowOf(site, url, pathname) {
2612
2741
  }
2613
2742
  function sitemapLocalesByRoute(site) {
2614
2743
  const byRoute = /* @__PURE__ */ new Map();
2744
+ const axis = new Set(site.localeAxis.locales.map((l) => l.toLowerCase()));
2615
2745
  for (const entry of site.discovery?.urls ?? []) {
2616
2746
  let pathname;
2617
2747
  try {
@@ -2621,6 +2751,7 @@ function sitemapLocalesByRoute(site) {
2621
2751
  }
2622
2752
  const { locale } = splitRoute(pathname);
2623
2753
  if (locale === "x-default") continue;
2754
+ if (!axis.has(locale.toLowerCase())) continue;
2624
2755
  const route = rowOf(site, entry.loc, pathname);
2625
2756
  const set = byRoute.get(route) ?? /* @__PURE__ */ new Set();
2626
2757
  set.add(locale.toLowerCase());
@@ -2635,6 +2766,8 @@ var hreflangMissing = {
2635
2766
  id: "hreflang.missing",
2636
2767
  severity: "error",
2637
2768
  summary: "Pages on a multilingual site must advertise their locale alternates",
2769
+ rigor: "vendor-spec",
2770
+ sources: ["google-hreflang", "whatwg-html-link-types"],
2638
2771
  appliesTo: isMultilingual,
2639
2772
  check: ({ site, issue }) => {
2640
2773
  const locales = site.localeAxis.locales.join(", ");
@@ -2668,62 +2801,57 @@ var hreflangMissing = {
2668
2801
  );
2669
2802
  }
2670
2803
  };
2671
- var hreflangSitemapMismatch = {
2672
- id: "hreflang.sitemap-mismatch",
2673
- severity: "warning",
2674
- summary: "`<head>` alternates and sitemap locale coverage must agree",
2675
- appliesTo: (site) => isMultilingual(site) && (site.discovery?.urls.length ?? 0) > 0,
2676
- check: ({ site, issue }) => {
2677
- const bySitemap = sitemapLocalesByRoute(site);
2678
- const findings = [];
2679
- for (const page of site.pages) {
2680
- const head = declaredLocales(page);
2681
- if (head.size === 0) continue;
2682
- let pathname;
2683
- try {
2684
- pathname = new URL(page.fetch.finalUrl).pathname;
2685
- } catch {
2686
- continue;
2687
- }
2688
- const route = rowOf(site, page.fetch.finalUrl, pathname);
2689
- const inSitemap = bySitemap.get(route);
2690
- if (!inSitemap || inSitemap.size === 0) continue;
2691
- const onlyInSitemap = sorted(new Set([...inSitemap].filter((l) => !head.has(l))));
2692
- const onlyInHead = sorted(new Set([...head].filter((l) => !inSitemap.has(l))));
2693
- if (onlyInSitemap.length === 0 && onlyInHead.length === 0) continue;
2694
- const parts = [];
2695
- if (onlyInSitemap.length > 0) {
2696
- parts.push(
2697
- `the sitemap lists ${onlyInSitemap.join(", ")} but the \`<head>\` does not advertise ${onlyInSitemap.length > 1 ? "them" : "it"}`
2698
- );
2699
- }
2700
- if (onlyInHead.length > 0) {
2701
- parts.push(
2702
- `the \`<head>\` advertises ${onlyInHead.join(", ")} but the sitemap has no entry for ${onlyInHead.length > 1 ? "them" : "it"}`
2703
- );
2704
- }
2705
- findings.push(
2706
- issue({
2707
- pageUrl: page.fetch.finalUrl,
2708
- message: `Route \`${route}\`: ${parts.join("; ")}. Both are derived from the same intent and must not disagree.`,
2709
- origin: { kind: "link", rel: "alternate" },
2710
- fix: {
2711
- title: "Derive both from one locale-availability source",
2712
- snippet: [
2713
- "// Compute availability once, feed both the <head> and the sitemap.",
2714
- "const localesFor = (slug: string) =>",
2715
- " allDocs.filter((d) => d.slug === slug && !d.draft).map((d) => d.locale);",
2716
- "",
2717
- "// generateMetadata(): alternates.languages \u2190 localesFor(slug)",
2718
- "// sitemap.ts: alternates.languages \u2190 localesFor(slug)"
2719
- ].join("\n"),
2720
- language: "ts"
2721
- }
2722
- })
2723
- );
2804
+ function coverageGaps(site) {
2805
+ const bySitemap = sitemapLocalesByRoute(site);
2806
+ const gaps = [];
2807
+ for (const page of site.pages) {
2808
+ const head = declaredLocales(page);
2809
+ if (head.size === 0) continue;
2810
+ let pathname;
2811
+ try {
2812
+ pathname = new URL(page.fetch.finalUrl).pathname;
2813
+ } catch {
2814
+ continue;
2724
2815
  }
2725
- return findings;
2816
+ const route = rowOf(site, page.fetch.finalUrl, pathname);
2817
+ const inSitemap = bySitemap.get(route);
2818
+ if (!inSitemap || inSitemap.size === 0) continue;
2819
+ gaps.push({
2820
+ page,
2821
+ route,
2822
+ onlyInSitemap: sorted(new Set([...inSitemap].filter((l) => !head.has(l))))
2823
+ });
2726
2824
  }
2825
+ return gaps;
2826
+ }
2827
+ var ONE_SOURCE_FIX = {
2828
+ title: "Derive both from one locale-availability source",
2829
+ snippet: [
2830
+ "// Compute availability once, feed both the <head> and the sitemap.",
2831
+ "const localesFor = (slug: string) =>",
2832
+ " allDocs.filter((d) => d.slug === slug && !d.draft).map((d) => d.locale);",
2833
+ "",
2834
+ "// generateMetadata(): alternates.languages \u2190 localesFor(slug)",
2835
+ "// sitemap.ts: alternates.languages \u2190 localesFor(slug)"
2836
+ ].join("\n"),
2837
+ language: "ts"
2838
+ };
2839
+ var bothApply = (site) => isMultilingual(site) && (site.discovery?.urls.length ?? 0) > 0;
2840
+ var hreflangClusterIncomplete = {
2841
+ id: "hreflang.cluster-incomplete",
2842
+ severity: "warning",
2843
+ summary: "Every locale the sitemap publishes must appear in the route's `hreflang` cluster",
2844
+ rigor: "vendor-spec",
2845
+ sources: ["google-hreflang"],
2846
+ appliesTo: bothApply,
2847
+ check: ({ site, issue }) => coverageGaps(site).filter((gap) => gap.onlyInSitemap.length > 0).map(
2848
+ ({ page, route, onlyInSitemap }) => issue({
2849
+ pageUrl: page.fetch.finalUrl,
2850
+ message: `Route \`${route}\`: the sitemap lists ${onlyInSitemap.join(", ")} but the \`<head>\` does not advertise ${onlyInSitemap.length > 1 ? "them" : "it"}. The site publishes ${onlyInSitemap.length > 1 ? "those versions" : "that version"} and leaves ${onlyInSitemap.length > 1 ? "them" : "it"} outside the cluster, so ${onlyInSitemap.length > 1 ? "they compete" : "it competes"} with this page instead of consolidating with it.`,
2851
+ origin: { kind: "link", rel: "alternate" },
2852
+ fix: ONE_SOURCE_FIX
2853
+ })
2854
+ )
2727
2855
  };
2728
2856
  function metaRobotsTokens(page) {
2729
2857
  const raw = page.meta.robots?.value ?? "";
@@ -2735,7 +2863,9 @@ var robotsBlocksSite = {
2735
2863
  id: "robots.blocks-site",
2736
2864
  severity: "error",
2737
2865
  summary: "`robots.txt` must not forbid crawling a site that asks to be indexed",
2738
- appliesTo: (site) => site.robots?.found === true && site.robots.blocksAll,
2866
+ rigor: "vendor-spec",
2867
+ sources: ["ietf-rfc9309", "google-robots-intro"],
2868
+ appliesTo: (site) => site.robots?.found === true && !robotsAllows(site.robots.groups, "/").allowed,
2739
2869
  check: ({ site, issue }) => {
2740
2870
  const robotsUrl = site.robots?.url ?? `${site.origin}/robots.txt`;
2741
2871
  const asking = site.pages.filter((page) => metaRobotsTokens(page).has("index"));
@@ -2801,106 +2931,764 @@ var iconsIcoMissing = {
2801
2931
  });
2802
2932
  }
2803
2933
  };
2804
- var SITE_RULES = [
2805
- hreflangMissing,
2806
- hreflangSitemapMismatch,
2807
- iconsIcoMissing,
2808
- robotsBlocksSite
2809
- ];
2810
- function getSiteRule(id) {
2811
- return SITE_RULES.find((rule) => rule.id === id);
2812
- }
2813
-
2814
- // src/lib/core/lint-site.ts
2815
- function lintSite(site, rules = SITE_RULES) {
2816
- const drafts = [];
2817
- for (const rule of rules) {
2818
- if (rule.appliesTo && !rule.appliesTo(site)) continue;
2819
- let result;
2820
- try {
2821
- result = rule.check({
2822
- site,
2823
- issue: (input) => ({
2824
- ruleId: rule.id,
2825
- severity: input.severity ?? rule.severity,
2826
- message: input.message,
2827
- origin: input.origin,
2828
- fix: input.fix,
2829
- suggestion: input.suggestion,
2830
- pageUrl: input.pageUrl
2831
- })
2832
- });
2833
- } catch (err) {
2834
- drafts.push({
2835
- ruleId: "engine.site-rule-crashed",
2836
- severity: "info",
2837
- message: `Site rule \`${rule.id}\` threw: ${err instanceof Error ? err.message : String(err)}`,
2838
- pageUrl: site.origin
2934
+ var ROBOTS_BYTE_LIMIT = 500 * 1024;
2935
+ var robotstxtUnreachable = {
2936
+ id: "robotstxt.unreachable",
2937
+ severity: "error",
2938
+ summary: "A robots.txt that errors is read as forbidding the whole site",
2939
+ rigor: "spec-required",
2940
+ sources: ["ietf-rfc9309"],
2941
+ // A 404 is not a failure: §2.3.1.3 says an absent file allows everything,
2942
+ // and most sites that have none mean exactly that. Only a server that
2943
+ // answered badly, or did not answer, triggers the disallow-everything rule.
2944
+ appliesTo: (site) => site.robots !== void 0 && isRobotsFailure(site.robots.status),
2945
+ check: ({ site, issue }) => {
2946
+ const probe = site.robots;
2947
+ const detail = probe.status === 0 ? "the request failed" : `the origin answered ${probe.status}`;
2948
+ return issue({
2949
+ pageUrl: probe.url,
2950
+ message: `\`robots.txt\` could not be read: ${detail}. RFC 9309 \xA72.3.1.4 tells a crawler to assume a complete disallow for as long as this lasts \u2014 an outage on this one file takes the whole site out of the index.`,
2951
+ origin: { kind: "computed" }
2952
+ });
2953
+ }
2954
+ };
2955
+ var robotstxtOversized = {
2956
+ id: "robotstxt.oversized",
2957
+ severity: "error",
2958
+ summary: "Rules past 500 KiB of robots.txt are not guaranteed to be read",
2959
+ rigor: "spec-required",
2960
+ sources: ["ietf-rfc9309"],
2961
+ appliesTo: (site) => (site.robots?.byteLength ?? 0) > ROBOTS_BYTE_LIMIT,
2962
+ check: ({ site, issue }) => {
2963
+ const probe = site.robots;
2964
+ const kib = Math.round(probe.byteLength / 1024);
2965
+ return issue({
2966
+ pageUrl: probe.url,
2967
+ message: `\`robots.txt\` is ${kib} KiB. A parser is only required to honour the first 500 KiB (RFC 9309 \xA72.4), so every rule past that point silently does not exist.`,
2968
+ origin: { kind: "computed" }
2969
+ });
2970
+ }
2971
+ };
2972
+ var robotstxtInvalidLine = {
2973
+ id: "robotstxt.invalid-line",
2974
+ severity: "warning",
2975
+ summary: "Every line of robots.txt should parse as something",
2976
+ rigor: "spec-required",
2977
+ sources: ["ietf-rfc9309"],
2978
+ appliesTo: (site) => (site.robots?.invalidLines.length ?? 0) > 0,
2979
+ check: ({ site, issue }) => {
2980
+ const probe = site.robots;
2981
+ const shown = probe.invalidLines.slice(0, 5);
2982
+ const rest = probe.invalidLines.length - shown.length;
2983
+ return issue({
2984
+ pageUrl: probe.url,
2985
+ message: `\`robots.txt\` has ${probe.invalidLines.length} line${probe.invalidLines.length === 1 ? "" : "s"} that parse as nothing: ` + shown.map((l) => `line ${l.line} (${l.reason})`).join(", ") + `${rest > 0 ? `, and ${rest} more` : ""}. A crawler drops them silently, so the rule you meant to write is simply absent.`,
2986
+ origin: { kind: "computed" }
2987
+ });
2988
+ }
2989
+ };
2990
+ var robotstxtUnknownDirective = {
2991
+ id: "robotstxt.unknown-directive",
2992
+ severity: "info",
2993
+ summary: "Non-standard robots.txt directives are read by some crawlers and ignored by others",
2994
+ rigor: "guideline",
2995
+ sources: ["ietf-rfc9309", "google-robots-intro"],
2996
+ appliesTo: (site) => (site.robots?.unknownDirectives.length ?? 0) > 0,
2997
+ check: ({ site, issue }) => {
2998
+ const probe = site.robots;
2999
+ const names = [...new Set(probe.unknownDirectives.map((d) => d.name))];
3000
+ return issue({
3001
+ pageUrl: probe.url,
3002
+ message: `\`robots.txt\` uses ${names.map((n) => `\`${n}\``).join(", ")}, which RFC 9309 does not define. Some crawlers honour ${names.length === 1 ? "it" : "them"} and Google ignores ${names.length === 1 ? "it" : "them"} \u2014 so this is worth knowing, not necessarily worth changing.`,
3003
+ origin: { kind: "computed" }
3004
+ });
3005
+ }
3006
+ };
3007
+ var robotstxtCrossOrigin = {
3008
+ id: "robotstxt.cross-origin",
3009
+ severity: "warning",
3010
+ summary: "`/robots.txt` should not redirect to another origin",
3011
+ rigor: "vendor-spec",
3012
+ sources: ["ietf-rfc9309"],
3013
+ appliesTo: (site) => site.robots?.redirects.crossOrigin === true,
3014
+ check: ({ site, issue }) => {
3015
+ const probe = site.robots;
3016
+ return issue({
3017
+ pageUrl: probe.url,
3018
+ message: `\`robots.txt\` redirects to \`${probe.redirects.finalUrl}\`, on another origin. RFC 9309 \xA72.3.1.2 permits following it, so this works \u2014 but the policy for this site now lives somewhere this site does not control, and it is usually a proxy accident rather than a decision.`,
3019
+ origin: { kind: "computed" }
3020
+ });
3021
+ }
3022
+ };
3023
+ var robotstxtSitemapRelative = {
3024
+ id: "robotstxt.sitemap.relative",
3025
+ severity: "error",
3026
+ summary: "A `Sitemap:` declaration must be an absolute URL",
3027
+ rigor: "spec-required",
3028
+ sources: ["sitemaps-protocol", "ietf-rfc9309"],
3029
+ appliesTo: (site) => (site.robots?.sitemaps ?? []).some((s) => !isAbsolute(s.value)),
3030
+ check: ({ site, issue }) => {
3031
+ const probe = site.robots;
3032
+ const relative = probe.sitemaps.filter((s) => !isAbsolute(s.value));
3033
+ return issue({
3034
+ pageUrl: probe.url,
3035
+ message: `\`Sitemap:\` must be a full URL: ${relative.map((s) => `line ${s.line} declares \`${s.value}\``).join(", ")}. robots.txt is fetched on its own, so there is no page for a consumer to resolve a relative path against.`,
3036
+ origin: { kind: "computed" }
3037
+ });
3038
+ }
3039
+ };
3040
+ var robotsBlocksPage = {
3041
+ id: "robots.blocks-page",
3042
+ severity: "error",
3043
+ summary: "A page asking to be indexed must not be disallowed by robots.txt",
3044
+ rigor: "vendor-spec",
3045
+ sources: ["ietf-rfc9309", "google-robots-intro"],
3046
+ appliesTo: (site) => site.robots?.found === true && robotsAllows(site.robots.groups, "/").allowed,
3047
+ check: ({ site, issue }) => {
3048
+ const probe = site.robots;
3049
+ return site.pages.flatMap((page) => {
3050
+ if (!metaRobotsTokens(page).has("index")) return [];
3051
+ let path;
3052
+ try {
3053
+ const url = new URL(page.fetch.finalUrl);
3054
+ path = `${url.pathname}${url.search}`;
3055
+ } catch {
3056
+ return [];
3057
+ }
3058
+ const decision = robotsAllows(probe.groups, path);
3059
+ if (decision.allowed || !decision.rule) return [];
3060
+ return issue({
3061
+ pageUrl: page.fetch.finalUrl,
3062
+ message: `Page declares \`<meta name="robots" content="index">\` but \`robots.txt\` line ${decision.rule.line} disallows \`${decision.rule.pattern}\` for \`${decision.group}\`. robots.txt wins: the page is never fetched, so the tag asking for it is never read.`,
3063
+ origin: { kind: "computed" }
2839
3064
  });
2840
- continue;
2841
- }
2842
- if (!result) continue;
2843
- if (Array.isArray(result)) drafts.push(...result);
2844
- else drafts.push(result);
3065
+ });
2845
3066
  }
2846
- return sortSiteIssues(drafts);
2847
- }
2848
- function sortSiteIssues(drafts) {
2849
- const byPage = [...drafts].sort((a, b) => a.pageUrl.localeCompare(b.pageUrl));
2850
- return sortIssues(byPage);
2851
- }
2852
-
2853
- // src/lib/core/locales.ts
2854
- function localePrefixOf(url) {
2855
- let pathname;
2856
- try {
2857
- pathname = new URL(url).pathname;
2858
- } catch {
2859
- return null;
3067
+ };
3068
+ var CHANGEFREQ = /* @__PURE__ */ new Set(["always", "hourly", "daily", "weekly", "monthly", "yearly", "never"]);
3069
+ var SAMPLE = 5;
3070
+ function sample(locs) {
3071
+ const shown = locs.slice(0, SAMPLE);
3072
+ const rest = locs.length - shown.length;
3073
+ return `${shown.map((l) => `\`${l}\``).join(", ")}${rest > 0 ? `, and ${rest} more` : ""}`;
3074
+ }
3075
+ var sitemapMissing = {
3076
+ id: "sitemap.missing",
3077
+ severity: "warning",
3078
+ summary: "A site should publish a sitemap",
3079
+ rigor: "guideline",
3080
+ sources: ["sitemaps-protocol", "google-sitemaps"],
3081
+ // A warning rather than an error, and the plan settles it: a small, fully
3082
+ // linked site genuinely may not need one. Google's own guidance says
3083
+ // "usually worth having", which is exactly a guideline.
3084
+ appliesTo: (site) => site.discovery !== void 0 && !site.discovery.diagnostics.found,
3085
+ check: ({ site, issue }) => issue({
3086
+ pageUrl: `${site.origin}/sitemap.xml`,
3087
+ message: "No sitemap was found \u2014 not declared in `robots.txt`, and not at a well-known path. Discovery then depends entirely on what links to what, which is the part of a site nobody audits.",
3088
+ origin: { kind: "computed" }
3089
+ })
3090
+ };
3091
+ var sitemapUnparsable = {
3092
+ id: "sitemap.unparsable",
3093
+ severity: "error",
3094
+ summary: "A located sitemap must be a well-formed `<urlset>` or `<sitemapindex>`",
3095
+ rigor: "spec-required",
3096
+ sources: ["sitemaps-protocol"],
3097
+ appliesTo: (site) => site.discovery?.diagnostics.found === true && !site.discovery.diagnostics.wellFormed,
3098
+ check: ({ site, issue }) => {
3099
+ const diagnostics = site.discovery.diagnostics;
3100
+ return issue({
3101
+ pageUrl: diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3102
+ message: "A sitemap was served but does not parse as XML. The usual cause is an HTML error page answered with a 200 \u2014 which reads as a healthy sitemap to anything that only checks the status.",
3103
+ origin: { kind: "computed" }
3104
+ });
2860
3105
  }
2861
- const first = pathname.split("/").filter(Boolean)[0];
2862
- if (first && looksLikeLocaleSegment(first)) return first.toLowerCase();
2863
- return null;
2864
- }
2865
- function prefixesIn(urls) {
2866
- const out = /* @__PURE__ */ new Set();
2867
- for (const url of urls ?? []) {
2868
- const locale = localePrefixOf(url);
2869
- if (locale) out.add(locale);
3106
+ };
3107
+ var sitemapEmpty = {
3108
+ id: "sitemap.empty",
3109
+ severity: "warning",
3110
+ summary: "A sitemap that parses should list something",
3111
+ rigor: "guideline",
3112
+ sources: ["sitemaps-protocol"],
3113
+ // Only when the crawl found pages: an empty sitemap on a site with no pages
3114
+ // is consistent, and saying otherwise would be noise.
3115
+ appliesTo: (site) => site.discovery?.diagnostics.wellFormed === true && site.discovery.urls.length === 0 && site.pages.length > 0,
3116
+ check: ({ site, issue }) => {
3117
+ const diagnostics = site.discovery.diagnostics;
3118
+ return issue({
3119
+ pageUrl: diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3120
+ message: `The sitemap parses and lists no URLs, while the crawl found ${site.pages.length}. goflag falls back to crawling \u2014 a consumer that trusts the sitemap has nothing to read.`,
3121
+ origin: { kind: "computed" }
3122
+ });
2870
3123
  }
2871
- out.delete("x-default");
2872
- return out;
2873
- }
2874
- function describeCandidates(pages) {
2875
- const byTag = /* @__PURE__ */ new Map();
2876
- for (const page of pages) {
2877
- const tag = localePrefixOf(page.fetch.finalUrl);
2878
- if (!tag || tag === "x-default") continue;
2879
- const entry = byTag.get(tag) ?? { pages: 0, langs: /* @__PURE__ */ new Set() };
2880
- entry.pages += 1;
2881
- const lang = page.raw.htmlLang?.trim().toLowerCase();
2882
- if (lang) entry.langs.add(lang);
2883
- byTag.set(tag, entry);
3124
+ };
3125
+ var sitemapIndexChildError = {
3126
+ id: "sitemap.index.child-error",
3127
+ severity: "error",
3128
+ summary: "Every child of a sitemap index must be reachable and parseable",
3129
+ rigor: "spec-required",
3130
+ sources: ["sitemaps-protocol"],
3131
+ appliesTo: (site) => (site.discovery?.diagnostics.childSitemapErrors ?? 0) > 0,
3132
+ check: ({ site, issue }) => {
3133
+ const diagnostics = site.discovery.diagnostics;
3134
+ const total = diagnostics.childSitemapCount;
3135
+ return issue({
3136
+ pageUrl: diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3137
+ message: `${diagnostics.childSitemapErrors} of ${total} child sitemaps could not be read. The index declares an inventory and part of it is missing, so whatever those documents listed is invisible \u2014 and nothing says how much that is.`,
3138
+ origin: { kind: "computed" }
3139
+ });
2884
3140
  }
2885
- const candidates = [];
2886
- for (const [tag, { pages: n, langs }] of byTag) {
2887
- const observedLangs = [...langs].sort();
2888
- const htmlLangAgrees = observedLangs.length === 0 ? void 0 : observedLangs.some((l) => l.split("-")[0] === tag.split("-")[0]);
2889
- candidates.push({
2890
- tag,
2891
- pages: n,
2892
- isKnownLanguage: isKnownLanguageTag(tag),
2893
- htmlLangAgrees,
2894
- observedLangs
3141
+ };
3142
+ var sitemapEntryInvalidUrl = {
3143
+ id: "sitemap.entry.invalid-url",
3144
+ severity: "error",
3145
+ summary: "Every `<loc>` must be an absolute, parseable URL",
3146
+ rigor: "spec-required",
3147
+ sources: ["sitemaps-protocol"],
3148
+ appliesTo: (site) => (site.discovery?.urls ?? []).some((entry) => !isAbsolute(entry.loc)),
3149
+ check: ({ site, issue }) => {
3150
+ const bad = site.discovery.urls.filter((entry) => !isAbsolute(entry.loc));
3151
+ return issue({
3152
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3153
+ message: `${bad.length} \`<loc>\` value${bad.length === 1 ? " is" : "s are"} not an absolute URL: ${sample(bad.map((e) => e.loc))}. A sitemap is fetched on its own, so a consumer has nothing to resolve them against.`,
3154
+ origin: { kind: "computed" }
2895
3155
  });
2896
3156
  }
2897
- return candidates.sort(
2898
- (a, b) => Number(b.htmlLangAgrees ?? false) - Number(a.htmlLangAgrees ?? false) || Number(b.isKnownLanguage) - Number(a.isKnownLanguage) || b.pages - a.pages || a.tag.localeCompare(b.tag)
3157
+ };
3158
+ var DOCUMENT_URL_LIMIT = 5e4;
3159
+ var DOCUMENT_BYTE_LIMIT = 52428800;
3160
+ function declaredCount(doc) {
3161
+ return doc.kind === "index" ? doc.childLocs.length : doc.urlCount;
3162
+ }
3163
+ function oversizedDocuments(site) {
3164
+ return (site.discovery?.documents ?? []).filter(
3165
+ (doc) => doc.kind !== "unparsable" && (declaredCount(doc) > DOCUMENT_URL_LIMIT || doc.byteLength > DOCUMENT_BYTE_LIMIT)
2899
3166
  );
2900
3167
  }
2901
- function deriveLocaleAxis(input) {
2902
- const explicit = new Set(
2903
- (input.explicit ?? []).map((l) => l.trim().toLowerCase()).filter(Boolean)
3168
+ var sitemapLimitsExceeded = {
3169
+ id: "sitemap.limits.exceeded",
3170
+ severity: "error",
3171
+ summary: "No single sitemap document may exceed 50,000 entries or 50 MB",
3172
+ rigor: "spec-required",
3173
+ sources: ["sitemaps-protocol"],
3174
+ appliesTo: (site) => oversizedDocuments(site).length > 0,
3175
+ check: ({ site, issue }) => oversizedDocuments(site).map((doc) => {
3176
+ const count2 = declaredCount(doc);
3177
+ const noun = doc.kind === "index" ? "child sitemaps" : "URLs";
3178
+ const over = [];
3179
+ if (count2 > DOCUMENT_URL_LIMIT) {
3180
+ over.push(`${count2.toLocaleString("en-US")} ${noun} against a ceiling of 50,000`);
3181
+ }
3182
+ if (doc.byteLength > DOCUMENT_BYTE_LIMIT) {
3183
+ over.push(
3184
+ `${(doc.byteLength / 1048576).toFixed(1)} MB uncompressed against a ceiling of 50 MB`
3185
+ );
3186
+ }
3187
+ return issue({
3188
+ pageUrl: doc.url,
3189
+ message: `This sitemap document declares ${over.join(" and ")}. A consumer may stop reading at the limit, so everything past it is published in name only \u2014 split the document and reference the parts from an index.`,
3190
+ origin: { kind: "computed" }
3191
+ });
3192
+ })
3193
+ };
3194
+ function outOfScopeEntries(site) {
3195
+ const out = [];
3196
+ for (const entry of site.discovery?.urls ?? []) {
3197
+ if (!entry.documentUrl) continue;
3198
+ let scope;
3199
+ let loc;
3200
+ try {
3201
+ scope = new URL(entry.documentUrl);
3202
+ loc = new URL(entry.loc);
3203
+ } catch {
3204
+ continue;
3205
+ }
3206
+ const prefix = scope.pathname.replace(/[^/]*$/, "");
3207
+ if (prefix === "/") continue;
3208
+ if (loc.origin !== scope.origin) continue;
3209
+ if (loc.pathname.startsWith(prefix)) continue;
3210
+ out.push({ loc: entry.loc, from: entry.documentUrl });
3211
+ }
3212
+ return out;
3213
+ }
3214
+ var sitemapEntryOutOfScope = {
3215
+ id: "sitemap.entry.out-of-scope",
3216
+ severity: "error",
3217
+ summary: "A sitemap may only list URLs under its own directory",
3218
+ rigor: "spec-required",
3219
+ sources: ["sitemaps-protocol"],
3220
+ appliesTo: (site) => outOfScopeEntries(site).length > 0,
3221
+ check: ({ site, issue }) => {
3222
+ const bad = outOfScopeEntries(site);
3223
+ const byDocument = /* @__PURE__ */ new Map();
3224
+ for (const { loc, from } of bad) {
3225
+ byDocument.set(from, [...byDocument.get(from) ?? [], loc]);
3226
+ }
3227
+ return [...byDocument].map(
3228
+ ([from, locs]) => issue({
3229
+ pageUrl: from,
3230
+ message: `${locs.length} entr${locs.length === 1 ? "y is" : "ies are"} outside this sitemap's directory \`${new URL(from).pathname.replace(/[^/]*$/, "")}\`: ${sample(locs)}. A sitemap only speaks for the path it is served from, so a consumer may drop ${locs.length === 1 ? "it" : "them"} \u2014 serve the document from the root, or move the entries into a sitemap that covers them.`,
3231
+ origin: { kind: "computed" }
3232
+ })
3233
+ );
3234
+ }
3235
+ };
3236
+ var sitemapEntryCrossHost = {
3237
+ id: "sitemap.entry.cross-host",
3238
+ severity: "error",
3239
+ summary: "A sitemap should only list URLs on its own host",
3240
+ rigor: "spec-required",
3241
+ sources: ["sitemaps-protocol"],
3242
+ // `www` and the apex are different hosts to a consumer, which is the case
3243
+ // this actually catches: a sitemap generated against one and served on the
3244
+ // other. The cross-submission escape hatch needs the other host's robots.txt
3245
+ // and arrives with the cross-artefact rules.
3246
+ appliesTo: (site) => crossHostEntries(site).length > 0,
3247
+ check: ({ site, issue }) => {
3248
+ const bad = crossHostEntries(site);
3249
+ return issue({
3250
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3251
+ message: `${bad.length} entr${bad.length === 1 ? "y names a host" : "ies name hosts"} other than this sitemap's: ${sample(bad)}. A consumer may drop them \u2014 the sitemap only speaks for the host that serves it.`,
3252
+ origin: { kind: "computed" }
3253
+ });
3254
+ }
3255
+ };
3256
+ var sitemapEntryProtocolMismatch = {
3257
+ id: "sitemap.entry.protocol-mismatch",
3258
+ severity: "warning",
3259
+ summary: "A sitemap should not mix `http` and `https` entries",
3260
+ rigor: "spec-required",
3261
+ sources: ["sitemaps-protocol"],
3262
+ appliesTo: (site) => protocolsIn(site).size > 1,
3263
+ check: ({ site, issue }) => {
3264
+ const protocols = [...protocolsIn(site)].sort();
3265
+ return issue({
3266
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3267
+ message: `The sitemap lists both ${protocols.join(" and ")} URLs. One of the two sets names pages the site does not serve at those addresses, and a consumer has no way to tell which.`,
3268
+ origin: { kind: "computed" }
3269
+ });
3270
+ }
3271
+ };
3272
+ var sitemapLastmodInvalid = {
3273
+ id: "sitemap.lastmod.invalid",
3274
+ severity: "warning",
3275
+ summary: "`<lastmod>` must be a W3C Datetime, and must not be in the future",
3276
+ rigor: "spec-required",
3277
+ sources: ["sitemaps-protocol", "google-sitemaps"],
3278
+ appliesTo: (site) => badLastmods(site).length > 0,
3279
+ check: ({ site, issue }) => {
3280
+ const bad = badLastmods(site);
3281
+ const malformed = bad.filter((e) => e.reason === "malformed");
3282
+ const future = bad.filter((e) => e.reason === "future");
3283
+ const parts = [];
3284
+ if (malformed.length > 0) {
3285
+ parts.push(
3286
+ `${malformed.length} not a W3C Datetime (${sample(malformed.map((e) => e.value))})`
3287
+ );
3288
+ }
3289
+ if (future.length > 0) {
3290
+ parts.push(`${future.length} dated in the future (${sample(future.map((e) => e.value))})`);
3291
+ }
3292
+ return issue({
3293
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3294
+ message: `\`<lastmod>\` values a consumer cannot use: ${parts.join("; ")}. Google ignores the field entirely when it stops trusting it, so one bad batch costs the whole site the signal.`,
3295
+ origin: { kind: "computed" }
3296
+ });
3297
+ }
3298
+ };
3299
+ var sitemapFieldInvalid = {
3300
+ id: "sitemap.field.invalid",
3301
+ severity: "warning",
3302
+ summary: "`<changefreq>` and `<priority>` must hold the values the protocol defines",
3303
+ rigor: "spec-required",
3304
+ sources: ["sitemaps-protocol", "google-sitemaps"],
3305
+ appliesTo: (site) => badFields(site).length > 0,
3306
+ check: ({ site, issue }) => {
3307
+ const bad = badFields(site);
3308
+ return issue({
3309
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3310
+ message: `${bad.length} entr${bad.length === 1 ? "y carries a field" : "ies carry fields"} outside the protocol's values: ${sample(bad)}. Google ignores both fields either way \u2014 so this is worth fixing or deleting, never worth trusting.`,
3311
+ origin: { kind: "computed" }
3312
+ });
3313
+ }
3314
+ };
3315
+ function sitemapHost(site) {
3316
+ const url = site.discovery?.diagnostics.sitemapUrl;
3317
+ try {
3318
+ return url ? new URL(url).host : new URL(site.origin).host;
3319
+ } catch {
3320
+ return void 0;
3321
+ }
3322
+ }
3323
+ function crossHostEntries(site) {
3324
+ const host = sitemapHost(site);
3325
+ if (!host) return [];
3326
+ return (site.discovery?.urls ?? []).filter((entry) => {
3327
+ if (!isAbsolute(entry.loc)) return false;
3328
+ try {
3329
+ return new URL(entry.loc).host !== host;
3330
+ } catch {
3331
+ return false;
3332
+ }
3333
+ }).map((entry) => entry.loc);
3334
+ }
3335
+ function protocolsIn(site) {
3336
+ const protocols = /* @__PURE__ */ new Set();
3337
+ for (const entry of site.discovery?.urls ?? []) {
3338
+ try {
3339
+ protocols.add(new URL(entry.loc).protocol.replace(":", ""));
3340
+ } catch {
3341
+ continue;
3342
+ }
3343
+ }
3344
+ return protocols;
3345
+ }
3346
+ var W3C_DATETIME = /^\d{4}(-\d{2}(-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2}))?)?)?$/;
3347
+ function badLastmods(site) {
3348
+ const out = [];
3349
+ const tomorrow = Date.now() + 24 * 60 * 60 * 1e3;
3350
+ for (const entry of site.discovery?.urls ?? []) {
3351
+ const value = entry.lastmod?.trim();
3352
+ if (!value) continue;
3353
+ if (!W3C_DATETIME.test(value)) {
3354
+ out.push({ value, reason: "malformed" });
3355
+ continue;
3356
+ }
3357
+ const parsed = Date.parse(value);
3358
+ if (Number.isFinite(parsed) && parsed > tomorrow) out.push({ value, reason: "future" });
3359
+ }
3360
+ return out;
3361
+ }
3362
+ function badFields(site) {
3363
+ const out = [];
3364
+ for (const entry of site.discovery?.urls ?? []) {
3365
+ const changefreq = entry.changefreq?.trim().toLowerCase();
3366
+ if (changefreq && !CHANGEFREQ.has(changefreq)) {
3367
+ out.push(`${entry.loc} \u2014 changefreq \`${entry.changefreq}\``);
3368
+ continue;
3369
+ }
3370
+ const priority = entry.priority?.trim();
3371
+ if (priority === void 0) continue;
3372
+ const value = Number(priority);
3373
+ if (!Number.isFinite(value) || value < 0 || value > 1) {
3374
+ out.push(`${entry.loc} \u2014 priority \`${entry.priority}\``);
3375
+ }
3376
+ }
3377
+ return out;
3378
+ }
3379
+ function sameUrlKey(url) {
3380
+ try {
3381
+ const parsed = new URL(url);
3382
+ parsed.hash = "";
3383
+ const path = parsed.pathname.replace(/\/+$/, "");
3384
+ return `${parsed.origin}${path || "/"}${parsed.search}`;
3385
+ } catch {
3386
+ return void 0;
3387
+ }
3388
+ }
3389
+ function pagesByUrl(site) {
3390
+ const byUrl = /* @__PURE__ */ new Map();
3391
+ for (const page of site.pages) {
3392
+ const key = sameUrlKey(page.fetch.finalUrl);
3393
+ if (key) byUrl.set(key, page);
3394
+ }
3395
+ return byUrl;
3396
+ }
3397
+ function saysNoindex(page) {
3398
+ if (metaRobotsTokens(page).has("noindex")) return true;
3399
+ const header = page.fetch.headers["x-robots-tag"] ?? "";
3400
+ return header.split(",").map((token) => token.trim().toLowerCase()).includes("noindex");
3401
+ }
3402
+ var sitemapEntryBlockedByRobots = {
3403
+ id: "sitemap.entry.blocked-by-robots",
3404
+ severity: "error",
3405
+ summary: "A sitemap must not list URLs that robots.txt forbids fetching",
3406
+ rigor: "vendor-spec",
3407
+ sources: ["ietf-rfc9309", "sitemaps-protocol"],
3408
+ // Skipped when the whole origin is disallowed: `robots.blocks-site` is that
3409
+ // finding, and repeating it once per sitemap entry would bury it.
3410
+ appliesTo: (site) => site.robots?.found === true && robotsAllows(site.robots.groups, "/").allowed && (site.discovery?.urls.length ?? 0) > 0,
3411
+ check: ({ site, issue }) => {
3412
+ const probe = site.robots;
3413
+ const blocked = [];
3414
+ for (const entry of site.discovery.urls) {
3415
+ let path;
3416
+ try {
3417
+ const url = new URL(entry.loc);
3418
+ path = `${url.pathname}${url.search}`;
3419
+ } catch {
3420
+ continue;
3421
+ }
3422
+ if (!robotsAllows(probe.groups, path).allowed) blocked.push(entry.loc);
3423
+ }
3424
+ if (blocked.length === 0) return [];
3425
+ return issue({
3426
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3427
+ message: `${blocked.length} sitemap entr${blocked.length === 1 ? "y is" : "ies are"} disallowed by \`robots.txt\`: ${sample(blocked)}. The sitemap says "index this" and robots.txt says "never fetch it" \u2014 both cannot hold, and robots.txt is the one that decides.`,
3428
+ origin: { kind: "computed" }
3429
+ });
3430
+ }
3431
+ };
3432
+ var sitemapEntryNoindex = {
3433
+ id: "sitemap.entry.noindex",
3434
+ severity: "warning",
3435
+ summary: "A sitemap must not list URLs that ask not to be indexed",
3436
+ rigor: "vendor-spec",
3437
+ sources: ["sitemaps-protocol", "google-robots-meta"],
3438
+ // Only the crawled pages can be judged: a sitemap entry goflag never fetched
3439
+ // has no `noindex` to have seen, and assuming one either way would invent a
3440
+ // finding or hide one.
3441
+ appliesTo: (site) => (site.discovery?.urls.length ?? 0) > 0,
3442
+ check: ({ site, issue }) => {
3443
+ const byUrl = pagesByUrl(site);
3444
+ const conflicting = [];
3445
+ for (const entry of site.discovery.urls) {
3446
+ const key = sameUrlKey(entry.loc);
3447
+ const page = key ? byUrl.get(key) : void 0;
3448
+ if (page && saysNoindex(page)) conflicting.push(entry.loc);
3449
+ }
3450
+ if (conflicting.length === 0) return [];
3451
+ return issue({
3452
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3453
+ message: `${conflicting.length} sitemap entr${conflicting.length === 1 ? "y declares" : "ies declare"} \`noindex\`: ${sample(conflicting)}. "Please index this" and "do not index this" are the same site's two answers to one question.`,
3454
+ origin: { kind: "computed" }
3455
+ });
3456
+ }
3457
+ };
3458
+ var sitemapEntryNonCanonical = {
3459
+ id: "sitemap.entry.non-canonical",
3460
+ severity: "warning",
3461
+ summary: "A sitemap should list canonical URLs",
3462
+ rigor: "vendor-spec",
3463
+ sources: ["sitemaps-protocol", "google-canonicalization"],
3464
+ appliesTo: (site) => (site.discovery?.urls.length ?? 0) > 0,
3465
+ check: ({ site, issue }) => {
3466
+ const byUrl = pagesByUrl(site);
3467
+ const wrong = [];
3468
+ for (const entry of site.discovery.urls) {
3469
+ const key = sameUrlKey(entry.loc);
3470
+ const page = key ? byUrl.get(key) : void 0;
3471
+ const canonical = page?.links.canonical;
3472
+ if (!page || !canonical) continue;
3473
+ const canonicalKey = sameUrlKey(canonical);
3474
+ if (canonicalKey && canonicalKey !== key) {
3475
+ wrong.push(`${entry.loc} \u2192 ${canonical}`);
3476
+ }
3477
+ }
3478
+ if (wrong.length === 0) return [];
3479
+ return issue({
3480
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3481
+ message: `${wrong.length} sitemap entr${wrong.length === 1 ? "y names a page whose canonical" : "ies name pages whose canonicals"} point elsewhere: ${sample(wrong)}. The sitemap is a list of what to index, so it should name the URL the site itself prefers.`,
3482
+ origin: { kind: "computed" }
3483
+ });
3484
+ }
3485
+ };
3486
+ var sitemapOrphans = {
3487
+ id: "sitemap.orphans",
3488
+ severity: "warning",
3489
+ summary: "Indexable pages the crawl found should be listed in the sitemap",
3490
+ rigor: "guideline",
3491
+ sources: ["sitemaps-protocol", "google-sitemaps"],
3492
+ // One finding with a count and a sample, not one per page — the same shape
3493
+ // translation holes take, and for the same reason: forty repeats of one
3494
+ // omission is noise, and the omission is a property of the sitemap.
3495
+ appliesTo: (site) => site.discovery?.diagnostics.found === true && site.discovery.urls.length > 0,
3496
+ check: ({ site, issue }) => {
3497
+ const listed = new Set(
3498
+ site.discovery.urls.map((entry) => sameUrlKey(entry.loc)).filter(Boolean)
3499
+ );
3500
+ const orphans = site.pages.filter((page) => !saysNoindex(page)).map((page) => ({ page, key: sameUrlKey(page.fetch.finalUrl) })).filter(({ key }) => key !== void 0 && !listed.has(key)).map(({ page }) => page.fetch.finalUrl);
3501
+ if (orphans.length === 0) return [];
3502
+ return issue({
3503
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3504
+ message: `${orphans.length} crawled page${orphans.length === 1 ? "" : "s"} ask${orphans.length === 1 ? "s" : ""} to be indexed and ${orphans.length === 1 ? "is" : "are"} absent from the sitemap: ${sample(orphans)}. A consumer that reads the sitemap rather than following links will never see ${orphans.length === 1 ? "it" : "them"}.`,
3505
+ origin: { kind: "computed" }
3506
+ });
3507
+ }
3508
+ };
3509
+ function entryProbes(site) {
3510
+ return [...site.sitemapEntries?.byUrl.values() ?? []];
3511
+ }
3512
+ function coverageNote(site) {
3513
+ const unprobed = site.sitemapEntries?.unprobed ?? 0;
3514
+ if (unprobed === 0) return "";
3515
+ return ` ${unprobed} further entr${unprobed === 1 ? "y was" : "ies were"} not checked \u2014 this count is a floor, not a total.`;
3516
+ }
3517
+ var sitemapEntryUnreachable = {
3518
+ id: "sitemap.entry.unreachable",
3519
+ severity: "error",
3520
+ summary: "Every URL a sitemap lists must answer",
3521
+ rigor: "vendor-spec",
3522
+ sources: ["sitemaps-protocol", "google-sitemaps"],
3523
+ appliesTo: (site) => entryProbes(site).some((probe) => isDead(probe.status)),
3524
+ check: ({ site, issue }) => {
3525
+ const dead = entryProbes(site).filter((probe) => isDead(probe.status));
3526
+ return issue({
3527
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3528
+ message: `${dead.length} sitemap entr${dead.length === 1 ? "y does" : "ies do"} not answer: ${sample(dead.map((probe) => `${probe.url} (${probe.status === 0 ? "no response" : `HTTP ${probe.status}`})`))}. A sitemap is a list of pages to index, so every dead entry spends crawl budget on a promise the site does not keep.${coverageNote(site)}`,
3529
+ origin: { kind: "computed" }
3530
+ });
3531
+ }
3532
+ };
3533
+ var sitemapEntryRedirects = {
3534
+ id: "sitemap.entry.redirects",
3535
+ severity: "warning",
3536
+ summary: "A sitemap should list final URLs, not URLs that redirect",
3537
+ rigor: "guideline",
3538
+ sources: ["sitemaps-protocol", "google-sitemaps"],
3539
+ appliesTo: (site) => entryProbes(site).some((probe) => probe.redirected && !isDead(probe.status)),
3540
+ check: ({ site, issue }) => {
3541
+ const moved = entryProbes(site).filter((probe) => probe.redirected && !isDead(probe.status));
3542
+ return issue({
3543
+ pageUrl: site.discovery.diagnostics.sitemapUrl ?? `${site.origin}/sitemap.xml`,
3544
+ message: `${moved.length} sitemap entr${moved.length === 1 ? "y redirects" : "ies redirect"}: ${sample(moved.map((probe) => `${probe.url} \u2192 ${probe.finalUrl}`))}. Google asks for the final URL, and every hop is crawl budget spent plus one more chance for a consumer to disagree about which address is the page.${coverageNote(site)}`,
3545
+ origin: { kind: "computed" }
3546
+ });
3547
+ }
3548
+ };
3549
+ function isDead(status) {
3550
+ return status === 0 || status >= 400;
3551
+ }
3552
+ function isRobotsFailure(status) {
3553
+ return status === 0 || status >= 500;
3554
+ }
3555
+ function isAbsolute(value) {
3556
+ try {
3557
+ const url = new URL(value);
3558
+ return url.protocol === "http:" || url.protocol === "https:";
3559
+ } catch {
3560
+ return false;
3561
+ }
3562
+ }
3563
+ var SITE_RULES = [
3564
+ hreflangMissing,
3565
+ hreflangClusterIncomplete,
3566
+ iconsIcoMissing,
3567
+ robotsBlocksPage,
3568
+ robotsBlocksSite,
3569
+ robotstxtCrossOrigin,
3570
+ robotstxtInvalidLine,
3571
+ robotstxtOversized,
3572
+ robotstxtSitemapRelative,
3573
+ robotstxtUnknownDirective,
3574
+ robotstxtUnreachable,
3575
+ sitemapEmpty,
3576
+ sitemapEntryBlockedByRobots,
3577
+ sitemapEntryCrossHost,
3578
+ sitemapEntryInvalidUrl,
3579
+ sitemapEntryNoindex,
3580
+ sitemapEntryNonCanonical,
3581
+ sitemapEntryOutOfScope,
3582
+ sitemapEntryProtocolMismatch,
3583
+ sitemapEntryRedirects,
3584
+ sitemapEntryUnreachable,
3585
+ sitemapFieldInvalid,
3586
+ sitemapIndexChildError,
3587
+ sitemapLastmodInvalid,
3588
+ sitemapLimitsExceeded,
3589
+ sitemapMissing,
3590
+ sitemapOrphans,
3591
+ sitemapUnparsable
3592
+ ];
3593
+ function getSiteRule(id) {
3594
+ return SITE_RULES.find((rule) => rule.id === id);
3595
+ }
3596
+
3597
+ // src/lib/core/lint-site.ts
3598
+ function lintSite(site, rules = SITE_RULES) {
3599
+ const drafts = [];
3600
+ for (const rule of rules) {
3601
+ if (rule.appliesTo && !rule.appliesTo(site)) continue;
3602
+ let result;
3603
+ try {
3604
+ result = rule.check({
3605
+ site,
3606
+ issue: (input) => ({
3607
+ ruleId: rule.id,
3608
+ severity: input.severity ?? rule.severity,
3609
+ message: input.message,
3610
+ origin: input.origin,
3611
+ fix: input.fix,
3612
+ suggestion: input.suggestion,
3613
+ pageUrl: input.pageUrl,
3614
+ // Stamped from the descriptor, never from the check: a rule does not
3615
+ // get to claim a rigor per finding. Absent on the three rules that
3616
+ // predate the field, which is how the gap stays visible instead of
3617
+ // being filled with a plausible citation.
3618
+ ...rule.rigor ? { rigor: rule.rigor, sources: rule.sources ?? [] } : {}
3619
+ })
3620
+ });
3621
+ } catch (err) {
3622
+ drafts.push({
3623
+ ruleId: "engine.site-rule-crashed",
3624
+ severity: "info",
3625
+ message: `Site rule \`${rule.id}\` threw: ${err instanceof Error ? err.message : String(err)}`,
3626
+ pageUrl: site.origin
3627
+ });
3628
+ continue;
3629
+ }
3630
+ if (!result) continue;
3631
+ if (Array.isArray(result)) drafts.push(...result);
3632
+ else drafts.push(result);
3633
+ }
3634
+ return sortSiteIssues(drafts);
3635
+ }
3636
+ function sortSiteIssues(drafts) {
3637
+ const byPage = [...drafts].sort((a, b) => a.pageUrl.localeCompare(b.pageUrl));
3638
+ return sortIssues(byPage);
3639
+ }
3640
+
3641
+ // src/lib/core/locales.ts
3642
+ function localePrefixOf(url) {
3643
+ let pathname;
3644
+ try {
3645
+ pathname = new URL(url).pathname;
3646
+ } catch {
3647
+ return null;
3648
+ }
3649
+ const first = pathname.split("/").filter(Boolean)[0];
3650
+ if (first && looksLikeLocaleSegment(first)) return first.toLowerCase();
3651
+ return null;
3652
+ }
3653
+ function prefixesIn(urls) {
3654
+ const out = /* @__PURE__ */ new Set();
3655
+ for (const url of urls ?? []) {
3656
+ const locale = localePrefixOf(url);
3657
+ if (locale) out.add(locale);
3658
+ }
3659
+ out.delete("x-default");
3660
+ return out;
3661
+ }
3662
+ function describeCandidates(pages) {
3663
+ const byTag = /* @__PURE__ */ new Map();
3664
+ for (const page of pages) {
3665
+ const tag = localePrefixOf(page.fetch.finalUrl);
3666
+ if (!tag || tag === "x-default") continue;
3667
+ const entry = byTag.get(tag) ?? { pages: 0, langs: /* @__PURE__ */ new Set() };
3668
+ entry.pages += 1;
3669
+ const lang = page.raw.htmlLang?.trim().toLowerCase();
3670
+ if (lang) entry.langs.add(lang);
3671
+ byTag.set(tag, entry);
3672
+ }
3673
+ const candidates = [];
3674
+ for (const [tag, { pages: n, langs }] of byTag) {
3675
+ const observedLangs = [...langs].sort();
3676
+ const htmlLangAgrees = observedLangs.length === 0 ? void 0 : observedLangs.some((l) => l.split("-")[0] === tag.split("-")[0]);
3677
+ candidates.push({
3678
+ tag,
3679
+ pages: n,
3680
+ isKnownLanguage: isKnownLanguageTag(tag),
3681
+ htmlLangAgrees,
3682
+ observedLangs
3683
+ });
3684
+ }
3685
+ return candidates.sort(
3686
+ (a, b) => Number(b.htmlLangAgrees ?? false) - Number(a.htmlLangAgrees ?? false) || Number(b.isKnownLanguage) - Number(a.isKnownLanguage) || b.pages - a.pages || a.tag.localeCompare(b.tag)
3687
+ );
3688
+ }
3689
+ function deriveLocaleAxis(input) {
3690
+ const explicit = new Set(
3691
+ (input.explicit ?? []).map((l) => l.trim().toLowerCase()).filter(Boolean)
2904
3692
  );
2905
3693
  explicit.delete("x-default");
2906
3694
  const fromSitemap = prefixesIn(input.sitemapUrls);
@@ -3030,14 +3818,15 @@ async function discoverSitemap(baseUrl, options = {}) {
3030
3818
  };
3031
3819
  const robots = await probeRobots(origin, { signal: options.signal });
3032
3820
  diagnostics.robotsFound = robots.found;
3821
+ const declaredUrls = robots.sitemaps.map((entry) => entry.value);
3033
3822
  const candidates = [];
3034
- for (const declared of robots.sitemaps) {
3823
+ for (const declared of declaredUrls) {
3035
3824
  candidates.push({ url: declared, source: "robots", declaredInRobots: true });
3036
3825
  }
3037
3826
  candidates.push({
3038
3827
  url: new URL("/sitemap.xml", origin).toString(),
3039
3828
  source: "well-known",
3040
- declaredInRobots: robots.sitemaps.includes(new URL("/sitemap.xml", origin).toString())
3829
+ declaredInRobots: declaredUrls.includes(new URL("/sitemap.xml", origin).toString())
3041
3830
  });
3042
3831
  candidates.push({
3043
3832
  url: new URL("/sitemap_index.xml", origin).toString(),
@@ -3068,9 +3857,23 @@ async function discoverSitemap(baseUrl, options = {}) {
3068
3857
  diagnostics.atWellKnownPath = isWellKnownPath(candidate.url);
3069
3858
  const collected = [];
3070
3859
  const seen = /* @__PURE__ */ new Set();
3860
+ const documents = [];
3071
3861
  let truncated = false;
3862
+ documents.push({
3863
+ url: candidate.url,
3864
+ status: doc.status,
3865
+ byteLength: doc.byteLength,
3866
+ gzipped: doc.gzipped,
3867
+ kind: parsed.kind === "index" ? "index" : "urlset",
3868
+ childLocs: parsed.kind === "index" ? [...parsed.sitemaps] : [],
3869
+ // What the document declares, not what survived dedupe and the cap: the
3870
+ // ceiling is on the file's contents, and a post-dedupe count would answer
3871
+ // a question the protocol never asked.
3872
+ urlCount: parsed.kind === "urlset" ? parsed.urls.length : 0,
3873
+ declaredInRobots: candidate.declaredInRobots
3874
+ });
3072
3875
  if (parsed.kind === "urlset") {
3073
- truncated = pushEntries(collected, seen, parsed.urls, maxUrls);
3876
+ truncated = pushEntries(collected, seen, parsed.urls, maxUrls, candidate.url);
3074
3877
  } else {
3075
3878
  diagnostics.isIndex = true;
3076
3879
  const children = parsed.sitemaps.slice(0, maxSitemaps);
@@ -3088,17 +3891,44 @@ async function discoverSitemap(baseUrl, options = {}) {
3088
3891
  }
3089
3892
  const childDoc = await fetchDoc(childUrl, options);
3090
3893
  if (!childDoc.body) {
3894
+ documents.push({
3895
+ url: childUrl,
3896
+ status: childDoc.status,
3897
+ byteLength: childDoc.byteLength,
3898
+ gzipped: childDoc.gzipped,
3899
+ kind: "unparsable",
3900
+ childLocs: [],
3901
+ urlCount: 0,
3902
+ declaredInRobots: declaredUrls.includes(childUrl),
3903
+ parentUrl: candidate.url
3904
+ });
3091
3905
  diagnostics.childSitemapErrors += 1;
3092
3906
  diagnostics.warnings.push(`Child sitemap unreachable: ${childUrl}`);
3093
3907
  continue;
3094
3908
  }
3095
3909
  const childParsed = parseSitemap(childDoc.body);
3910
+ documents.push({
3911
+ url: childUrl,
3912
+ status: childDoc.status,
3913
+ byteLength: childDoc.byteLength,
3914
+ gzipped: childDoc.gzipped,
3915
+ // A child that is itself an index is recorded as one. This run still
3916
+ // treats it as an error below, because that is what it has always
3917
+ // done and replacing that verdict needs a source no specification
3918
+ // provides — see `docs/sitemap-robots-plan.md` §4.3. The tree now
3919
+ // says which of the two it was, which is the part that was lost.
3920
+ kind: childParsed.kind === "unknown" ? "unparsable" : childParsed.kind,
3921
+ childLocs: childParsed.kind === "index" ? [...childParsed.sitemaps] : [],
3922
+ urlCount: childParsed.kind === "urlset" ? childParsed.urls.length : 0,
3923
+ declaredInRobots: declaredUrls.includes(childUrl),
3924
+ parentUrl: candidate.url
3925
+ });
3096
3926
  if (childParsed.kind !== "urlset") {
3097
3927
  diagnostics.childSitemapErrors += 1;
3098
3928
  diagnostics.warnings.push(`Child sitemap not a urlset: ${childUrl}`);
3099
3929
  continue;
3100
3930
  }
3101
- if (pushEntries(collected, seen, childParsed.urls, maxUrls)) truncated = true;
3931
+ if (pushEntries(collected, seen, childParsed.urls, maxUrls, childUrl)) truncated = true;
3102
3932
  }
3103
3933
  }
3104
3934
  diagnostics.urlCount = collected.length;
@@ -3110,6 +3940,7 @@ async function discoverSitemap(baseUrl, options = {}) {
3110
3940
  baseUrl,
3111
3941
  source: candidate.source,
3112
3942
  urls: collected,
3943
+ documents,
3113
3944
  diagnostics,
3114
3945
  truncated
3115
3946
  };
@@ -3118,95 +3949,497 @@ async function discoverSitemap(baseUrl, options = {}) {
3118
3949
  return crawlSite(baseUrl, origin, options, diagnostics);
3119
3950
  }
3120
3951
  diagnostics.warnings.push("No sitemap found.");
3121
- return { origin, baseUrl, source: "well-known", urls: [], diagnostics, truncated: false };
3952
+ return {
3953
+ origin,
3954
+ baseUrl,
3955
+ source: "well-known",
3956
+ urls: [],
3957
+ documents: [],
3958
+ diagnostics,
3959
+ truncated: false
3960
+ };
3961
+ }
3962
+ async function crawlSite(baseUrl, origin, options, diagnostics) {
3963
+ const result = await crawl({
3964
+ entryUrl: baseUrl,
3965
+ depth: options.crawlDepth ?? DEFAULTS.crawlDepth,
3966
+ maxPages: options.crawlMaxPages ?? DEFAULTS.crawlMaxPages,
3967
+ inspectOptions: {
3968
+ mode: "static",
3969
+ probes: false,
3970
+ allowInsecureTls: options.allowInsecureTls,
3971
+ signal: options.signal
3972
+ }
3973
+ });
3974
+ const urls = dedupe(result.visited).map((loc) => ({ loc }));
3975
+ diagnostics.urlCount = urls.length;
3976
+ diagnostics.warnings.push(
3977
+ diagnostics.found ? "Sitemap found but lists no URLs \u2014 crawled the site instead." : "No sitemap found \u2014 URLs were discovered by crawling links."
3978
+ );
3979
+ return {
3980
+ origin,
3981
+ baseUrl,
3982
+ source: "crawl",
3983
+ urls,
3984
+ // No document declared these: the crawl found them by following links. An
3985
+ // empty tree is the honest answer, and the rules that read it are the ones
3986
+ // that judge documents — which this run has none of.
3987
+ documents: [],
3988
+ diagnostics,
3989
+ truncated: result.truncated
3990
+ };
3991
+ }
3992
+ function pushEntries(out, seen, entries, cap, documentUrl) {
3993
+ for (const entry of entries) {
3994
+ if (out.length >= cap) return true;
3995
+ if (seen.has(entry.loc)) continue;
3996
+ seen.add(entry.loc);
3997
+ out.push({ ...entry, documentUrl });
3998
+ }
3999
+ return out.length >= cap;
4000
+ }
4001
+ function dedupe(urls) {
4002
+ return Array.from(new Set(urls));
4003
+ }
4004
+ function isWellKnownPath(url) {
4005
+ try {
4006
+ const path = new URL(url).pathname.toLowerCase();
4007
+ return path === "/sitemap.xml" || path === "/sitemap_index.xml";
4008
+ } catch {
4009
+ return false;
4010
+ }
4011
+ }
4012
+ async function fetchDoc(url, options) {
4013
+ const { signal, cleanup } = combineSignals(
4014
+ options.signal,
4015
+ options.timeoutMs ?? DEFAULTS.timeoutMs
4016
+ );
4017
+ const restoreTls = relaxTlsIfRequested2(options.allowInsecureTls);
4018
+ try {
4019
+ const res = await fetch(url, {
4020
+ signal,
4021
+ redirect: "follow",
4022
+ headers: {
4023
+ accept: "application/xml,text/xml,application/gzip;q=0.9,*/*;q=0.5"
4024
+ }
4025
+ });
4026
+ if (!res.ok) return { status: res.status, byteLength: 0, gzipped: false };
4027
+ const encoding = res.headers.get("content-encoding")?.toLowerCase() ?? "";
4028
+ const looksGzip = url.toLowerCase().endsWith(".gz") || encoding.includes("gzip");
4029
+ if (looksGzip) {
4030
+ const buf = Buffer.from(await res.arrayBuffer());
4031
+ try {
4032
+ return withLength(res.status, gunzipSync(buf).toString("utf8"), true);
4033
+ } catch {
4034
+ return withLength(res.status, buf.toString("utf8"), true);
4035
+ }
4036
+ }
4037
+ return withLength(res.status, await res.text(), false);
4038
+ } catch (err) {
4039
+ const reason = err instanceof Error ? err.message : String(err);
4040
+ return { status: 0, error: reason || "network error", byteLength: 0, gzipped: false };
4041
+ } finally {
4042
+ cleanup();
4043
+ restoreTls();
4044
+ }
4045
+ }
4046
+ function withLength(status, body, gzipped) {
4047
+ return { status, body, byteLength: Buffer.byteLength(body, "utf8"), gzipped };
4048
+ }
4049
+ function relaxTlsIfRequested2(enabled) {
4050
+ if (!enabled) return () => void 0;
4051
+ const previous = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
4052
+ process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
4053
+ return () => {
4054
+ if (previous === void 0) delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
4055
+ else process.env.NODE_TLS_REJECT_UNAUTHORIZED = previous;
4056
+ };
4057
+ }
4058
+
4059
+ // src/lib/core/net/fetch-url.ts
4060
+ var DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
4061
+ var DEFAULT_TIMEOUT_MS = 8e3;
4062
+ var DEFAULT_MAX_BYTES = 3 * 1024 * 1024;
4063
+ var DEFAULT_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
4064
+ var TEXTY = /(text\/|application\/(xhtml\+xml|xml|json|.*\+xml|.*\+json|javascript))/;
4065
+ async function fetchUrl(url, options = {}) {
4066
+ const method = options.method ?? "GET";
4067
+ const redirect = options.redirect ?? "follow";
4068
+ const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
4069
+ const startedAt = Date.now();
4070
+ const { signal, cleanup } = combineSignals(
4071
+ options.signal,
4072
+ options.timeoutMs ?? DEFAULT_TIMEOUT_MS
4073
+ );
4074
+ const restoreTls = relaxTlsIfRequested3(options.allowInsecureTls);
4075
+ try {
4076
+ const res = await fetch(url, {
4077
+ method,
4078
+ signal,
4079
+ redirect,
4080
+ headers: {
4081
+ "user-agent": options.userAgent ?? DEFAULT_USER_AGENT,
4082
+ accept: options.accept ?? DEFAULT_ACCEPT
4083
+ }
4084
+ });
4085
+ const contentType = normaliseContentType(res.headers.get("content-type"));
4086
+ const result = {
4087
+ requestedUrl: url,
4088
+ finalUrl: res.url || url,
4089
+ status: res.status,
4090
+ redirected: res.redirected,
4091
+ redirectChain: [],
4092
+ contentType,
4093
+ durationMs: Date.now() - startedAt
4094
+ };
4095
+ const retryAfter = res.headers.get("retry-after");
4096
+ if (retryAfter) result.retryAfter = retryAfter;
4097
+ if (redirect === "manual" && isRedirectStatus(res.status)) {
4098
+ const location = res.headers.get("location");
4099
+ if (location) {
4100
+ try {
4101
+ const next = new URL(location, url).toString();
4102
+ result.finalUrl = next;
4103
+ result.redirectChain = [next];
4104
+ result.redirected = true;
4105
+ } catch {
4106
+ }
4107
+ }
4108
+ await res.body?.cancel().catch(() => void 0);
4109
+ return result;
4110
+ }
4111
+ if (method === "HEAD") {
4112
+ await res.body?.cancel().catch(() => void 0);
4113
+ return result;
4114
+ }
4115
+ if (contentType && !TEXTY.test(contentType)) {
4116
+ await res.body?.cancel().catch(() => void 0);
4117
+ return result;
4118
+ }
4119
+ const { text, truncated } = await readCappedText(res, maxBytes);
4120
+ result.body = text;
4121
+ if (truncated) result.truncated = true;
4122
+ return result;
4123
+ } catch (err) {
4124
+ return {
4125
+ requestedUrl: url,
4126
+ finalUrl: url,
4127
+ status: 0,
4128
+ redirected: false,
4129
+ redirectChain: [],
4130
+ reason: classifyError(err, options.signal),
4131
+ durationMs: Date.now() - startedAt
4132
+ };
4133
+ } finally {
4134
+ cleanup();
4135
+ restoreTls();
4136
+ }
4137
+ }
4138
+ function isRedirectStatus(status) {
4139
+ return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
4140
+ }
4141
+ function normaliseContentType(raw) {
4142
+ if (!raw) return void 0;
4143
+ const semi = raw.indexOf(";");
4144
+ return (semi === -1 ? raw : raw.slice(0, semi)).trim().toLowerCase() || void 0;
4145
+ }
4146
+ async function readCappedText(res, maxBytes) {
4147
+ const reader = res.body?.getReader();
4148
+ if (!reader) {
4149
+ const text = await res.text();
4150
+ if (text.length > maxBytes) {
4151
+ return { text: text.slice(0, maxBytes), truncated: true };
4152
+ }
4153
+ return { text, truncated: false };
4154
+ }
4155
+ const chunks = [];
4156
+ let received = 0;
4157
+ let truncated = false;
4158
+ for (; ; ) {
4159
+ const { done, value } = await reader.read();
4160
+ if (done) break;
4161
+ if (!value) continue;
4162
+ chunks.push(value);
4163
+ received += value.byteLength;
4164
+ if (received >= maxBytes) {
4165
+ truncated = true;
4166
+ await reader.cancel().catch(() => void 0);
4167
+ break;
4168
+ }
4169
+ }
4170
+ const buf = Buffer.concat(chunks.map((c) => Buffer.from(c)));
4171
+ const sliced = buf.byteLength > maxBytes ? buf.subarray(0, maxBytes) : buf;
4172
+ return { text: sliced.toString("utf8"), truncated };
4173
+ }
4174
+ function classifyError(err, callerSignal) {
4175
+ if (isAbortError(err)) {
4176
+ return callerSignal?.aborted ? "abort" : "timeout";
4177
+ }
4178
+ const code = errorCode(err);
4179
+ if (code) {
4180
+ if (code === "ENOTFOUND" || code === "EAI_AGAIN") return "dns";
4181
+ if (code.startsWith("CERT_") || code.includes("SELF_SIGNED") || code.includes("CERT")) {
4182
+ return "tls";
4183
+ }
4184
+ }
4185
+ const message = err instanceof Error ? err.message.toLowerCase() : String(err).toLowerCase();
4186
+ if (message.includes("certificate") || message.includes("tls") || message.includes("ssl")) {
4187
+ return "tls";
4188
+ }
4189
+ return "network";
4190
+ }
4191
+ function isAbortError(err) {
4192
+ return err instanceof Error && err.name === "AbortError";
4193
+ }
4194
+ function errorCode(err) {
4195
+ if (err && typeof err === "object") {
4196
+ const direct = err.code;
4197
+ if (typeof direct === "string") return direct;
4198
+ const cause = err.cause;
4199
+ if (cause && typeof cause === "object") {
4200
+ const causeCode = cause.code;
4201
+ if (typeof causeCode === "string") return causeCode;
4202
+ }
4203
+ }
4204
+ return void 0;
4205
+ }
4206
+ function relaxTlsIfRequested3(enabled) {
4207
+ if (!enabled) return () => void 0;
4208
+ const previous = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
4209
+ process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
4210
+ return () => {
4211
+ if (previous === void 0) delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
4212
+ else process.env.NODE_TLS_REJECT_UNAUTHORIZED = previous;
4213
+ };
4214
+ }
4215
+
4216
+ // src/lib/core/links/classify.ts
4217
+ function classifyLink(input) {
4218
+ if (input.status === 0) return "broken";
4219
+ if (input.loop) return "broken";
4220
+ if (input.softNotFound) return "warning";
4221
+ if (input.status === 403 || input.status === 429) return "blocked";
4222
+ if (input.status >= 400) return "broken";
4223
+ if (input.status >= 200 && input.status < 300) {
4224
+ return input.redirected ? "redirect" : "ok";
4225
+ }
4226
+ return "broken";
4227
+ }
4228
+
4229
+ // src/lib/core/links/check.ts
4230
+ var DEFAULTS2 = {
4231
+ timeoutMs: 8e3,
4232
+ maxRedirects: 10,
4233
+ retries: 1,
4234
+ backoffBaseMs: 300,
4235
+ maxBackoffMs: 3e3,
4236
+ softBodyBytes: 16 * 1024
4237
+ };
4238
+ var NOT_FOUND_RE = /\b(not\s+found|page\s+doesn'?t\s+exist|404|no\s+such\s+page)\b/i;
4239
+ var SOFT_404_BODY_LIMIT = 2048;
4240
+ async function checkLink(url, options = {}) {
4241
+ const checkedAt = (/* @__PURE__ */ new Date()).toISOString();
4242
+ const started = Date.now();
4243
+ const scheme = schemeOf(url);
4244
+ if (scheme !== "http" && scheme !== "https") {
4245
+ return {
4246
+ url,
4247
+ finalUrl: url,
4248
+ status: 0,
4249
+ verdict: "skipped",
4250
+ method: "GET",
4251
+ redirectChain: [],
4252
+ reason: scheme ? `unsupported scheme: ${scheme}` : "not a URL",
4253
+ checkedAt,
4254
+ durationMs: Date.now() - started
4255
+ };
4256
+ }
4257
+ const outcome = await probeWithRetry(url, options);
4258
+ const softNotFound = isSoftNotFound(outcome);
4259
+ const verdict = classifyLink({
4260
+ status: outcome.status,
4261
+ redirected: outcome.redirectChain.length > 0,
4262
+ reason: outcome.reason,
4263
+ softNotFound,
4264
+ loop: outcome.loop
4265
+ });
4266
+ return {
4267
+ url,
4268
+ finalUrl: outcome.finalUrl,
4269
+ status: outcome.status,
4270
+ verdict,
4271
+ method: outcome.method,
4272
+ redirectChain: outcome.redirectChain,
4273
+ reason: describeReason(outcome, softNotFound),
4274
+ checkedAt,
4275
+ durationMs: Date.now() - started
4276
+ };
4277
+ }
4278
+ async function probeWithRetry(url, options) {
4279
+ const retries = options.retries ?? DEFAULTS2.retries;
4280
+ const sleep = options.sleep ?? defaultSleep;
4281
+ let outcome = await probeWithHeadFallback(url, options);
4282
+ for (let attempt = 0; attempt < retries; attempt++) {
4283
+ if (!isTransient(outcome)) break;
4284
+ if (options.signal?.aborted) break;
4285
+ const delay = backoffDelay(outcome, attempt, options);
4286
+ if (delay > 0) await sleep(delay);
4287
+ outcome = await probeWithHeadFallback(url, options);
4288
+ }
4289
+ return outcome;
4290
+ }
4291
+ async function probeWithHeadFallback(url, options) {
4292
+ const head = await followRedirects(url, "HEAD", options);
4293
+ if (headRejected(head) || head.status >= 200 && head.status < 300) {
4294
+ const get = await followRedirects(url, "GET", options);
4295
+ if (get.status === 0 && head.status !== 0 && !headRejected(head)) return head;
4296
+ return get;
4297
+ }
4298
+ return head;
4299
+ }
4300
+ function headRejected(o) {
4301
+ return o.status === 405 || o.status === 501 || o.status === 403 || o.status === 0;
4302
+ }
4303
+ async function followRedirects(url, method, options) {
4304
+ const maxRedirects = options.maxRedirects ?? DEFAULTS2.maxRedirects;
4305
+ const chain = [];
4306
+ const visited = /* @__PURE__ */ new Set([url]);
4307
+ let current = url;
4308
+ for (let hop = 0; hop <= maxRedirects; hop++) {
4309
+ const res = await fetchUrl(current, {
4310
+ method,
4311
+ redirect: "manual",
4312
+ timeoutMs: options.timeoutMs ?? DEFAULTS2.timeoutMs,
4313
+ allowInsecureTls: options.allowInsecureTls,
4314
+ userAgent: options.userAgent,
4315
+ maxBytes: method === "GET" ? options.softBodyBytes ?? DEFAULTS2.softBodyBytes : void 0
4316
+ });
4317
+ if (res.status === 0) {
4318
+ return {
4319
+ status: 0,
4320
+ finalUrl: current,
4321
+ redirectChain: chain,
4322
+ reason: res.reason,
4323
+ loop: false,
4324
+ method
4325
+ };
4326
+ }
4327
+ const next = res.redirectChain[0];
4328
+ if (next) {
4329
+ chain.push(next);
4330
+ if (visited.has(next)) {
4331
+ return { status: res.status, finalUrl: next, redirectChain: chain, loop: true, method };
4332
+ }
4333
+ visited.add(next);
4334
+ current = next;
4335
+ continue;
4336
+ }
4337
+ return {
4338
+ status: res.status,
4339
+ finalUrl: res.finalUrl,
4340
+ redirectChain: chain,
4341
+ loop: false,
4342
+ body: res.body,
4343
+ retryAfter: res.retryAfter,
4344
+ method
4345
+ };
4346
+ }
4347
+ return { status: 0, finalUrl: current, redirectChain: chain, loop: true, method };
4348
+ }
4349
+ function isSoftNotFound(o) {
4350
+ if (o.status < 200 || o.status >= 300) return false;
4351
+ if (o.body === void 0) return false;
4352
+ if (o.body.length > SOFT_404_BODY_LIMIT) return false;
4353
+ return NOT_FOUND_RE.test(o.body);
4354
+ }
4355
+ function isTransient(o) {
4356
+ if (o.status === 0) return true;
4357
+ if (o.status === 429) return true;
4358
+ if (o.status >= 500) return true;
4359
+ return false;
3122
4360
  }
3123
- async function crawlSite(baseUrl, origin, options, diagnostics) {
3124
- const result = await crawl({
3125
- entryUrl: baseUrl,
3126
- depth: options.crawlDepth ?? DEFAULTS.crawlDepth,
3127
- maxPages: options.crawlMaxPages ?? DEFAULTS.crawlMaxPages,
3128
- inspectOptions: {
3129
- mode: "static",
3130
- probes: false,
3131
- allowInsecureTls: options.allowInsecureTls,
3132
- signal: options.signal
3133
- }
3134
- });
3135
- const urls = dedupe(result.visited).map((loc) => ({ loc }));
3136
- diagnostics.urlCount = urls.length;
3137
- diagnostics.warnings.push(
3138
- diagnostics.found ? "Sitemap found but lists no URLs \u2014 crawled the site instead." : "No sitemap found \u2014 URLs were discovered by crawling links."
3139
- );
3140
- return {
3141
- origin,
3142
- baseUrl,
3143
- source: "crawl",
3144
- urls,
3145
- diagnostics,
3146
- truncated: result.truncated
3147
- };
4361
+ function backoffDelay(o, attempt, options) {
4362
+ const max = options.maxBackoffMs ?? DEFAULTS2.maxBackoffMs;
4363
+ const retryAfterMs = parseRetryAfter(o.retryAfter);
4364
+ if (retryAfterMs !== void 0) return Math.min(retryAfterMs, max);
4365
+ const base = options.backoffBaseMs ?? DEFAULTS2.backoffBaseMs;
4366
+ const exp = base * 2 ** attempt;
4367
+ const jitter = Math.floor(Math.random() * base);
4368
+ return Math.min(exp + jitter, max);
3148
4369
  }
3149
- function pushEntries(out, seen, entries, cap) {
3150
- for (const entry of entries) {
3151
- if (out.length >= cap) return true;
3152
- if (seen.has(entry.loc)) continue;
3153
- seen.add(entry.loc);
3154
- out.push(entry);
3155
- }
3156
- return out.length >= cap;
4370
+ function parseRetryAfter(value) {
4371
+ if (!value) return void 0;
4372
+ const seconds = Number(value);
4373
+ if (Number.isFinite(seconds)) return Math.max(0, seconds * 1e3);
4374
+ const date = Date.parse(value);
4375
+ if (!Number.isNaN(date)) return Math.max(0, date - Date.now());
4376
+ return void 0;
3157
4377
  }
3158
- function dedupe(urls) {
3159
- return Array.from(new Set(urls));
4378
+ function describeReason(o, softNotFound) {
4379
+ if (o.reason) return o.reason;
4380
+ if (o.loop) return "redirect loop";
4381
+ if (softNotFound) return "soft-404";
4382
+ if (o.status === 429) return "429 rate-limited";
4383
+ if (o.status === 403) return "403 forbidden";
4384
+ if (o.status >= 400) return `${o.status}`;
4385
+ return void 0;
3160
4386
  }
3161
- function isWellKnownPath(url) {
4387
+ function schemeOf(url) {
3162
4388
  try {
3163
- const path = new URL(url).pathname.toLowerCase();
3164
- return path === "/sitemap.xml" || path === "/sitemap_index.xml";
4389
+ return new URL(url).protocol.replace(/:$/, "").toLowerCase();
3165
4390
  } catch {
3166
- return false;
4391
+ return null;
3167
4392
  }
3168
4393
  }
3169
- async function fetchDoc(url, options) {
3170
- const { signal, cleanup } = combineSignals(
3171
- options.signal,
3172
- options.timeoutMs ?? DEFAULTS.timeoutMs
3173
- );
3174
- const restoreTls = relaxTlsIfRequested2(options.allowInsecureTls);
3175
- try {
3176
- const res = await fetch(url, {
3177
- signal,
3178
- redirect: "follow",
3179
- headers: {
3180
- accept: "application/xml,text/xml,application/gzip;q=0.9,*/*;q=0.5"
3181
- }
3182
- });
3183
- if (!res.ok) return { status: res.status };
3184
- const encoding = res.headers.get("content-encoding")?.toLowerCase() ?? "";
3185
- const looksGzip = url.toLowerCase().endsWith(".gz") || encoding.includes("gzip");
3186
- if (looksGzip) {
3187
- const buf = Buffer.from(await res.arrayBuffer());
3188
- try {
3189
- return { status: res.status, body: gunzipSync(buf).toString("utf8") };
3190
- } catch {
3191
- return { status: res.status, body: buf.toString("utf8") };
3192
- }
4394
+ function defaultSleep(ms) {
4395
+ return new Promise((resolve) => setTimeout(resolve, ms));
4396
+ }
4397
+
4398
+ // src/lib/core/sitemap/entries.ts
4399
+ async function probeSitemapEntries(locs, options) {
4400
+ const byUrl = /* @__PURE__ */ new Map();
4401
+ const toFetch = [];
4402
+ for (const loc of locs) {
4403
+ if (byUrl.has(loc)) continue;
4404
+ const crawled = options.crawled.get(loc);
4405
+ if (crawled !== void 0) {
4406
+ byUrl.set(loc, { url: loc, status: crawled, via: "crawl", finalUrl: loc, redirected: false });
4407
+ continue;
3193
4408
  }
3194
- return { status: res.status, body: await res.text() };
3195
- } catch (err) {
3196
- const reason = err instanceof Error ? err.message : String(err);
3197
- return { status: 0, error: reason || "network error" };
3198
- } finally {
3199
- cleanup();
3200
- restoreTls();
4409
+ const checked = options.checked.get(loc);
4410
+ if (checked) {
4411
+ byUrl.set(loc, fromCheck(loc, checked, "link-audit"));
4412
+ continue;
4413
+ }
4414
+ toFetch.push(loc);
3201
4415
  }
4416
+ const budget = toFetch.slice(0, options.maxProbes);
4417
+ const unprobed = toFetch.length - budget.length;
4418
+ const queue = [...budget];
4419
+ const workers = Math.min(options.concurrency ?? 6, queue.length);
4420
+ await Promise.all(
4421
+ Array.from({ length: workers }, async () => {
4422
+ for (let url = queue.pop(); url !== void 0; url = queue.pop()) {
4423
+ const check = await checkLink(url, {
4424
+ signal: options.signal,
4425
+ timeoutMs: options.timeoutMs,
4426
+ allowInsecureTls: options.allowInsecureTls
4427
+ }).catch(() => void 0);
4428
+ if (check) byUrl.set(url, fromCheck(url, check, "probe"));
4429
+ }
4430
+ })
4431
+ );
4432
+ return { byUrl, unprobed };
3202
4433
  }
3203
- function relaxTlsIfRequested2(enabled) {
3204
- if (!enabled) return () => void 0;
3205
- const previous = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
3206
- process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
3207
- return () => {
3208
- if (previous === void 0) delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
3209
- else process.env.NODE_TLS_REJECT_UNAUTHORIZED = previous;
4434
+ function fromCheck(url, check, via) {
4435
+ return {
4436
+ url,
4437
+ status: check.status,
4438
+ via,
4439
+ finalUrl: check.finalUrl,
4440
+ // A redirect that resolved, or one that did not: either way the entry did
4441
+ // not name the URL that answers, which is what the rule is about.
4442
+ redirected: check.finalUrl !== url || check.redirectChain.length > 0
3210
4443
  };
3211
4444
  }
3212
4445
 
@@ -3697,161 +4930,87 @@ var PROSE_RULES = [
3697
4930
  ];
3698
4931
  var PROSE_RULE_BY_ID = new Map(PROSE_RULES.map((r) => [r.id, r]));
3699
4932
 
3700
- // src/lib/core/net/fetch-url.ts
3701
- var DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
3702
- var DEFAULT_TIMEOUT_MS = 8e3;
3703
- var DEFAULT_MAX_BYTES = 3 * 1024 * 1024;
3704
- var DEFAULT_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
3705
- var TEXTY = /(text\/|application\/(xhtml\+xml|xml|json|.*\+xml|.*\+json|javascript))/;
3706
- async function fetchUrl(url, options = {}) {
3707
- const method = options.method ?? "GET";
3708
- const redirect = options.redirect ?? "follow";
3709
- const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
3710
- const startedAt = Date.now();
3711
- const { signal, cleanup } = combineSignals(
3712
- options.signal,
3713
- options.timeoutMs ?? DEFAULT_TIMEOUT_MS
3714
- );
3715
- const restoreTls = relaxTlsIfRequested3(options.allowInsecureTls);
3716
- try {
3717
- const res = await fetch(url, {
3718
- method,
3719
- signal,
3720
- redirect,
3721
- headers: {
3722
- "user-agent": options.userAgent ?? DEFAULT_USER_AGENT,
3723
- accept: options.accept ?? DEFAULT_ACCEPT
3724
- }
3725
- });
3726
- const contentType = normaliseContentType(res.headers.get("content-type"));
3727
- const result = {
3728
- requestedUrl: url,
3729
- finalUrl: res.url || url,
3730
- status: res.status,
3731
- redirected: res.redirected,
3732
- redirectChain: [],
3733
- contentType,
3734
- durationMs: Date.now() - startedAt
3735
- };
3736
- const retryAfter = res.headers.get("retry-after");
3737
- if (retryAfter) result.retryAfter = retryAfter;
3738
- if (redirect === "manual" && isRedirectStatus(res.status)) {
3739
- const location = res.headers.get("location");
3740
- if (location) {
3741
- try {
3742
- const next = new URL(location, url).toString();
3743
- result.finalUrl = next;
3744
- result.redirectChain = [next];
3745
- result.redirected = true;
3746
- } catch {
3747
- }
4933
+ // src/lib/rules/site-prose.ts
4934
+ var hreflangSitemapMismatch = {
4935
+ id: "hreflang.sitemap-mismatch",
4936
+ summary: "`<head>` alternates advertise a locale the sitemap does not list",
4937
+ prose: "Your `<head>` advertises a translation your sitemap does not list. Is the sitemap missing an entry, or is the alternate pointing at a page you did not mean to publish?",
4938
+ why: "The two are derived from one intent by different code paths, so a disagreement means one of them is wrong. No specification says which: the declaration methods are equivalent, and a page can legitimately be cross-linked and kept out of the sitemap.",
4939
+ appliesTo: (site) => site.localeAxis.multilingual && (site.discovery?.urls.length ?? 0) > 0,
4940
+ ask: (site) => {
4941
+ const axis = new Set(site.localeAxis.locales.map((l) => l.toLowerCase()));
4942
+ const bySitemap = /* @__PURE__ */ new Map();
4943
+ for (const entry of site.discovery?.urls ?? []) {
4944
+ let pathname;
4945
+ try {
4946
+ pathname = new URL(entry.loc).pathname;
4947
+ } catch {
4948
+ continue;
3748
4949
  }
3749
- await res.body?.cancel().catch(() => void 0);
3750
- return result;
3751
- }
3752
- if (method === "HEAD") {
3753
- await res.body?.cancel().catch(() => void 0);
3754
- return result;
3755
- }
3756
- if (contentType && !TEXTY.test(contentType)) {
3757
- await res.body?.cancel().catch(() => void 0);
3758
- return result;
3759
- }
3760
- const { text, truncated } = await readCappedText(res, maxBytes);
3761
- result.body = text;
3762
- if (truncated) result.truncated = true;
3763
- return result;
3764
- } catch (err) {
3765
- return {
3766
- requestedUrl: url,
3767
- finalUrl: url,
3768
- status: 0,
3769
- redirected: false,
3770
- redirectChain: [],
3771
- reason: classifyError(err, options.signal),
3772
- durationMs: Date.now() - startedAt
3773
- };
3774
- } finally {
3775
- cleanup();
3776
- restoreTls();
3777
- }
3778
- }
3779
- function isRedirectStatus(status) {
3780
- return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
3781
- }
3782
- function normaliseContentType(raw) {
3783
- if (!raw) return void 0;
3784
- const semi = raw.indexOf(";");
3785
- return (semi === -1 ? raw : raw.slice(0, semi)).trim().toLowerCase() || void 0;
3786
- }
3787
- async function readCappedText(res, maxBytes) {
3788
- const reader = res.body?.getReader();
3789
- if (!reader) {
3790
- const text = await res.text();
3791
- if (text.length > maxBytes) {
3792
- return { text: text.slice(0, maxBytes), truncated: true };
3793
- }
3794
- return { text, truncated: false };
3795
- }
3796
- const chunks = [];
3797
- let received = 0;
3798
- let truncated = false;
3799
- for (; ; ) {
3800
- const { done, value } = await reader.read();
3801
- if (done) break;
3802
- if (!value) continue;
3803
- chunks.push(value);
3804
- received += value.byteLength;
3805
- if (received >= maxBytes) {
3806
- truncated = true;
3807
- await reader.cancel().catch(() => void 0);
3808
- break;
4950
+ const { locale } = splitRoute(pathname);
4951
+ if (locale === "x-default") continue;
4952
+ if (!axis.has(locale.toLowerCase())) continue;
4953
+ const route = site.clusterRouteOf?.(entry.loc) ?? splitRoute(pathname).route;
4954
+ const set = bySitemap.get(route) ?? /* @__PURE__ */ new Set();
4955
+ set.add(locale.toLowerCase());
4956
+ bySitemap.set(route, set);
3809
4957
  }
3810
- }
3811
- const buf = Buffer.concat(chunks.map((c) => Buffer.from(c)));
3812
- const sliced = buf.byteLength > maxBytes ? buf.subarray(0, maxBytes) : buf;
3813
- return { text: sliced.toString("utf8"), truncated };
3814
- }
3815
- function classifyError(err, callerSignal) {
3816
- if (isAbortError(err)) {
3817
- return callerSignal?.aborted ? "abort" : "timeout";
3818
- }
3819
- const code = errorCode(err);
3820
- if (code) {
3821
- if (code === "ENOTFOUND" || code === "EAI_AGAIN") return "dns";
3822
- if (code.startsWith("CERT_") || code.includes("SELF_SIGNED") || code.includes("CERT")) {
3823
- return "tls";
4958
+ const drafts = [];
4959
+ for (const page of site.pages) {
4960
+ const head = /* @__PURE__ */ new Set();
4961
+ for (const alt of page.links.alternates) {
4962
+ if (alt.isXDefault) continue;
4963
+ const tag = alt.hreflang?.trim().toLowerCase();
4964
+ if (tag) head.add(tag);
4965
+ }
4966
+ if (head.size === 0) continue;
4967
+ let pathname;
4968
+ try {
4969
+ pathname = new URL(page.fetch.finalUrl).pathname;
4970
+ } catch {
4971
+ continue;
4972
+ }
4973
+ const route = site.clusterRouteOf?.(page.fetch.finalUrl) ?? splitRoute(pathname).route;
4974
+ const inSitemap = bySitemap.get(route);
4975
+ if (!inSitemap || inSitemap.size === 0) continue;
4976
+ const unlisted = [...head].filter((l) => !inSitemap.has(l)).sort();
4977
+ if (unlisted.length === 0) continue;
4978
+ drafts.push({
4979
+ pageUrl: page.fetch.finalUrl,
4980
+ evidence: {
4981
+ route,
4982
+ // Both sides, verbatim and sorted, so an agent can see the shape of
4983
+ // the disagreement rather than take our word for its size.
4984
+ headAdvertises: [...head].sort(),
4985
+ sitemapLists: [...inSitemap].sort(),
4986
+ advertisedButUnlisted: unlisted
4987
+ }
4988
+ });
3824
4989
  }
4990
+ return drafts;
3825
4991
  }
3826
- const message = err instanceof Error ? err.message.toLowerCase() : String(err).toLowerCase();
3827
- if (message.includes("certificate") || message.includes("tls") || message.includes("ssl")) {
3828
- return "tls";
3829
- }
3830
- return "network";
3831
- }
3832
- function isAbortError(err) {
3833
- return err instanceof Error && err.name === "AbortError";
3834
- }
3835
- function errorCode(err) {
3836
- if (err && typeof err === "object") {
3837
- const direct = err.code;
3838
- if (typeof direct === "string") return direct;
3839
- const cause = err.cause;
3840
- if (cause && typeof cause === "object") {
3841
- const causeCode = cause.code;
3842
- if (typeof causeCode === "string") return causeCode;
4992
+ };
4993
+ var SITE_PROSE_RULES = [hreflangSitemapMismatch];
4994
+ function collectSiteAdvisories(site, rules = SITE_PROSE_RULES) {
4995
+ const out = [];
4996
+ for (const rule of rules) {
4997
+ if (rule.appliesTo && !rule.appliesTo(site)) continue;
4998
+ for (const draft of rule.ask(site)) {
4999
+ out.push({
5000
+ ruleId: rule.id,
5001
+ kind: "prose",
5002
+ prose: rule.prose,
5003
+ // `null` where nothing supports the question, which is exactly the
5004
+ // state this rule is in and the reason it stopped being a verdict.
5005
+ rigor: rule.rigor ?? null,
5006
+ sources: rule.sources ?? [],
5007
+ evidence: draft.evidence,
5008
+ verdict: "needs-judgment",
5009
+ pageUrl: draft.pageUrl
5010
+ });
3843
5011
  }
3844
5012
  }
3845
- return void 0;
3846
- }
3847
- function relaxTlsIfRequested3(enabled) {
3848
- if (!enabled) return () => void 0;
3849
- const previous = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
3850
- process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
3851
- return () => {
3852
- if (previous === void 0) delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
3853
- else process.env.NODE_TLS_REJECT_UNAUTHORIZED = previous;
3854
- };
5013
+ return out;
3855
5014
  }
3856
5015
 
3857
5016
  // src/lib/core/links/extract.ts
@@ -3937,188 +5096,6 @@ function resolveBase2($, pageUrl) {
3937
5096
  return pageUrl;
3938
5097
  }
3939
5098
 
3940
- // src/lib/core/links/classify.ts
3941
- function classifyLink(input) {
3942
- if (input.status === 0) return "broken";
3943
- if (input.loop) return "broken";
3944
- if (input.softNotFound) return "warning";
3945
- if (input.status === 403 || input.status === 429) return "blocked";
3946
- if (input.status >= 400) return "broken";
3947
- if (input.status >= 200 && input.status < 300) {
3948
- return input.redirected ? "redirect" : "ok";
3949
- }
3950
- return "broken";
3951
- }
3952
-
3953
- // src/lib/core/links/check.ts
3954
- var DEFAULTS2 = {
3955
- timeoutMs: 8e3,
3956
- maxRedirects: 10,
3957
- retries: 1,
3958
- backoffBaseMs: 300,
3959
- maxBackoffMs: 3e3,
3960
- softBodyBytes: 16 * 1024
3961
- };
3962
- var NOT_FOUND_RE = /\b(not\s+found|page\s+doesn'?t\s+exist|404|no\s+such\s+page)\b/i;
3963
- var SOFT_404_BODY_LIMIT = 2048;
3964
- async function checkLink(url, options = {}) {
3965
- const checkedAt = (/* @__PURE__ */ new Date()).toISOString();
3966
- const started = Date.now();
3967
- const scheme = schemeOf(url);
3968
- if (scheme !== "http" && scheme !== "https") {
3969
- return {
3970
- url,
3971
- finalUrl: url,
3972
- status: 0,
3973
- verdict: "skipped",
3974
- method: "GET",
3975
- redirectChain: [],
3976
- reason: scheme ? `unsupported scheme: ${scheme}` : "not a URL",
3977
- checkedAt,
3978
- durationMs: Date.now() - started
3979
- };
3980
- }
3981
- const outcome = await probeWithRetry(url, options);
3982
- const softNotFound = isSoftNotFound(outcome);
3983
- const verdict = classifyLink({
3984
- status: outcome.status,
3985
- redirected: outcome.redirectChain.length > 0,
3986
- reason: outcome.reason,
3987
- softNotFound,
3988
- loop: outcome.loop
3989
- });
3990
- return {
3991
- url,
3992
- finalUrl: outcome.finalUrl,
3993
- status: outcome.status,
3994
- verdict,
3995
- method: outcome.method,
3996
- redirectChain: outcome.redirectChain,
3997
- reason: describeReason(outcome, softNotFound),
3998
- checkedAt,
3999
- durationMs: Date.now() - started
4000
- };
4001
- }
4002
- async function probeWithRetry(url, options) {
4003
- const retries = options.retries ?? DEFAULTS2.retries;
4004
- const sleep = options.sleep ?? defaultSleep;
4005
- let outcome = await probeWithHeadFallback(url, options);
4006
- for (let attempt = 0; attempt < retries; attempt++) {
4007
- if (!isTransient(outcome)) break;
4008
- if (options.signal?.aborted) break;
4009
- const delay = backoffDelay(outcome, attempt, options);
4010
- if (delay > 0) await sleep(delay);
4011
- outcome = await probeWithHeadFallback(url, options);
4012
- }
4013
- return outcome;
4014
- }
4015
- async function probeWithHeadFallback(url, options) {
4016
- const head = await followRedirects(url, "HEAD", options);
4017
- if (headRejected(head) || head.status >= 200 && head.status < 300) {
4018
- const get = await followRedirects(url, "GET", options);
4019
- if (get.status === 0 && head.status !== 0 && !headRejected(head)) return head;
4020
- return get;
4021
- }
4022
- return head;
4023
- }
4024
- function headRejected(o) {
4025
- return o.status === 405 || o.status === 501 || o.status === 403 || o.status === 0;
4026
- }
4027
- async function followRedirects(url, method, options) {
4028
- const maxRedirects = options.maxRedirects ?? DEFAULTS2.maxRedirects;
4029
- const chain = [];
4030
- const visited = /* @__PURE__ */ new Set([url]);
4031
- let current = url;
4032
- for (let hop = 0; hop <= maxRedirects; hop++) {
4033
- const res = await fetchUrl(current, {
4034
- method,
4035
- redirect: "manual",
4036
- timeoutMs: options.timeoutMs ?? DEFAULTS2.timeoutMs,
4037
- allowInsecureTls: options.allowInsecureTls,
4038
- userAgent: options.userAgent,
4039
- maxBytes: method === "GET" ? options.softBodyBytes ?? DEFAULTS2.softBodyBytes : void 0
4040
- });
4041
- if (res.status === 0) {
4042
- return {
4043
- status: 0,
4044
- finalUrl: current,
4045
- redirectChain: chain,
4046
- reason: res.reason,
4047
- loop: false,
4048
- method
4049
- };
4050
- }
4051
- const next = res.redirectChain[0];
4052
- if (next) {
4053
- chain.push(next);
4054
- if (visited.has(next)) {
4055
- return { status: res.status, finalUrl: next, redirectChain: chain, loop: true, method };
4056
- }
4057
- visited.add(next);
4058
- current = next;
4059
- continue;
4060
- }
4061
- return {
4062
- status: res.status,
4063
- finalUrl: res.finalUrl,
4064
- redirectChain: chain,
4065
- loop: false,
4066
- body: res.body,
4067
- retryAfter: res.retryAfter,
4068
- method
4069
- };
4070
- }
4071
- return { status: 0, finalUrl: current, redirectChain: chain, loop: true, method };
4072
- }
4073
- function isSoftNotFound(o) {
4074
- if (o.status < 200 || o.status >= 300) return false;
4075
- if (o.body === void 0) return false;
4076
- if (o.body.length > SOFT_404_BODY_LIMIT) return false;
4077
- return NOT_FOUND_RE.test(o.body);
4078
- }
4079
- function isTransient(o) {
4080
- if (o.status === 0) return true;
4081
- if (o.status === 429) return true;
4082
- if (o.status >= 500) return true;
4083
- return false;
4084
- }
4085
- function backoffDelay(o, attempt, options) {
4086
- const max = options.maxBackoffMs ?? DEFAULTS2.maxBackoffMs;
4087
- const retryAfterMs = parseRetryAfter(o.retryAfter);
4088
- if (retryAfterMs !== void 0) return Math.min(retryAfterMs, max);
4089
- const base = options.backoffBaseMs ?? DEFAULTS2.backoffBaseMs;
4090
- const exp = base * 2 ** attempt;
4091
- const jitter = Math.floor(Math.random() * base);
4092
- return Math.min(exp + jitter, max);
4093
- }
4094
- function parseRetryAfter(value) {
4095
- if (!value) return void 0;
4096
- const seconds = Number(value);
4097
- if (Number.isFinite(seconds)) return Math.max(0, seconds * 1e3);
4098
- const date = Date.parse(value);
4099
- if (!Number.isNaN(date)) return Math.max(0, date - Date.now());
4100
- return void 0;
4101
- }
4102
- function describeReason(o, softNotFound) {
4103
- if (o.reason) return o.reason;
4104
- if (o.loop) return "redirect loop";
4105
- if (softNotFound) return "soft-404";
4106
- if (o.status === 429) return "429 rate-limited";
4107
- if (o.status === 403) return "403 forbidden";
4108
- if (o.status >= 400) return `${o.status}`;
4109
- return void 0;
4110
- }
4111
- function schemeOf(url) {
4112
- try {
4113
- return new URL(url).protocol.replace(/:$/, "").toLowerCase();
4114
- } catch {
4115
- return null;
4116
- }
4117
- }
4118
- function defaultSleep(ms) {
4119
- return new Promise((resolve) => setTimeout(resolve, ms));
4120
- }
4121
-
4122
5099
  // src/lib/core/links/types.ts
4123
5100
  function emptyVerdictSummary() {
4124
5101
  return { ok: 0, redirect: 0, broken: 0, blocked: 0, warning: 0, skipped: 0 };
@@ -4450,14 +5427,14 @@ async function attachManifests(pages, options) {
4450
5427
  }
4451
5428
  }
4452
5429
  async function probeAssets(pages, options) {
4453
- const sameOrigin = (url) => {
5430
+ const sameOrigin2 = (url) => {
4454
5431
  try {
4455
5432
  return new URL(url).origin === new URL(options.origin).origin;
4456
5433
  } catch {
4457
5434
  return false;
4458
5435
  }
4459
5436
  };
4460
- const wantedUrl = (url) => /^https?:\/\//i.test(url) && (options.checkExternal || sameOrigin(url));
5437
+ const wantedUrl = (url) => /^https?:\/\//i.test(url) && (options.checkExternal || sameOrigin2(url));
4461
5438
  const wanted = /* @__PURE__ */ new Set();
4462
5439
  for (const page of pages) {
4463
5440
  for (const image of page.openGraph.images) {
@@ -4630,7 +5607,14 @@ async function runAudit(inputUrl, options = {}) {
4630
5607
  severity: issue.severity,
4631
5608
  message: issue.message,
4632
5609
  why: rule?.title,
4633
- fix: issue.fix?.snippet
5610
+ fix: issue.fix?.snippet,
5611
+ // The fingerprint above is unchanged on purpose: it keys on the rule,
5612
+ // the route and the occurrence, so carrying more of the finding cannot
5613
+ // move a baseline entry (`./diff.ts` matches on `id` alone).
5614
+ rigor: issue.rigor,
5615
+ sources: issue.sources,
5616
+ observed: issue.observed,
5617
+ expected: issue.expected
4634
5618
  });
4635
5619
  }
4636
5620
  }
@@ -4677,6 +5661,33 @@ async function runAudit(inputUrl, options = {}) {
4677
5661
  ),
4678
5662
  ...issue
4679
5663
  }));
5664
+ const linkDiscovery = syntheticDiscovery(origin, entry, okPages, crawlResult.truncated);
5665
+ const linkReport = await runLinkAudit(linkDiscovery, {
5666
+ allowInsecureTls: options.allowInsecureTls,
5667
+ checkExternal,
5668
+ timeoutMs: options.timeoutMs,
5669
+ maxPages: effectiveMaxPages,
5670
+ signal: options.signal,
5671
+ onProgress: (p) => options.onProgress?.({
5672
+ phase: p.phase === "scan" ? "scan" : "links",
5673
+ done: p.done,
5674
+ total: p.total
5675
+ })
5676
+ });
5677
+ warnings.push(...linkReport.diagnostics.warnings);
5678
+ const sitemapEntries = discovery ? await probeSitemapEntries(
5679
+ discovery.urls.map((entry2) => entry2.loc),
5680
+ {
5681
+ crawled: new Map(
5682
+ crawlResult.pages.map((page) => [page.fetch.finalUrl, page.fetch.status])
5683
+ ),
5684
+ checked: new Map(Object.values(linkReport.checks).map((check) => [check.url, check])),
5685
+ maxProbes: effectiveMaxPages,
5686
+ signal: options.signal,
5687
+ timeoutMs: options.timeoutMs,
5688
+ allowInsecureTls: options.allowInsecureTls
5689
+ }
5690
+ ) : void 0;
4680
5691
  const siteContext = {
4681
5692
  origin,
4682
5693
  pages: htmlPages,
@@ -4685,6 +5696,7 @@ async function runAudit(inputUrl, options = {}) {
4685
5696
  discovery,
4686
5697
  robots,
4687
5698
  favicon,
5699
+ sitemapEntries,
4688
5700
  // Same index the matrix was built from, so a rule that groups by route
4689
5701
  // and the grid that reports holes cannot disagree about which URLs are
4690
5702
  // one page.
@@ -4703,23 +5715,19 @@ async function runAudit(inputUrl, options = {}) {
4703
5715
  severity: draft.severity,
4704
5716
  message: draft.message,
4705
5717
  why: rule?.summary,
4706
- fix: draft.fix?.snippet
5718
+ fix: draft.fix?.snippet,
5719
+ rigor: draft.rigor,
5720
+ sources: draft.sources
4707
5721
  };
4708
5722
  });
4709
- const linkDiscovery = syntheticDiscovery(origin, entry, okPages, crawlResult.truncated);
4710
- const linkReport = await runLinkAudit(linkDiscovery, {
4711
- allowInsecureTls: options.allowInsecureTls,
4712
- checkExternal,
4713
- timeoutMs: options.timeoutMs,
4714
- maxPages: effectiveMaxPages,
4715
- signal: options.signal,
4716
- onProgress: (p) => options.onProgress?.({
4717
- phase: p.phase === "scan" ? "scan" : "links",
4718
- done: p.done,
4719
- total: p.total
4720
- })
4721
- });
4722
- warnings.push(...linkReport.diagnostics.warnings);
5723
+ if (options.advisories) {
5724
+ for (const advisory of collectSiteAdvisories(siteContext)) {
5725
+ advisories.push({
5726
+ ...advisory,
5727
+ id: fingerprint("advisory", advisory.ruleId, routeKey(advisory.pageUrl))
5728
+ });
5729
+ }
5730
+ }
4723
5731
  const brokenLinks = [];
4724
5732
  for (const { pageUrl, broken } of linkReport.brokenByPage) {
4725
5733
  for (const check of broken) {
@@ -4856,6 +5864,10 @@ function syntheticDiscovery(origin, baseUrl, pages, truncated) {
4856
5864
  baseUrl,
4857
5865
  source: "crawl",
4858
5866
  urls: pages.map((p) => ({ loc: p.fetch.finalUrl })),
5867
+ // Crawl-derived: no sitemap document was read, so the tree is empty rather
5868
+ // than absent. The document rules ask "what did each document declare?" and
5869
+ // must see that nothing declared anything.
5870
+ documents: [],
4859
5871
  diagnostics: {
4860
5872
  found: false,
4861
5873
  status: 0,
@@ -5209,7 +6221,8 @@ function rollupRows(issues, c) {
5209
6221
  const lines = [];
5210
6222
  for (const issue of issues) {
5211
6223
  const sev = issue.severity === "error" ? c(ANSI2.red, "error") : issue.severity === "warning" ? c(ANSI2.yellow, "warn ") : c(ANSI2.dim, "info ");
5212
- lines.push(` ${sev} ${c(ANSI2.cyan, issue.ruleId)} ${c(ANSI2.dim, `\xD7${issue.count}`)}`);
6224
+ const rigor = issue.rigor ? c(ANSI2.dim, ` [${issue.rigor}]`) : "";
6225
+ lines.push(` ${sev} ${c(ANSI2.cyan, issue.ruleId)}${rigor} ${c(ANSI2.dim, `\xD7${issue.count}`)}`);
5213
6226
  if (issue.why) lines.push(` ${c(ANSI2.dim, stripBackticks2(issue.why))}`);
5214
6227
  if (issue.fix) {
5215
6228
  const [head, ...rest] = issue.fix.split("\n");
@@ -5288,6 +6301,7 @@ function rollupByRule(issues) {
5288
6301
  groups.set(issue.ruleId, {
5289
6302
  ruleId: issue.ruleId,
5290
6303
  severity: issue.severity,
6304
+ rigor: issue.rigor,
5291
6305
  why: issue.why,
5292
6306
  fix: issue.fix,
5293
6307
  sample: issue.message,
@@ -5494,6 +6508,7 @@ export {
5494
6508
  PROFILES,
5495
6509
  PROFILE_NAMES,
5496
6510
  PROSE_RULES,
6511
+ SITE_PROSE_RULES,
5497
6512
  routeKey,
5498
6513
  targetKey,
5499
6514
  fingerprint,
@@ -5506,4 +6521,4 @@ export {
5506
6521
  summarize,
5507
6522
  Logger
5508
6523
  };
5509
- //# sourceMappingURL=chunk-TAZEAIFI.js.map
6524
+ //# sourceMappingURL=chunk-CWEJ6EXT.js.map