nfunc-mcp 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/README.md +85 -335
  2. package/dist/index.js +27 -4
  3. package/dist/index.js.map +1 -1
  4. package/dist/mappers/a11yDedupe.d.ts +25 -0
  5. package/dist/mappers/a11yDedupe.js +95 -0
  6. package/dist/mappers/a11yDedupe.js.map +1 -0
  7. package/dist/mappers/compositeScore.d.ts +23 -0
  8. package/dist/mappers/compositeScore.js +107 -0
  9. package/dist/mappers/compositeScore.js.map +1 -0
  10. package/dist/mappers/correlator.js +94 -33
  11. package/dist/mappers/correlator.js.map +1 -1
  12. package/dist/mappers/defectFormatter.js +129 -15
  13. package/dist/mappers/defectFormatter.js.map +1 -1
  14. package/dist/mappers/labFieldComparator.d.ts +62 -0
  15. package/dist/mappers/labFieldComparator.js +134 -0
  16. package/dist/mappers/labFieldComparator.js.map +1 -0
  17. package/dist/mappers/priorityMapper.d.ts +38 -0
  18. package/dist/mappers/priorityMapper.js +126 -0
  19. package/dist/mappers/priorityMapper.js.map +1 -1
  20. package/dist/mappers/psiAggregator.d.ts +130 -0
  21. package/dist/mappers/psiAggregator.js +293 -0
  22. package/dist/mappers/psiAggregator.js.map +1 -0
  23. package/dist/mappers/webVitalsMapper.d.ts +52 -0
  24. package/dist/mappers/webVitalsMapper.js +131 -0
  25. package/dist/mappers/webVitalsMapper.js.map +1 -0
  26. package/dist/tools/accessibility.js +72 -31
  27. package/dist/tools/accessibility.js.map +1 -1
  28. package/dist/tools/lighthouse.d.ts +14 -0
  29. package/dist/tools/lighthouse.js +134 -34
  30. package/dist/tools/lighthouse.js.map +1 -1
  31. package/dist/tools/performanceAudit.d.ts +2 -0
  32. package/dist/tools/performanceAudit.js +446 -0
  33. package/dist/tools/performanceAudit.js.map +1 -0
  34. package/dist/tools/performanceAuditPlan.d.ts +2 -0
  35. package/dist/tools/performanceAuditPlan.js +438 -0
  36. package/dist/tools/performanceAuditPlan.js.map +1 -0
  37. package/dist/tools/qaGate.js +109 -34
  38. package/dist/tools/qaGate.js.map +1 -1
  39. package/dist/utils/csvReader.d.ts +20 -0
  40. package/dist/utils/csvReader.js +172 -0
  41. package/dist/utils/csvReader.js.map +1 -0
  42. package/dist/utils/httpClient.d.ts +84 -0
  43. package/dist/utils/httpClient.js +171 -0
  44. package/dist/utils/httpClient.js.map +1 -0
  45. package/dist/utils/outputParsers.d.ts +25 -0
  46. package/dist/utils/outputParsers.js +42 -0
  47. package/dist/utils/outputParsers.js.map +1 -1
  48. package/dist/utils/psiAuth.d.ts +26 -0
  49. package/dist/utils/psiAuth.js +36 -0
  50. package/dist/utils/psiAuth.js.map +1 -0
  51. package/dist/utils/psiParser.d.ts +124 -0
  52. package/dist/utils/psiParser.js +200 -0
  53. package/dist/utils/psiParser.js.map +1 -0
  54. package/dist/utils/publicUrl.d.ts +17 -0
  55. package/dist/utils/publicUrl.js +115 -0
  56. package/dist/utils/publicUrl.js.map +1 -0
  57. package/dist/utils/sitemapReader.d.ts +27 -0
  58. package/dist/utils/sitemapReader.js +272 -0
  59. package/dist/utils/sitemapReader.js.map +1 -0
  60. package/dist/utils/urlClassifier.d.ts +45 -0
  61. package/dist/utils/urlClassifier.js +267 -0
  62. package/dist/utils/urlClassifier.js.map +1 -0
  63. package/docs/manual.md +558 -0
  64. package/docs/psi-report-spec.md +174 -0
  65. package/package.json +14 -4
@@ -0,0 +1,200 @@
1
+ /**
2
+ * PageSpeed Insights response parsing.
3
+ *
4
+ * One PSI call returns two unrelated datasets: `lighthouseResult` (a lab run
5
+ * on Google's hardware) and `loadingExperience` (CrUX field data from real
6
+ * Chrome users). They are measured differently, they disagree routinely, and
7
+ * conflating them produces confidently wrong reports — so they stay separate
8
+ * all the way through this module and only meet in the comparator.
9
+ *
10
+ * The lab half is the same LHR schema `parseLighthouseJSON` already handles
11
+ * and is delegated to it unchanged. Everything here is about the field half
12
+ * and about pulling numeric lab metrics that survive aggregation.
13
+ */
14
+ import { parseLighthouseJSON } from "./outputParsers.js";
15
+ /** PSI returned a 200 whose Lighthouse run did not actually complete. */
16
+ export class PsiRuntimeError extends Error {
17
+ code;
18
+ constructor(code, message) {
19
+ super(message);
20
+ this.code = code;
21
+ this.name = "PsiRuntimeError";
22
+ }
23
+ }
24
+ /**
25
+ * CrUX metric key → our vital name.
26
+ *
27
+ * FIRST_INPUT_DELAY_MS is intentionally not mapped. FID was retired in March
28
+ * 2024 and replaced by INP; it still appears in some responses, and treating
29
+ * it as a current metric would report a defect against a standard Google no
30
+ * longer measures. Read it only if a caller ever needs historical data.
31
+ */
32
+ const CRUX_KEYS = {
33
+ LARGEST_CONTENTFUL_PAINT_MS: "lcp",
34
+ INTERACTION_TO_NEXT_PAINT: "inp",
35
+ CUMULATIVE_LAYOUT_SHIFT_SCORE: "cls",
36
+ FIRST_CONTENTFUL_PAINT_MS: "fcp",
37
+ EXPERIMENTAL_TIME_TO_FIRST_BYTE: "ttfb",
38
+ };
39
+ /**
40
+ * CLS is the one field metric PSI returns scaled by 100: a `percentile` of 55
41
+ * means a CLS of 0.55, and the distribution bucket boundaries (0-10, 10-25)
42
+ * are scaled the same way. Comparing the raw integer against the 0.25 "poor"
43
+ * threshold marks every site on earth as catastrophic; forgetting the scaling
44
+ * entirely turns a 0.55 into a 55. Verified against real responses.
45
+ */
46
+ function scaleCruxValue(vital, percentile) {
47
+ return vital === "cls" ? percentile / 100 : percentile;
48
+ }
49
+ function toCategory(raw) {
50
+ return raw === "FAST" || raw === "AVERAGE" || raw === "SLOW" ? raw : "NONE";
51
+ }
52
+ function toDistribution(raw) {
53
+ const share = (i) => {
54
+ const p = raw?.[i]?.proportion;
55
+ return typeof p === "number" ? Number(p.toFixed(4)) : 0;
56
+ };
57
+ // PSI always emits exactly three buckets, ordered good → poor.
58
+ return { good: share(0), needsImprovement: share(1), poor: share(2) };
59
+ }
60
+ function readMetric(raw, vital, source) {
61
+ if (!raw || typeof raw.percentile !== "number")
62
+ return null;
63
+ return {
64
+ p75: scaleCruxValue(vital, raw.percentile),
65
+ category: toCategory(raw.category),
66
+ distribution: toDistribution(raw.distributions),
67
+ source,
68
+ };
69
+ }
70
+ /**
71
+ * Merge URL-level and origin-level CrUX into one metric set.
72
+ *
73
+ * Field availability is **per metric, not per URL** — a page can have enough
74
+ * traffic for CrUX to report CLS but not LCP. A real response in the Five
75
+ * Below batch carried two of the five metrics at URL level while the origin
76
+ * carried all five, which is why the manual audit shows "No field data" for
77
+ * one metric and a category for another on the same row.
78
+ *
79
+ * So the fallback runs per metric, and each metric records where it came from.
80
+ * A single response-level `field_source` flag would have to either discard the
81
+ * URL-level metrics that do exist or silently label origin data as page data,
82
+ * and origin CrUX for a homepage says nothing about a checkout page.
83
+ */
84
+ export function parseCrux(urlLevel, originLevel, originFallback) {
85
+ const metrics = {};
86
+ let fromUrl = 0;
87
+ let fromOrigin = 0;
88
+ /**
89
+ * PSI performs its own origin fallback, and does it silently.
90
+ *
91
+ * When a URL has too little traffic, `loadingExperience` comes back populated
92
+ * with **origin-level** numbers and the only signal is its `id`, which holds
93
+ * the origin instead of the URL. Trusting the block's position in the
94
+ * response therefore labels origin data as page data: two different paginated
95
+ * URLs came back with byte-identical p75 values, both marked `source: "url"`.
96
+ *
97
+ * Comparing the two blocks' ids catches it without needing the requested URL.
98
+ */
99
+ const psiSubstitutedOrigin = Boolean(urlLevel?.id) && Boolean(originLevel?.id) && urlLevel?.id === originLevel?.id;
100
+ const urlLevelSource = psiSubstitutedOrigin ? "origin" : "url";
101
+ for (const [cruxKey, vital] of Object.entries(CRUX_KEYS)) {
102
+ const atUrl = readMetric(urlLevel?.metrics?.[cruxKey], vital, urlLevelSource);
103
+ if (atUrl) {
104
+ metrics[vital] = atUrl;
105
+ if (urlLevelSource === "url")
106
+ fromUrl++;
107
+ else
108
+ fromOrigin++;
109
+ continue;
110
+ }
111
+ if (!originFallback)
112
+ continue;
113
+ const atOrigin = readMetric(originLevel?.metrics?.[cruxKey], vital, "origin");
114
+ if (atOrigin) {
115
+ metrics[vital] = atOrigin;
116
+ fromOrigin++;
117
+ }
118
+ }
119
+ if (fromUrl === 0 && fromOrigin === 0)
120
+ return null;
121
+ // Only trust the URL-level verdict when the URL actually supplied data;
122
+ // otherwise the origin's verdict is the one describing these numbers.
123
+ const overallRaw = fromUrl > 0 ? urlLevel?.overall_category : originLevel?.overall_category;
124
+ return {
125
+ collectionPeriodDays: 28,
126
+ overallCategory: overallRaw ? toCategory(overallRaw) : null,
127
+ primarySource: fromUrl >= fromOrigin ? "url" : "origin",
128
+ metrics,
129
+ };
130
+ }
131
+ /**
132
+ * Lab metrics as numbers.
133
+ *
134
+ * `numericValue`, never `displayValue`. The Five Below CSV recorded TTFB as
135
+ * "Root document took 930 ms" — a localised string containing a non-breaking
136
+ * space — because it read displayValue, which made the whole column
137
+ * unaggregatable while `numericValue: 929` sat in the same audit object.
138
+ * Formatting is a report-time concern; nothing upstream of the report should
139
+ * hold a number as text.
140
+ */
141
+ function parseLabMetrics(auditsJson) {
142
+ const audits = JSON.parse(auditsJson);
143
+ const num = (id) => {
144
+ const v = audits.audits?.[id]?.numericValue;
145
+ return typeof v === "number" ? v : null;
146
+ };
147
+ const ms = (id) => {
148
+ const v = num(id);
149
+ return v === null ? null : Math.round(v);
150
+ };
151
+ return {
152
+ lcpMs: ms("largest-contentful-paint"),
153
+ fcpMs: ms("first-contentful-paint"),
154
+ // CLS is unitless and small; rounding it to an integer would destroy it.
155
+ cls: num("cumulative-layout-shift"),
156
+ tbtMs: ms("total-blocking-time"),
157
+ speedIndexMs: ms("speed-index"),
158
+ ttiMs: ms("interactive"),
159
+ ttfbMs: ms("server-response-time"),
160
+ };
161
+ }
162
+ /**
163
+ * Parse a full PSI v5 response.
164
+ *
165
+ * Throws `PsiRuntimeError` when Lighthouse itself failed inside a 200
166
+ * response. PSI does this routinely — a page that times out or refuses the
167
+ * fetch comes back as HTTP 200 with `lighthouseResult.runtimeError` set and
168
+ * every score null. Parsing it anyway records a real page as scoring zero
169
+ * across the board, which is worse than a failed run, because a zero looks
170
+ * like data. Callers should treat this as retryable.
171
+ */
172
+ export function parsePsiResponse(rawJson, options) {
173
+ const { strategy, originFallback = true } = options;
174
+ const response = JSON.parse(rawJson);
175
+ const lhr = response.lighthouseResult;
176
+ if (!lhr) {
177
+ throw new PsiRuntimeError("NO_LIGHTHOUSE_RESULT", "PSI response contained no lighthouseResult.");
178
+ }
179
+ if (lhr.runtimeError?.code) {
180
+ throw new PsiRuntimeError(lhr.runtimeError.code, lhr.runtimeError.message ?? "Lighthouse reported a runtime error.");
181
+ }
182
+ // parseLighthouseJSON takes the serialised LHR. Re-serialising the object we
183
+ // already hold is a little wasteful, but it keeps the Phase 2-13 parser and
184
+ // its priority mapping untouched, which is worth more than the milliseconds.
185
+ const lhrJson = JSON.stringify(lhr);
186
+ const lighthouse = parseLighthouseJSON(lhrJson);
187
+ return {
188
+ requestedUrl: lhr.requestedUrl ?? "",
189
+ finalUrl: lhr.finalDisplayedUrl ?? lhr.finalUrl ?? lhr.requestedUrl ?? "",
190
+ strategy,
191
+ fetchTime: lhr.fetchTime ?? null,
192
+ lighthouseVersion: lhr.lighthouseVersion ?? null,
193
+ scores: lighthouse.categoryScores,
194
+ lab: parseLabMetrics(lhrJson),
195
+ field: parseCrux(response.loadingExperience, response.originLoadingExperience, originFallback),
196
+ lighthouse,
197
+ runWarnings: lhr.runWarnings ?? [],
198
+ };
199
+ }
200
+ //# sourceMappingURL=psiParser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"psiParser.js","sourceRoot":"","sources":["../../src/utils/psiParser.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,mBAAmB,EAAyB,MAAM,oBAAoB,CAAC;AAgEhF,yEAAyE;AACzE,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAE7B;IADX,YACW,IAAY,EACrB,OAAe;QAEf,KAAK,CAAC,OAAO,CAAC,CAAC;QAHN,SAAI,GAAJ,IAAI,CAAQ;QAIrB,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AA+BD;;;;;;;GAOG;AACH,MAAM,SAAS,GAA6B;IAC1C,2BAA2B,EAAE,KAAK;IAClC,yBAAyB,EAAE,KAAK;IAChC,6BAA6B,EAAE,KAAK;IACpC,yBAAyB,EAAE,KAAK;IAChC,+BAA+B,EAAE,MAAM;CACxC,CAAC;AAEF;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,KAAe,EAAE,UAAkB;IACzD,OAAO,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;AACzD,CAAC;AAED,SAAS,UAAU,CAAC,GAAuB;IACzC,OAAO,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;AAC9E,CAAC;AAED,SAAS,cAAc,CACrB,GAAmC;IAEnC,MAAM,KAAK,GAAG,CAAC,CAAS,EAAU,EAAE;QAClC,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC;QAC/B,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC,CAAC;IACF,+DAA+D;IAC/D,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,UAAU,CACjB,GAA8B,EAC9B,KAAe,EACf,MAAmB;IAEnB,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC5D,OAAO;QACL,GAAG,EAAE,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,UAAU,CAAC;QAC1C,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClC,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC;QAC/C,MAAM;KACP,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,SAAS,CACvB,QAA0C,EAC1C,WAA6C,EAC7C,cAAuB;IAEvB,MAAM,OAAO,GAA0C,EAAE,CAAC;IAC1D,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB;;;;;;;;;;OAUG;IACH,MAAM,oBAAoB,GACxB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,IAAI,QAAQ,EAAE,EAAE,KAAK,WAAW,EAAE,EAAE,CAAC;IACxF,MAAM,cAAc,GAAgB,oBAAoB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAE5E,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC;QAC9E,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;YACvB,IAAI,cAAc,KAAK,KAAK;gBAAE,OAAO,EAAE,CAAC;;gBACnC,UAAU,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,cAAc;YAAE,SAAS;QAC9B,MAAM,QAAQ,GAAG,UAAU,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC9E,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC;YAC1B,UAAU,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnD,wEAAwE;IACxE,sEAAsE;IACtE,MAAM,UAAU,GACd,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC,CAAC,WAAW,EAAE,gBAAgB,CAAC;IAE3E,OAAO;QACL,oBAAoB,EAAE,EAAE;QACxB,eAAe,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI;QAC3D,aAAa,EAAE,OAAO,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ;QACvD,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,eAAe,CAAC,UAAkB;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAEnC,CAAC;IACF,MAAM,GAAG,GAAG,CAAC,EAAU,EAAiB,EAAE;QACxC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,YAAY,CAAC;QAC5C,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1C,CAAC,CAAC;IACF,MAAM,EAAE,GAAG,CAAC,EAAU,EAAiB,EAAE;QACvC,MAAM,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC;IAEF,OAAO;QACL,KAAK,EAAE,EAAE,CAAC,0BAA0B,CAAC;QACrC,KAAK,EAAE,EAAE,CAAC,wBAAwB,CAAC;QACnC,yEAAyE;QACzE,GAAG,EAAE,GAAG,CAAC,yBAAyB,CAAC;QACnC,KAAK,EAAE,EAAE,CAAC,qBAAqB,CAAC;QAChC,YAAY,EAAE,EAAE,CAAC,aAAa,CAAC;QAC/B,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC;QACxB,MAAM,EAAE,EAAE,CAAC,sBAAsB,CAAC;KACnC,CAAC;AACJ,CAAC;AAQD;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAe,EACf,OAAwB;IAExB,MAAM,EAAE,QAAQ,EAAE,cAAc,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;IACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,gBAAgB,CAAC;IAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,eAAe,CACvB,sBAAsB,EACtB,6CAA6C,CAC9C,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC;QAC3B,MAAM,IAAI,eAAe,CACvB,GAAG,CAAC,YAAY,CAAC,IAAI,EACrB,GAAG,CAAC,YAAY,CAAC,OAAO,IAAI,sCAAsC,CACnE,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,4EAA4E;IAC5E,6EAA6E;IAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,UAAU,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEhD,OAAO;QACL,YAAY,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,iBAAiB,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,YAAY,IAAI,EAAE;QACzE,QAAQ;QACR,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,IAAI;QAChC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,IAAI,IAAI;QAChD,MAAM,EAAE,UAAU,CAAC,cAAc;QACjC,GAAG,EAAE,eAAe,CAAC,OAAO,CAAC;QAC7B,KAAK,EAAE,SAAS,CACd,QAAQ,CAAC,iBAAiB,EAC1B,QAAQ,CAAC,uBAAuB,EAChC,cAAc,CACf;QACD,UAAU;QACV,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;KACnC,CAAC;AACJ,CAAC"}
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Whether PSI can reach a URL at all.
3
+ *
4
+ * PSI fetches the target from Google's own infrastructure, so "can I open this
5
+ * in my browser" is not the test — the page has to be reachable from the
6
+ * public internet. Localhost and RFC1918 addresses are not degraded cases that
7
+ * return partial data; they cannot be audited, full stop, and the only useful
8
+ * response is to say so before spending a request finding out.
9
+ */
10
+ export type ReachVerdict = "public" | "loopback" | "private" | "non_http" | "malformed";
11
+ export interface ReachResult {
12
+ verdict: ReachVerdict;
13
+ auditable: boolean;
14
+ reason?: string;
15
+ }
16
+ export declare function checkPublicReachability(rawUrl: string): ReachResult;
17
+ export declare function isSessionGated(rawUrl: string): boolean;
@@ -0,0 +1,115 @@
1
+ /**
2
+ * Whether PSI can reach a URL at all.
3
+ *
4
+ * PSI fetches the target from Google's own infrastructure, so "can I open this
5
+ * in my browser" is not the test — the page has to be reachable from the
6
+ * public internet. Localhost and RFC1918 addresses are not degraded cases that
7
+ * return partial data; they cannot be audited, full stop, and the only useful
8
+ * response is to say so before spending a request finding out.
9
+ */
10
+ /** IPv4 in a private or link-local range, per RFC1918 / RFC3927. */
11
+ function isPrivateIPv4(host) {
12
+ const parts = host.split(".");
13
+ if (parts.length !== 4)
14
+ return false;
15
+ const [a, b] = parts.map(Number);
16
+ if (parts.some((p) => !/^\d+$/.test(p)) || [a, b].some(Number.isNaN))
17
+ return false;
18
+ if (a === 10)
19
+ return true;
20
+ if (a === 172 && b >= 16 && b <= 31)
21
+ return true;
22
+ if (a === 192 && b === 168)
23
+ return true;
24
+ if (a === 169 && b === 254)
25
+ return true; // link-local
26
+ if (a === 100 && b >= 64 && b <= 127)
27
+ return true; // CGNAT
28
+ return false;
29
+ }
30
+ function isLoopback(host) {
31
+ if (host === "localhost" || host.endsWith(".localhost"))
32
+ return true;
33
+ if (host === "::1" || host === "[::1]")
34
+ return true;
35
+ return /^127\./.test(host);
36
+ }
37
+ /**
38
+ * Hostnames that never resolve on the public internet. `.local` is mDNS,
39
+ * `.internal` and `.test`/`.invalid` are reserved, and a bare hostname with no
40
+ * dot is a LAN name.
41
+ */
42
+ function isNonPublicName(host) {
43
+ if (/\.(local|internal|localdomain|test|invalid|example)$/.test(host))
44
+ return true;
45
+ return !host.includes(".") && !/^\d+$/.test(host);
46
+ }
47
+ export function checkPublicReachability(rawUrl) {
48
+ let url;
49
+ try {
50
+ url = new URL(rawUrl);
51
+ }
52
+ catch {
53
+ return { verdict: "malformed", auditable: false, reason: `"${rawUrl}" is not a valid URL.` };
54
+ }
55
+ if (url.protocol !== "http:" && url.protocol !== "https:") {
56
+ return {
57
+ verdict: "non_http",
58
+ auditable: false,
59
+ reason: `PSI only audits http(s) URLs; got "${url.protocol}".`,
60
+ };
61
+ }
62
+ const host = url.hostname.toLowerCase();
63
+ if (isLoopback(host)) {
64
+ return {
65
+ verdict: "loopback",
66
+ auditable: false,
67
+ reason: "PSI fetches pages from Google's infrastructure, so it cannot reach " +
68
+ "localhost. Use run_lighthouse, which runs Chrome on this machine.",
69
+ };
70
+ }
71
+ if (isPrivateIPv4(host)) {
72
+ return {
73
+ verdict: "private",
74
+ auditable: false,
75
+ reason: `${host} is a private-network address and is not reachable from ` +
76
+ "Google's infrastructure. Use run_lighthouse for internal hosts.",
77
+ };
78
+ }
79
+ if (isNonPublicName(host)) {
80
+ return {
81
+ verdict: "private",
82
+ auditable: false,
83
+ reason: `"${host}" is not a publicly resolvable hostname. Use run_lighthouse ` +
84
+ "for hosts that only resolve on your network.",
85
+ };
86
+ }
87
+ return { verdict: "public", auditable: true };
88
+ }
89
+ /**
90
+ * Paths that PSI can fetch but cannot meaningfully audit, because an anonymous
91
+ * request does not see the real page.
92
+ *
93
+ * PSI has no session: it audits an empty cart, a redirect to a login form, or
94
+ * a search page with no query, and reports the result as if it were the page
95
+ * someone asked about. A confidently wrong number is worse than a gap, so
96
+ * these are flagged rather than run. `run_lighthouse` can carry cookies and is
97
+ * the right tool for them.
98
+ */
99
+ const SESSION_GATED = [
100
+ /(^|\/)(cart|basket|bag)(\/|$)/,
101
+ /(^|\/)(checkout|payment|order-confirmation)(\/|$)/,
102
+ /(^|\/)(account|my-account|profile|dashboard|orders)(\/|$)/,
103
+ /(^|\/)(login|signin|sign-in|register|signup|sign-up|logout)(\/|$)/,
104
+ /(^|\/)(wishlist|favorites|favourites|saved)(\/|$)/,
105
+ ];
106
+ export function isSessionGated(rawUrl) {
107
+ try {
108
+ const path = new URL(rawUrl).pathname.toLowerCase();
109
+ return SESSION_GATED.some((re) => re.test(path));
110
+ }
111
+ catch {
112
+ return false;
113
+ }
114
+ }
115
+ //# sourceMappingURL=publicUrl.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"publicUrl.js","sourceRoot":"","sources":["../../src/utils/publicUrl.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAUH,oEAAoE;AACpE,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACnF,IAAI,CAAC,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC;IACjD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IACxC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,aAAa;IACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,QAAQ;IAC3D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC;IACrE,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IACpD,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,sDAAsD,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACnF,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAAc;IACpD,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,MAAM,uBAAuB,EAAE,CAAC;IAC/F,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,OAAO;YACL,OAAO,EAAE,UAAU;YACnB,SAAS,EAAE,KAAK;YAChB,MAAM,EAAE,sCAAsC,GAAG,CAAC,QAAQ,IAAI;SAC/D,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAExC,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO;YACL,OAAO,EAAE,UAAU;YACnB,SAAS,EAAE,KAAK;YAChB,MAAM,EACJ,qEAAqE;gBACrE,mEAAmE;SACtE,CAAC;IACJ,CAAC;IACD,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,KAAK;YAChB,MAAM,EACJ,GAAG,IAAI,0DAA0D;gBACjE,iEAAiE;SACpE,CAAC;IACJ,CAAC;IACD,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,KAAK;YAChB,MAAM,EACJ,IAAI,IAAI,8DAA8D;gBACtE,8CAA8C;SACjD,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAChD,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,aAAa,GAAG;IACpB,+BAA+B;IAC/B,mDAAmD;IACnD,2DAA2D;IAC3D,mEAAmE;IACnE,mDAAmD;CACpD,CAAC;AAEF,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACpD,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Sitemap discovery.
3
+ *
4
+ * The cheapest honest way to learn which pages a site has. A sitemap is
5
+ * authoritative (the site published it), instant (one or two requests), and
6
+ * needs no crawling, no robots.txt politeness budget and no HTML parsing. A
7
+ * crawler is the fallback for sites without one, and is deliberately not built
8
+ * yet — everything downstream works without it.
9
+ *
10
+ * Sitemaps are parsed with regular expressions rather than an XML library.
11
+ * That is normally a mistake, but a sitemap is a machine-generated document
12
+ * with a fixed two-element vocabulary, and the alternative is a dependency
13
+ * carried by every install of this server for one code path. The parser reads
14
+ * <loc> only and ignores everything else, so malformed markup degrades to
15
+ * fewer URLs rather than to wrong ones.
16
+ */
17
+ export interface SitemapResult {
18
+ urls: string[];
19
+ /** Every sitemap document actually fetched, in order. */
20
+ sitemapsRead: string[];
21
+ /** True when a cap stopped the walk early — the URL list is a subset. */
22
+ truncated: boolean;
23
+ /** Every location tried, so a failure can say what was ruled out. */
24
+ attempted: string[];
25
+ warnings: string[];
26
+ }
27
+ export declare function readSitemap(origin: string): Promise<SitemapResult>;
@@ -0,0 +1,272 @@
1
+ /**
2
+ * Sitemap discovery.
3
+ *
4
+ * The cheapest honest way to learn which pages a site has. A sitemap is
5
+ * authoritative (the site published it), instant (one or two requests), and
6
+ * needs no crawling, no robots.txt politeness budget and no HTML parsing. A
7
+ * crawler is the fallback for sites without one, and is deliberately not built
8
+ * yet — everything downstream works without it.
9
+ *
10
+ * Sitemaps are parsed with regular expressions rather than an XML library.
11
+ * That is normally a mistake, but a sitemap is a machine-generated document
12
+ * with a fixed two-element vocabulary, and the alternative is a dependency
13
+ * carried by every install of this server for one code path. The parser reads
14
+ * <loc> only and ignores everything else, so malformed markup degrades to
15
+ * fewer URLs rather than to wrong ones.
16
+ */
17
+ import { gunzipSync } from "node:zlib";
18
+ import { httpGet } from "./httpClient.js";
19
+ /** A sitemap index can point at hundreds of children; walking all of them is rarely worth it. */
20
+ const MAX_SITEMAPS = 25;
21
+ const MAX_URLS = 50_000;
22
+ const FETCH_TIMEOUT_MS = 20_000;
23
+ const LOC_RE = /<loc>\s*([^<\s][^<]*?)\s*<\/loc>/gi;
24
+ function decodeXmlEntities(value) {
25
+ return value
26
+ .replace(/&lt;/g, "<")
27
+ .replace(/&gt;/g, ">")
28
+ .replace(/&quot;/g, '"')
29
+ .replace(/&apos;/g, "'")
30
+ .replace(/&#(\d+);/g, (_, d) => String.fromCharCode(Number(d)))
31
+ // Ampersand last, so "&amp;lt;" does not become "<".
32
+ .replace(/&amp;/g, "&");
33
+ }
34
+ function extractLocs(xml) {
35
+ const out = [];
36
+ for (const match of xml.matchAll(LOC_RE)) {
37
+ const loc = decodeXmlEntities(match[1].trim());
38
+ if (loc)
39
+ out.push(loc);
40
+ }
41
+ return out;
42
+ }
43
+ /** A <sitemapindex> points at more sitemaps; a <urlset> holds pages. */
44
+ function isSitemapIndex(xml) {
45
+ return /<sitemapindex[\s>]/i.test(xml);
46
+ }
47
+ /**
48
+ * Sitemap locations declared in robots.txt.
49
+ *
50
+ * Large sites frequently do not serve /sitemap.xml and only announce the real
51
+ * location here, so checking robots first turns a "no sitemap found" dead end
52
+ * into a hit. It is also the polite thing to read before touching a site.
53
+ */
54
+ async function sitemapsFromRobots(origin) {
55
+ const robotsUrl = new URL("/robots.txt", origin).toString();
56
+ const result = await httpGet(robotsUrl, { timeoutMs: 10_000, retries: 1 });
57
+ if (!result.ok)
58
+ return [];
59
+ const found = [];
60
+ for (const line of result.body.split(/\r?\n/)) {
61
+ const match = /^\s*sitemap:\s*(\S+)/i.exec(line);
62
+ if (match)
63
+ found.push(match[1]);
64
+ }
65
+ return found;
66
+ }
67
+ /**
68
+ * Non-HTML entries a performance audit should never spend a PSI call on.
69
+ * Image and video sitemaps are common and would otherwise fill the sample.
70
+ */
71
+ function looksAuditable(url) {
72
+ return !/\.(jpe?g|png|gif|webp|avif|svg|ico|pdf|zip|gz|mp4|webm|mp3|xml|json|txt|css|js)(\?|$)/i.test(url);
73
+ }
74
+ /**
75
+ * Sitemap locations worth guessing, beyond the two standard ones.
76
+ *
77
+ * Ordered by how often they pay off. These only run when the standard
78
+ * locations and robots.txt have produced nothing, so the common case still
79
+ * costs one or two requests — a 404 from Google's edge is cheap, but seven of
80
+ * them on every audit would not be.
81
+ */
82
+ const CANDIDATE_PATHS = [
83
+ "/sitemap-index.xml",
84
+ "/sitemap/sitemap.xml",
85
+ "/sitemap/index.xml",
86
+ "/wp-sitemap.xml", // WordPress 5.5+
87
+ "/sitemap_index.xml.gz",
88
+ "/sitemap1.xml",
89
+ "/sitemap.txt", // plain text, one URL per line
90
+ ];
91
+ /**
92
+ * `<link rel="sitemap">` in the homepage head.
93
+ *
94
+ * Rare but authoritative when present, and it costs one request we can often
95
+ * justify anyway. Only consulted after the cheaper guesses fail.
96
+ */
97
+ async function sitemapFromHomepageLink(origin) {
98
+ const result = await httpGet(origin, { timeoutMs: 15_000, retries: 0 });
99
+ if (!result.ok)
100
+ return [];
101
+ const found = [];
102
+ for (const match of result.body.matchAll(/<link\b[^>]*>/gi)) {
103
+ const tag = match[0];
104
+ if (!/rel=["']?sitemap["']?/i.test(tag))
105
+ continue;
106
+ const href = /href=["']([^"']+)["']/i.exec(tag)?.[1];
107
+ if (href) {
108
+ try {
109
+ found.push(new URL(href, origin).toString());
110
+ }
111
+ catch {
112
+ // Unparseable href — ignore rather than fail discovery over it.
113
+ }
114
+ }
115
+ }
116
+ return found;
117
+ }
118
+ /**
119
+ * A sitemap.txt is a bare newline-delimited URL list, not XML. Detected by
120
+ * content rather than extension, because servers are inconsistent about both.
121
+ */
122
+ function isPlainTextSitemap(body) {
123
+ const head = body.trimStart().slice(0, 200);
124
+ return !head.startsWith("<") && /^https?:\/\//im.test(head);
125
+ }
126
+ function extractPlainTextUrls(body) {
127
+ return body
128
+ .split(/\r?\n/)
129
+ .map((line) => line.trim())
130
+ .filter((line) => /^https?:\/\//i.test(line));
131
+ }
132
+ export async function readSitemap(origin) {
133
+ const warnings = [];
134
+ const sitemapsRead = [];
135
+ const attempted = [];
136
+ const urls = new Set();
137
+ const seen = new Set();
138
+ let truncated = false;
139
+ /**
140
+ * Tiers, cheapest and most authoritative first. Each is tried only if every
141
+ * earlier one came up empty, so a site that declares its sitemap properly
142
+ * still costs two requests while a site that hides it gets a real search.
143
+ */
144
+ const declared = await sitemapsFromRobots(origin);
145
+ const tiers = [
146
+ { name: "robots.txt", locations: async () => declared },
147
+ {
148
+ name: "standard locations",
149
+ locations: async () => [
150
+ new URL("/sitemap.xml", origin).toString(),
151
+ new URL("/sitemap_index.xml", origin).toString(),
152
+ ],
153
+ },
154
+ { name: "homepage <link rel=sitemap>", locations: () => sitemapFromHomepageLink(origin) },
155
+ {
156
+ name: "common CMS locations",
157
+ locations: async () => CANDIDATE_PATHS.map((path) => new URL(path, origin).toString()),
158
+ },
159
+ ];
160
+ for (const tier of tiers) {
161
+ // Stop on the first tier that finds a sitemap *document*, not the first
162
+ // that yields URLs. web.dev serves a valid index whose children time out;
163
+ // falling through on an empty result sent discovery guessing at nine more
164
+ // locations for 43s when the sitemap had already been located.
165
+ if (sitemapsRead.length > 0)
166
+ break;
167
+ const queue = await tier.locations();
168
+ let consecutiveFailures = 0;
169
+ while (queue.length > 0) {
170
+ if (sitemapsRead.length >= MAX_SITEMAPS || urls.size >= MAX_URLS) {
171
+ truncated = true;
172
+ break;
173
+ }
174
+ const next = queue.shift();
175
+ if (seen.has(next))
176
+ continue;
177
+ seen.add(next);
178
+ attempted.push(next);
179
+ // Large sites commonly serve .xml.gz. It arrives as an opaque gzip payload
180
+ // rather than with Content-Encoding, so fetch does not inflate it and a
181
+ // text read produces binary noise — hence the explicit binary path.
182
+ const gzipped = /\.gz(\?|$)/i.test(next);
183
+ // No retry: discovery must stay fast, and a sitemap that times out once
184
+ // usually times out again. Retrying doubled the cost of the slow case
185
+ // for no observed benefit.
186
+ const result = await httpGet(next, {
187
+ timeoutMs: FETCH_TIMEOUT_MS,
188
+ retries: 0,
189
+ raw: gzipped,
190
+ });
191
+ if (!result.ok) {
192
+ // A 404 or 403 on a guessed location is the expected answer to "is it
193
+ // here?" — cheap, informative, and no reason to stop guessing. Only
194
+ // expensive failures (timeouts, 5xx) count toward giving up.
195
+ if (result.status === 404 || result.status === 403)
196
+ continue;
197
+ consecutiveFailures++;
198
+ // A sitemap index can list hundreds of children. If the first few all
199
+ // time out, the rest almost certainly will too, at a full timeout each.
200
+ if (consecutiveFailures >= 3) {
201
+ warnings.push(`Gave up on ${tier.name} after ${consecutiveFailures} consecutive failures ` +
202
+ `(last: ${result.error ?? `HTTP ${result.status}`}).`);
203
+ break;
204
+ }
205
+ warnings.push(`Could not read ${next}: ${result.error ?? `HTTP ${result.status}`}`);
206
+ continue;
207
+ }
208
+ consecutiveFailures = 0;
209
+ let body = result.body;
210
+ if (gzipped) {
211
+ try {
212
+ body = gunzipSync(Buffer.from(result.bytes ?? new Uint8Array())).toString("utf8");
213
+ }
214
+ catch (err) {
215
+ warnings.push(`Could not decompress ${next}: ${err.message}`);
216
+ continue;
217
+ }
218
+ }
219
+ if (isPlainTextSitemap(body)) {
220
+ const plain = extractPlainTextUrls(body).filter(looksAuditable);
221
+ if (plain.length === 0)
222
+ continue;
223
+ sitemapsRead.push(next);
224
+ for (const loc of plain) {
225
+ if (urls.size >= MAX_URLS) {
226
+ truncated = true;
227
+ break;
228
+ }
229
+ urls.add(loc);
230
+ }
231
+ continue;
232
+ }
233
+ const locs = extractLocs(body);
234
+ if (locs.length === 0)
235
+ continue;
236
+ sitemapsRead.push(next);
237
+ if (isSitemapIndex(body)) {
238
+ for (const child of locs)
239
+ if (!seen.has(child))
240
+ queue.push(child);
241
+ continue;
242
+ }
243
+ for (const loc of locs) {
244
+ if (urls.size >= MAX_URLS) {
245
+ truncated = true;
246
+ break;
247
+ }
248
+ if (looksAuditable(loc))
249
+ urls.add(loc);
250
+ }
251
+ }
252
+ }
253
+ if (sitemapsRead.length > 0 && urls.size === 0) {
254
+ warnings.push(`Found a sitemap at ${sitemapsRead[0]} but could not extract any URLs from it — ` +
255
+ `its child documents failed to load or contained no page entries. Supply URLs ` +
256
+ `with discovery:"list" or discovery:"csv".`);
257
+ }
258
+ if (sitemapsRead.length === 0) {
259
+ warnings.push(`No sitemap found for ${origin}. Tried ${attempted.length} location(s): ` +
260
+ `robots.txt, the standard paths, the homepage <link rel="sitemap">, and ` +
261
+ `common CMS locations. ` +
262
+ `Supply URLs directly with discovery:"list", or point at a CSV export with ` +
263
+ `discovery:"csv" — an analytics top-pages export is the better input for a ` +
264
+ `performance audit anyway, since it is weighted by real traffic.`);
265
+ }
266
+ if (truncated) {
267
+ warnings.push(`Stopped after ${sitemapsRead.length} sitemap documents and ${urls.size} URLs. ` +
268
+ `The template breakdown below is based on that subset.`);
269
+ }
270
+ return { urls: [...urls], sitemapsRead, truncated, attempted, warnings };
271
+ }
272
+ //# sourceMappingURL=sitemapReader.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"sitemapReader.js","sourceRoot":"","sources":["../../src/utils/sitemapReader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAa1C,iGAAiG;AACjG,MAAM,YAAY,GAAG,EAAE,CAAC;AACxB,MAAM,QAAQ,GAAG,MAAM,CAAC;AACxB,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,MAAM,MAAM,GAAG,oCAAoC,CAAC;AAEpD,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,qDAAqD;SACpD,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/C,IAAI,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,wEAAwE;AACxE,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,MAAc;IAC9C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC5D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,CAAC,wFAAwF,CAAC,IAAI,CACnG,GAAG,CACJ,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG;IACtB,oBAAoB;IACpB,sBAAsB;IACtB,oBAAoB;IACpB,iBAAiB,EAAS,iBAAiB;IAC3C,uBAAuB;IACvB,eAAe;IACf,cAAc,EAAY,+BAA+B;CAC1D,CAAC;AAEF;;;;;GAKG;AACH,KAAK,UAAU,uBAAuB,CAAC,MAAc;IACnD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IACxE,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC5D,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAClD,MAAM,IAAI,GAAG,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACrD,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC/C,CAAC;YAAC,MAAM,CAAC;gBACP,gEAAgE;YAClE,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,OAAO,IAAI;SACR,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAc;IAC9C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB;;;;OAIG;IACH,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,KAAK,GAAgE;QACzE,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC,QAAQ,EAAE;QACvD;YACE,IAAI,EAAE,oBAAoB;YAC1B,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC;gBACrB,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE;gBAC1C,IAAI,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE;aACjD;SACF;QACD,EAAE,IAAI,EAAE,6BAA6B,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE;QACzF;YACE,IAAI,EAAE,sBAAsB;YAC5B,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;SACvF;KACF,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,wEAAwE;QACxE,0EAA0E;QAC1E,0EAA0E;QAC1E,+DAA+D;QAC/D,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM;QAEnC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,IAAI,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;gBACjE,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;YAED,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAY,CAAC;YACrC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAErB,2EAA2E;YAC3E,wEAAwE;YACxE,oEAAoE;YACpE,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,wEAAwE;YACxE,sEAAsE;YACtE,2BAA2B;YAC3B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;gBACjC,SAAS,EAAE,gBAAgB;gBAC3B,OAAO,EAAE,CAAC;gBACV,GAAG,EAAE,OAAO;aACb,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,sEAAsE;gBACtE,oEAAoE;gBACpE,6DAA6D;gBAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG;oBAAE,SAAS;gBAE7D,mBAAmB,EAAE,CAAC;gBACtB,sEAAsE;gBACtE,wEAAwE;gBACxE,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;oBAC7B,QAAQ,CAAC,IAAI,CACX,cAAc,IAAI,CAAC,IAAI,UAAU,mBAAmB,wBAAwB;wBAC1E,UAAU,MAAM,CAAC,KAAK,IAAI,QAAQ,MAAM,CAAC,MAAM,EAAE,IAAI,CACxD,CAAC;oBACF,MAAM;gBACR,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,kBAAkB,IAAI,KAAK,MAAM,CAAC,KAAK,IAAI,QAAQ,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACpF,SAAS;YACX,CAAC;YACD,mBAAmB,GAAG,CAAC,CAAC;YAExB,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,CAAC;oBACH,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACpF,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,QAAQ,CAAC,IAAI,CAAC,wBAAwB,IAAI,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;oBACzE,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBAChE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBACjC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACxB,IAAI,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;wBAC1B,SAAS,GAAG,IAAI,CAAC;wBACjB,MAAM;oBACR,CAAC;oBACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAChC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAExB,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,KAAK,MAAM,KAAK,IAAI,IAAI;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;wBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAClE,SAAS;YACX,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC1B,SAAS,GAAG,IAAI,CAAC;oBACjB,MAAM;gBACR,CAAC;gBACD,IAAI,cAAc,CAAC,GAAG,CAAC;oBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CACX,sBAAsB,YAAY,CAAC,CAAC,CAAC,4CAA4C;YAC/E,+EAA+E;YAC/E,2CAA2C,CAC9C,CAAC;IACJ,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CACX,wBAAwB,MAAM,WAAW,SAAS,CAAC,MAAM,gBAAgB;YACvE,yEAAyE;YACzE,wBAAwB;YACxB,4EAA4E;YAC5E,4EAA4E;YAC5E,iEAAiE,CACpE,CAAC;IACJ,CAAC;IACD,IAAI,SAAS,EAAE,CAAC;QACd,QAAQ,CAAC,IAAI,CACX,iBAAiB,YAAY,CAAC,MAAM,0BAA0B,IAAI,CAAC,IAAI,SAAS;YAC9E,uDAAuD,CAC1D,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAC3E,CAAC"}