@rivi-software/agentic-seo-check 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +49 -7
  2. package/dist/cli.js +28 -0
  3. package/dist/cli.js.map +1 -1
  4. package/dist/core/aeo/audit.d.ts +8 -0
  5. package/dist/core/aeo/audit.js +101 -17
  6. package/dist/core/aeo/audit.js.map +1 -1
  7. package/dist/core/aeo/bots.d.ts +48 -0
  8. package/dist/core/aeo/bots.js +151 -5
  9. package/dist/core/aeo/bots.js.map +1 -1
  10. package/dist/core/aeo/fixes.d.ts +38 -0
  11. package/dist/core/aeo/fixes.js +111 -0
  12. package/dist/core/aeo/fixes.js.map +1 -0
  13. package/dist/core/aeo/site-rules.d.ts +5 -0
  14. package/dist/core/aeo/site-rules.js +293 -10
  15. package/dist/core/aeo/site-rules.js.map +1 -1
  16. package/dist/core/audit.d.ts +8 -0
  17. package/dist/core/audit.js +144 -25
  18. package/dist/core/audit.js.map +1 -1
  19. package/dist/core/crawl.js +43 -4
  20. package/dist/core/crawl.js.map +1 -1
  21. package/dist/core/html/answer.d.ts +5 -2
  22. package/dist/core/html/answer.js +25 -7
  23. package/dist/core/html/answer.js.map +1 -1
  24. package/dist/core/html/dom.d.ts +13 -1
  25. package/dist/core/html/dom.js +30 -2
  26. package/dist/core/html/dom.js.map +1 -1
  27. package/dist/core/html/extract.js +122 -30
  28. package/dist/core/html/extract.js.map +1 -1
  29. package/dist/core/html/head.js +114 -19
  30. package/dist/core/html/head.js.map +1 -1
  31. package/dist/core/html/resources.js +15 -7
  32. package/dist/core/html/resources.js.map +1 -1
  33. package/dist/core/html/structured-data.js +21 -5
  34. package/dist/core/html/structured-data.js.map +1 -1
  35. package/dist/core/lighthouse.d.ts +5 -6
  36. package/dist/core/lighthouse.js +297 -7
  37. package/dist/core/lighthouse.js.map +1 -1
  38. package/dist/core/performance-fixes.d.ts +25 -0
  39. package/dist/core/performance-fixes.js +446 -0
  40. package/dist/core/performance-fixes.js.map +1 -0
  41. package/dist/core/performance-rules.js +56 -8
  42. package/dist/core/performance-rules.js.map +1 -1
  43. package/dist/core/recheck.d.ts +68 -0
  44. package/dist/core/recheck.js +247 -0
  45. package/dist/core/recheck.js.map +1 -0
  46. package/dist/core/render-fixes.d.ts +27 -0
  47. package/dist/core/render-fixes.js +164 -0
  48. package/dist/core/render-fixes.js.map +1 -0
  49. package/dist/core/render-rules.js +60 -21
  50. package/dist/core/render-rules.js.map +1 -1
  51. package/dist/core/render.d.ts +9 -1
  52. package/dist/core/render.js +36 -2
  53. package/dist/core/render.js.map +1 -1
  54. package/dist/core/report/markdown.js +68 -1
  55. package/dist/core/report/markdown.js.map +1 -1
  56. package/dist/core/robots.d.ts +5 -0
  57. package/dist/core/robots.js +8 -0
  58. package/dist/core/robots.js.map +1 -1
  59. package/dist/core/rules/aeo.js +339 -25
  60. package/dist/core/rules/aeo.js.map +1 -1
  61. package/dist/core/rules/canonical.js +107 -11
  62. package/dist/core/rules/canonical.js.map +1 -1
  63. package/dist/core/rules/content.js +65 -7
  64. package/dist/core/rules/content.js.map +1 -1
  65. package/dist/core/rules/document.js +73 -8
  66. package/dist/core/rules/document.js.map +1 -1
  67. package/dist/core/rules/fix.d.ts +55 -0
  68. package/dist/core/rules/fix.js +159 -0
  69. package/dist/core/rules/fix.js.map +1 -0
  70. package/dist/core/rules/head.js +81 -4
  71. package/dist/core/rules/head.js.map +1 -1
  72. package/dist/core/rules/headings.js +92 -9
  73. package/dist/core/rules/headings.js.map +1 -1
  74. package/dist/core/rules/hreflang.js +79 -9
  75. package/dist/core/rules/hreflang.js.map +1 -1
  76. package/dist/core/rules/images.js +49 -0
  77. package/dist/core/rules/images.js.map +1 -1
  78. package/dist/core/rules/index.d.ts +11 -0
  79. package/dist/core/rules/index.js +21 -1
  80. package/dist/core/rules/index.js.map +1 -1
  81. package/dist/core/rules/indexing.js +108 -1
  82. package/dist/core/rules/indexing.js.map +1 -1
  83. package/dist/core/rules/keyword.js +91 -1
  84. package/dist/core/rules/keyword.js.map +1 -1
  85. package/dist/core/rules/links.js +42 -0
  86. package/dist/core/rules/links.js.map +1 -1
  87. package/dist/core/rules/loading.js +54 -1
  88. package/dist/core/rules/loading.js.map +1 -1
  89. package/dist/core/rules/meta.js +36 -5
  90. package/dist/core/rules/meta.js.map +1 -1
  91. package/dist/core/rules/page-fixes.d.ts +39 -0
  92. package/dist/core/rules/page-fixes.js +107 -0
  93. package/dist/core/rules/page-fixes.js.map +1 -0
  94. package/dist/core/rules/quoted-tags.d.ts +23 -0
  95. package/dist/core/rules/quoted-tags.js +79 -0
  96. package/dist/core/rules/quoted-tags.js.map +1 -0
  97. package/dist/core/rules/response.js +71 -7
  98. package/dist/core/rules/response.js.map +1 -1
  99. package/dist/core/rules/rule.d.ts +17 -4
  100. package/dist/core/rules/rule.js +11 -2
  101. package/dist/core/rules/rule.js.map +1 -1
  102. package/dist/core/rules/social.js +101 -10
  103. package/dist/core/rules/social.js.map +1 -1
  104. package/dist/core/rules/structured-data-fixes.d.ts +33 -0
  105. package/dist/core/rules/structured-data-fixes.js +469 -0
  106. package/dist/core/rules/structured-data-fixes.js.map +1 -0
  107. package/dist/core/rules/structured-data-values.js +242 -31
  108. package/dist/core/rules/structured-data-values.js.map +1 -1
  109. package/dist/core/rules/structured-data.js +171 -4
  110. package/dist/core/rules/structured-data.js.map +1 -1
  111. package/dist/core/rules/title.js +39 -5
  112. package/dist/core/rules/title.js.map +1 -1
  113. package/dist/core/rules/urls.js +166 -0
  114. package/dist/core/rules/urls.js.map +1 -1
  115. package/dist/core/site-fixes.d.ts +111 -0
  116. package/dist/core/site-fixes.js +650 -0
  117. package/dist/core/site-fixes.js.map +1 -0
  118. package/dist/core/site-rules.js +474 -53
  119. package/dist/core/site-rules.js.map +1 -1
  120. package/dist/core/sitemap.d.ts +16 -0
  121. package/dist/core/sitemap.js +34 -0
  122. package/dist/core/sitemap.js.map +1 -1
  123. package/dist/core/types.d.ts +223 -3
  124. package/dist/core/types.js.map +1 -1
  125. package/dist/index.d.ts +2 -0
  126. package/dist/index.js +1 -0
  127. package/dist/index.js.map +1 -1
  128. package/dist/mcp.js +16 -2
  129. package/dist/mcp.js.map +1 -1
  130. package/package.json +5 -7
  131. package/skills/seo-audit/SKILL.md +23 -6
@@ -1,8 +1,11 @@
1
1
  import { normalizeForComparison, shingles, jaccard } from "./html/text.js";
2
2
  import { agentMatcher } from "./robots.js";
3
+ import { escapeAttribute, removeEdit, setAttributeEdit, slot, } from "./rules/fix.js";
3
4
  import { robotsDirectives } from "./rules/indexing.js";
4
5
  import { finding, GOOGLE_DOCS, cap } from "./rules/rule.js";
6
+ import { addLinkEdit, addToSitemapFix, blockedResourcesFix, canonicalEdit, imagesOf, blockingRule, lastmodFix, locationOf, linkSelector, linksTo, nestedIndexFix, hrefLike, onUrl, otherHostFix, redirectTo, redescribeEdit, removeNoindexEdits, returnLinkEdit, reheadEdit, relinkEdit, linesOf, retitleEdit, statusLine, removeFromSitemap, replaceInSitemap, robotsBlocksAllFix, robotsErrorFix, robotsMissingFix, robotsTooLargeFix, robotsUnsupportedFix, MAX_EDITS, sitemapErrorFix, sitemapMissingFix, splitSitemapFix, } from "./site-fixes.js";
5
7
  import { MIN_COMPARABLE_WORDS, SAME_TEXT } from "./site-probes.js";
8
+ import { sitemapLayout } from "./sitemap.js";
6
9
  import { normalizeUrl, normalizePath, trimTrailingSlashes } from "./url.js";
7
10
  const meta = (id, severity, description, recommendation, docs) => ({
8
11
  id,
@@ -85,8 +88,21 @@ const pathKey = (url) => {
85
88
  return url;
86
89
  }
87
90
  };
88
- const pushDuplicates = (out, rule, pages, pick, label) => {
91
+ /** The URL with its path lower-cased and trailing slashes dropped: one key for every spelling. */
92
+ const spellingKey = (url) => {
93
+ try {
94
+ const u = new URL(url);
95
+ const pathname = trimTrailingSlashes(u.pathname.toLowerCase()) || "/";
96
+ return `${u.origin}${pathname}${u.search}`;
97
+ }
98
+ catch {
99
+ return url;
100
+ }
101
+ };
102
+ const pushDuplicates = (out, rule, pages, pick, label, fixFor) => {
89
103
  const groups = new Map();
104
+ // Every page but the first of a group gets a value of its own; the first keeps what it has.
105
+ const others = [];
90
106
  // Pages are grouped by a lowercased key, but the evidence quotes the value as the first page
91
107
  // wrote it, so a report or pull request shows the text a reviewer would find on the site.
92
108
  const written = new Map();
@@ -97,7 +113,10 @@ const pushDuplicates = (out, rule, pages, pick, label) => {
97
113
  }
98
114
  const k = normalizeForComparison(value);
99
115
  pushTo(groups, k, p.url);
100
- if (!written.has(k)) {
116
+ if (written.has(k)) {
117
+ others.push(p);
118
+ }
119
+ else {
101
120
  written.set(k, value);
102
121
  }
103
122
  }
@@ -107,6 +126,7 @@ const pushDuplicates = (out, rule, pages, pick, label) => {
107
126
  }
108
127
  const pagesAffected = dupes.reduce((n, [, urls]) => n + urls.length, 0);
109
128
  out.push(finding(rule, `${dupes.length} ${label}(s) are shared by ${pagesAffected} pages.`, {
129
+ fix: others.flatMap((p) => fixFor(p) ?? []).slice(0, MAX_EDITS),
110
130
  evidence: {
111
131
  groups: cap(dupes.map(([k, urls]) => ({
112
132
  value: (written.get(k) ?? k).slice(0, 120),
@@ -116,6 +136,34 @@ const pushDuplicates = (out, rule, pages, pick, label) => {
116
136
  },
117
137
  }));
118
138
  };
139
+ /**
140
+ * The canonical each duplicate should name. The first page of a group is the shallowest the crawl
141
+ * reached: the others name it as canonical. Near-duplicates come in pairs, so a page can be in two
142
+ * groups (a~b and b~c, or a~c and b~c): each page gets one canonical, to a page that keeps its
143
+ * own, so the fix makes no chain and gives no page two.
144
+ */
145
+ const duplicateCanonicalEdits = (groups, byUrl) => {
146
+ const canonicalOf = new Map();
147
+ const roots = new Set();
148
+ const fix = [];
149
+ for (const { pages: [first, ...rest], } of groups) {
150
+ const root = first ? (canonicalOf.get(first) ?? first) : undefined;
151
+ for (const u of rest) {
152
+ const record = byUrl.get(u);
153
+ if (!root ||
154
+ !record ||
155
+ u === root ||
156
+ canonicalOf.has(u) ||
157
+ roots.has(u)) {
158
+ continue;
159
+ }
160
+ canonicalOf.set(u, root);
161
+ roots.add(root);
162
+ fix.push(canonicalEdit(record, root, `If ${u} is meant to be a page of its own, make its text distinct instead.`));
163
+ }
164
+ }
165
+ return fix;
166
+ };
119
167
  const pushDuplicateContent = (out, pages) => {
120
168
  const candidates = pages.filter((p) => p.page.wordCount >= 50);
121
169
  const groups = [];
@@ -156,7 +204,9 @@ const pushDuplicateContent = (out, pages) => {
156
204
  }
157
205
  }
158
206
  if (groups.length) {
159
- out.push(finding(m("site.duplicate-content"), `${groups.length} group(s) of pages have (near-)identical body text.`, { evidence: { groups: cap(groups) } }));
207
+ const byUrl = new Map(candidates.map((p) => [p.url, p]));
208
+ const fix = duplicateCanonicalEdits(groups, byUrl);
209
+ out.push(finding(m("site.duplicate-content"), `${groups.length} group(s) of pages have (near-)identical body text.`, { fix: fix.slice(0, MAX_EDITS), evidence: { groups: cap(groups) } }));
160
210
  }
161
211
  };
162
212
  /** Google reads the first 500 KiB of robots.txt and ignores the rest. */
@@ -164,7 +214,7 @@ const ROBOTS_MAX_BYTES = 512_000;
164
214
  const unsupportedRobotsLines = (text) => {
165
215
  const retired = [];
166
216
  const crawlDelay = [];
167
- for (const [i, raw] of text.split(/\r?\n/u).slice(0, 50_000).entries()) {
217
+ for (const [i, raw] of linesOf(text).slice(0, 50_000).entries()) {
168
218
  const line = raw.replace(/#.*$/u, "").trim();
169
219
  const field = /^(?<field>[a-z-]+)\s*:/iu
170
220
  .exec(line)
@@ -182,10 +232,14 @@ const unsupportedRobotsLines = (text) => {
182
232
  const robotsContentFindings = (ctx) => {
183
233
  const out = [];
184
234
  const bytes = ctx.robots?.bytes ?? 0;
235
+ const text = ctx.robotsText ?? ctx.robots?.text;
236
+ const robotsUrl = ctx.robots?.url ?? new URL("/robots.txt", ctx.startUrl).href;
185
237
  if (bytes > ROBOTS_MAX_BYTES) {
186
- out.push(finding(m("site.robots-txt-too-large"), `robots.txt is ${Math.round(bytes / 1024)} KiB; Google reads only the first 500 KiB.`, { evidence: { url: ctx.robots?.url, bytes } }));
238
+ out.push(finding(m("site.robots-txt-too-large"), `robots.txt is ${Math.round(bytes / 1024)} KiB; Google reads only the first 500 KiB.`, {
239
+ fix: robotsTooLargeFix(robotsUrl, text ?? "", bytes),
240
+ evidence: { url: ctx.robots?.url, bytes },
241
+ }));
187
242
  }
188
- const text = ctx.robotsText ?? ctx.robots?.text;
189
243
  if (!text) {
190
244
  return out;
191
245
  }
@@ -195,6 +249,7 @@ const robotsContentFindings = (ctx) => {
195
249
  ? `robots.txt has ${retired.length} noindex or nofollow line(s), which Google stopped reading in 2019.`
196
250
  : `robots.txt sets crawl-delay, which Google ignores.`, {
197
251
  severity: retired.length > 0 ? "warning" : "info",
252
+ fix: robotsUnsupportedFix(robotsUrl, text),
198
253
  evidence: {
199
254
  url: ctx.robots?.url,
200
255
  retired: cap(retired),
@@ -252,28 +307,44 @@ const blockedResourceFindings = ({ ctx, okPages, isIndexable, }) => {
252
307
  .toSorted((a, b) => b.pages - a.pages);
253
308
  return list.length > 0
254
309
  ? [
255
- finding(m("site.robots-blocks-resources"), `robots.txt keeps Googlebot from ${list.length} script(s) or stylesheet(s) the pages load.`, { evidence: { count: list.length, examples: cap(list) } }),
310
+ finding(m("site.robots-blocks-resources"), `robots.txt keeps Googlebot from ${list.length} script(s) or stylesheet(s) the pages load.`, {
311
+ fix: blockedResourcesFix(robots.url, text, list.map((r) => r.url)),
312
+ evidence: { count: list.length, examples: cap(list) },
313
+ }),
256
314
  ]
257
315
  : [];
258
316
  };
259
317
  /** Sitemaps over the protocol's limits, nested indexes, bad lastmod values and other hosts. */
318
+ /** Parts of at most 45,000 URLs or 45 MB, and at least two. */
319
+ const partsFor = (urls, bytes, perPart) => Math.max(2, Math.ceil((urls ?? 0) / 45_000), Math.ceil((bytes ?? 0) / perPart));
260
320
  const sitemapProtocolFindings = (ctx) => {
261
321
  const { problems } = ctx.sitemaps;
322
+ const layout = sitemapLayout(problems);
323
+ const today = new Date().toISOString().slice(0, 10);
262
324
  const out = [];
263
325
  const over = problems.oversized.filter((o) => o.overLimit);
264
326
  const unread = problems.oversized.filter((o) => !o.overLimit);
265
327
  if (over.length > 0) {
266
- out.push(finding(m("site.sitemap-too-large"), `${over.length} sitemap(s) exceed 50,000 URLs or 50 MB.`, { evidence: { sitemaps: cap(over) } }));
328
+ out.push(finding(m("site.sitemap-too-large"), `${over.length} sitemap(s) exceed 50,000 URLs or 50 MB.`, {
329
+ fix: over.map((o) => splitSitemapFix(o.sitemap, partsFor(o.urls, o.bytes, 45_000_000), "Serve this sitemap index where the large sitemap was, and move its URLs into the files it lists, each under 50,000 URLs and 50 MB.")),
330
+ key: "over-limit",
331
+ evidence: { sitemaps: cap(over) },
332
+ }));
267
333
  }
268
334
  if (unread.length > 0) {
269
335
  out.push(finding(m("site.sitemap-too-large"), `${unread.length} sitemap(s) are larger than the 15 MB this audit reads, so their URLs were not checked.`, {
270
336
  severity: "info",
271
337
  recommendation: "Check that each stays under 50 MB and 50,000 URLs; smaller sitemaps (10,000 URLs or fewer) are also quicker for search engines to fetch again.",
338
+ fix: unread.map((o) => splitSitemapFix(o.sitemap, partsFor(o.urls, o.bytes, 10_000_000), "Serve this sitemap index where the large sitemap was, and move its URLs into the files it lists, 10,000 URLs or fewer each, so search engines (and this audit) can read each one quickly.")),
339
+ key: "unread",
272
340
  evidence: { sitemaps: cap(unread) },
273
341
  }));
274
342
  }
275
343
  if (problems.nestedIndexes.length > 0) {
276
- out.push(finding(m("site.sitemap-index-nested"), `${problems.nestedIndexes.length} sitemap index(es) are listed by another sitemap index.`, { evidence: { indexes: cap(problems.nestedIndexes) } }));
344
+ out.push(finding(m("site.sitemap-index-nested"), `${problems.nestedIndexes.length} sitemap index(es) are listed by another sitemap index.`, {
345
+ fix: problems.nestedIndexes.map((n) => nestedIndexFix(layout, n)),
346
+ evidence: { indexes: cap(problems.nestedIndexes) },
347
+ }));
277
348
  }
278
349
  const examples = [
279
350
  [
@@ -292,22 +363,69 @@ const sitemapProtocolFindings = (ctx) => {
292
363
  "are on another host than the site",
293
364
  ],
294
365
  ];
366
+ const fixes = {
367
+ "site.sitemap-lastmod-invalid": (list) => lastmodFix(layout, list, "invalid", today),
368
+ "site.sitemap-lastmod-future": (list) => lastmodFix(layout, list, "future", today),
369
+ "site.sitemap-other-host": (list) => otherHostFix(layout, list, new URL(ctx.startUrl).origin),
370
+ };
295
371
  for (const [id, list, label] of examples) {
296
372
  if (list.count > 0) {
297
373
  out.push(finding(m(id), `${list.count} sitemap URL(s) ${label}.`, {
374
+ fix: fixes[id](list.examples),
298
375
  evidence: { count: list.count, examples: list.examples },
299
376
  }));
300
377
  }
301
378
  }
302
379
  return out;
303
380
  };
304
- const robotsFindings = (ctx) => {
381
+ /** The sitemap URL a robots.txt should name: the one found, else /sitemap.xml. */
382
+ const sitemapUrlOf = (ctx) => ctx.sitemaps.sources[0] ?? new URL("/sitemap.xml", ctx.startUrl).href;
383
+ /**
384
+ * Keeps blocked pages out of the sitemap and stops crawlers following links to them; unblocking
385
+ * them could expose pages kept private on purpose, so the write names that rule instead.
386
+ */
387
+ const blockedPageEdits = (scope, blocked) => {
388
+ const { ctx, key, pagesByKey } = scope;
389
+ const k = key(blocked);
390
+ const rule = ctx.robots && ctx.robotsText
391
+ ? blockingRule(ctx.robots.url, ctx.robotsText, blocked)
392
+ : null;
393
+ const unblock = `If the page belongs in search results, remove ${rule ? `"${rule}"` : "the rule that blocks it"} from robots.txt instead.`;
394
+ const listed = ctx.sitemaps.urls.find((u) => key(u) === k);
395
+ if (listed) {
396
+ return [
397
+ removeFromSitemap(sitemapLayout(ctx.sitemaps.problems), listed, `robots.txt blocks it, so search engines cannot crawl it. ${unblock}`),
398
+ ];
399
+ }
400
+ const sources = [...ctx.linkSources].flatMap(([target, from]) => key(target) === k ? [...from] : []);
401
+ const edits = sources.flatMap((source) => {
402
+ const record = pagesByKey.get(key(source));
403
+ return record
404
+ ? linksTo(record, k, key).map((link) => onUrl(setAttributeEdit(link.tag, linkSelector(link), "rel", link.rel ? `${link.rel} nofollow` : "nofollow", `So crawlers do not follow it to a page robots.txt blocks. ${unblock}`), locationOf(record)))
405
+ : [];
406
+ });
407
+ if (edits.length > 0 || !rule || !ctx.robots) {
408
+ return edits;
409
+ }
410
+ return [
411
+ {
412
+ in: "robots.txt",
413
+ action: "remove",
414
+ url: ctx.robots.url,
415
+ current: rule,
416
+ write: `Only if ${blocked} belongs in search results; if it is kept out on purpose, leave robots.txt as it is.`,
417
+ },
418
+ ];
419
+ };
420
+ const robotsFindings = (scope) => {
421
+ const { ctx } = scope;
305
422
  const out = [];
306
423
  /* robots.txt */
307
424
  if (ctx.robots) {
308
425
  if (ctx.robots.error ||
309
426
  (ctx.robots.status !== null && ctx.robots.status >= 500)) {
310
427
  out.push(finding(m("site.robots-txt-error"), `robots.txt at ${ctx.robots.url} could not be read (${ctx.robots.error ?? `HTTP ${ctx.robots.status}`}).`, {
428
+ fix: robotsErrorFix(ctx.robots.url, ctx.robots.status, ctx.robots.error),
311
429
  evidence: {
312
430
  url: ctx.robots.url,
313
431
  status: ctx.robots.status,
@@ -318,11 +436,13 @@ const robotsFindings = (ctx) => {
318
436
  else if (!ctx.robots.found) {
319
437
  out.push(finding(m("site.robots-txt-missing"), `No robots.txt at ${ctx.robots.url}.`, {
320
438
  severity: ctx.source === "directory" ? "info" : "warning",
439
+ fix: robotsMissingFix(ctx.robots.url, sitemapUrlOf(ctx)),
321
440
  evidence: { url: ctx.robots.url, status: ctx.robots.status },
322
441
  }));
323
442
  }
324
443
  else if (ctx.robotsAllows && !ctx.robotsAllows(ctx.startUrl)) {
325
444
  out.push(finding(m("site.robots-txt-blocks-all"), "robots.txt blocks the start URL for the crawler user agent.", {
445
+ fix: robotsBlocksAllFix(ctx.robots.url, ctx.robotsText ?? ctx.robots.text ?? "", ctx.startUrl),
326
446
  evidence: {
327
447
  url: ctx.robots.url,
328
448
  robotsTxt: ctx.robots.text?.slice(0, 500),
@@ -332,6 +452,10 @@ const robotsFindings = (ctx) => {
332
452
  }
333
453
  if (ctx.blockedByRobots.length) {
334
454
  out.push(finding(m("site.robots-blocked-pages"), `${ctx.blockedByRobots.length} linked/sitemap URL(s) are disallowed by robots.txt.`, {
455
+ fix: ctx.blockedByRobots
456
+ .slice(0, MAX_EDITS)
457
+ .flatMap((u) => blockedPageEdits(scope, u))
458
+ .slice(0, MAX_EDITS),
335
459
  evidence: {
336
460
  count: ctx.blockedByRobots.length,
337
461
  examples: cap(ctx.blockedByRobots),
@@ -343,6 +467,12 @@ const robotsFindings = (ctx) => {
343
467
  /** Noindex, non-canonical, broken and redirecting URLs listed in the sitemap. */
344
468
  const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
345
469
  const out = [];
470
+ const layout = sitemapLayout(ctx.sitemaps.problems);
471
+ const listed = new Set(ctx.sitemaps.urls.map(key));
472
+ /** Points an entry at `to`, or removes it when the sitemap lists `to` already. */
473
+ const repoint = (loc, to) => listed.has(key(to))
474
+ ? removeFromSitemap(layout, loc, `the sitemap lists ${to} already.`)
475
+ : replaceInSitemap(layout, loc, to);
346
476
  const noindex = [];
347
477
  const nonCanonical = [];
348
478
  const broken = [];
@@ -370,10 +500,14 @@ const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
370
500
  }
371
501
  }
372
502
  if (noindex.length) {
373
- out.push(finding(m("site.sitemap-noindex-page"), `${noindex.length} sitemap URL(s) are noindex.`, { evidence: { count: noindex.length, examples: cap(noindex) } }));
503
+ out.push(finding(m("site.sitemap-noindex-page"), `${noindex.length} sitemap URL(s) are noindex.`, {
504
+ fix: noindex.map((u) => removeFromSitemap(layout, u, "the page is noindex, which may be on purpose. If it belongs in search results, remove its noindex instead.")),
505
+ evidence: { count: noindex.length, examples: cap(noindex) },
506
+ }));
374
507
  }
375
508
  if (nonCanonical.length) {
376
509
  out.push(finding(m("site.sitemap-non-canonical"), `${nonCanonical.length} sitemap URL(s) canonicalise to another URL.`, {
510
+ fix: nonCanonical.map((n) => repoint(n.url, n.canonical)),
377
511
  evidence: {
378
512
  count: nonCanonical.length,
379
513
  examples: cap(nonCanonical),
@@ -381,18 +515,36 @@ const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
381
515
  }));
382
516
  }
383
517
  if (broken.length) {
384
- out.push(finding(m("site.sitemap-broken-url"), `${broken.length} sitemap URL(s) return errors.`, { evidence: { count: broken.length, examples: cap(broken) } }));
518
+ out.push(finding(m("site.sitemap-broken-url"), `${broken.length} sitemap URL(s) return errors.`, {
519
+ fix: broken.map((b) => removeFromSitemap(layout, b.url, `it answers ${b.status === null ? "with no response" : `HTTP ${b.status}`}. If the page should exist, restore it instead.`)),
520
+ evidence: { count: broken.length, examples: cap(broken) },
521
+ }));
385
522
  }
386
523
  if (redirecting.length) {
387
524
  out.push(finding(m("site.sitemap-redirecting-url"), `${redirecting.length} sitemap URL(s) redirect.`, {
525
+ fix: redirecting.map((r) => repoint(r.url, r.finalUrl)),
388
526
  evidence: { count: redirecting.length, examples: cap(redirecting) },
389
527
  }));
390
528
  }
391
529
  return out;
392
530
  };
393
531
  /** Indexable pages missing from the sitemap, and sitemap URLs nothing links to. */
394
- const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
532
+ /**
533
+ * Links an orphan from the home page, or takes it out of the sitemap when it does not load: a link
534
+ * to a broken page would only add a broken link.
535
+ */
536
+ const orphanEdit = (scope, home, url) => {
537
+ const check = scope.checksByKey.get(scope.key(url));
538
+ if (check && (check.error || (check.status ?? 0) >= 400)) {
539
+ return removeFromSitemap(sitemapLayout(scope.ctx.sitemaps.problems), url, `it answers ${check.status === null ? "with no response" : `HTTP ${check.status}`}, and no page links to it.`);
540
+ }
541
+ return addLinkEdit(home, url, `Words that say what ${url} is about. Put the link on the page closest in topic, or in the navigation; if the page is obsolete, remove it from the sitemap instead.`);
542
+ };
543
+ const sitemapCoverageFindings = (scope) => {
544
+ const { ctx, key, pagesByKey, okPages, isIndexable } = scope;
395
545
  const out = [];
546
+ const start = pagesByKey.get(key(ctx.startUrl));
547
+ const home = start ? locationOf(start) : ctx.startUrl;
396
548
  const sitemapKeys = new Set(ctx.sitemaps.urls.map(key));
397
549
  const notInSitemap = okPages
398
550
  .filter((p) => isIndexable(p) &&
@@ -401,6 +553,7 @@ const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
401
553
  .map((p) => p.url);
402
554
  if (notInSitemap.length) {
403
555
  out.push(finding(m("site.not-in-sitemap"), `${notInSitemap.length} crawled indexable page(s) are missing from the sitemap.`, {
556
+ fix: addToSitemapFix(sitemapLayout(ctx.sitemaps.problems), notInSitemap),
404
557
  evidence: {
405
558
  count: notInSitemap.length,
406
559
  examples: cap(notInSitemap),
@@ -415,7 +568,12 @@ const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
415
568
  linked.add(key(ctx.startUrl));
416
569
  const orphans = ctx.sitemaps.urls.filter((u) => !linked.has(key(u)));
417
570
  if (orphans.length) {
418
- out.push(finding(m("site.orphan-pages"), `${orphans.length} sitemap URL(s) are not linked from any crawled page.`, { evidence: { count: orphans.length, examples: cap(orphans) } }));
571
+ out.push(finding(m("site.orphan-pages"), `${orphans.length} sitemap URL(s) are not linked from any crawled page.`, {
572
+ fix: orphans
573
+ .slice(0, MAX_EDITS)
574
+ .map((u) => orphanEdit(scope, home, u)),
575
+ evidence: { count: orphans.length, examples: cap(orphans) },
576
+ }));
419
577
  }
420
578
  }
421
579
  return out;
@@ -424,7 +582,10 @@ const sitemapFindings = (scope) => {
424
582
  const { ctx } = scope;
425
583
  const out = [];
426
584
  if (ctx.sitemaps.errors.length) {
427
- out.push(finding(m("site.sitemap-error"), `${ctx.sitemaps.errors.length} sitemap problem(s).`, { evidence: { errors: cap(ctx.sitemaps.errors) } }));
585
+ out.push(finding(m("site.sitemap-error"), `${ctx.sitemaps.errors.length} sitemap problem(s).`, {
586
+ fix: sitemapErrorFix(ctx.sitemaps.errors.slice(0, 10), new URL(ctx.startUrl).origin),
587
+ evidence: { errors: cap(ctx.sitemaps.errors) },
588
+ }));
428
589
  }
429
590
  // A sitemap too large to read was still found.
430
591
  if (ctx.sitemaps.sources.length === 0 &&
@@ -432,6 +593,11 @@ const sitemapFindings = (scope) => {
432
593
  ctx.sitemaps.problems.oversized.length === 0) {
433
594
  out.push(finding(m("site.sitemap-missing"), "No sitemap was found.", {
434
595
  severity: ctx.source === "directory" ? "info" : "warning",
596
+ fix: sitemapMissingFix(sitemapUrlOf(ctx), scope.okPages
597
+ .filter((p) => scope.isIndexable(p) &&
598
+ (!p.page.canonical ||
599
+ scope.key(p.page.canonical) === scope.key(p.url)))
600
+ .map((p) => p.url), ctx.robots?.found ? ctx.robots.url : null),
435
601
  }));
436
602
  }
437
603
  if (ctx.sitemaps.urlCount > 0) {
@@ -439,9 +605,54 @@ const sitemapFindings = (scope) => {
439
605
  }
440
606
  return out;
441
607
  };
608
+ /**
609
+ * Points every link to `targetKey` on the pages in `sources` at `hrefFor(link, page)`, up to
610
+ * `MAX_EDITS` edits.
611
+ */
612
+ const relinkSources = (scope, sources, targetKey, hrefFor, write) => {
613
+ const edits = [];
614
+ for (const source of sources) {
615
+ const record = scope.pagesByKey.get(scope.key(source));
616
+ for (const link of record ? scope.linksTo(record, targetKey) : []) {
617
+ if (record && edits.length < MAX_EDITS) {
618
+ edits.push(relinkEdit(locationOf(record), link, hrefFor(link.href, record.url), write));
619
+ }
620
+ }
621
+ }
622
+ return edits;
623
+ };
624
+ /** A live page whose URL differs from `url` only in a trailing slash or letter case. */
625
+ const liveSpelling = (scope, url) => {
626
+ const match = scope.okPages.find((p) => spellingKey(p.url) === spellingKey(url) &&
627
+ scope.key(p.url) !== scope.key(url));
628
+ return match?.url ?? null;
629
+ };
630
+ /** Edits for broken internal links: the live spelling of the URL when there is one, else a slot. */
631
+ const brokenLinkEdits = (scope, broken) => broken
632
+ .flatMap((b) => {
633
+ const live = liveSpelling(scope, b.url);
634
+ return relinkSources(scope, scope.ctx.linkSources.get(b.key) ?? [], scope.key(b.url), (href, pageUrl) => live
635
+ ? hrefLike(href, live, pageUrl)
636
+ : slot("URL of the live page this link meant"), live
637
+ ? undefined
638
+ : `The URL of the page the link was meant for; remove the link if that page is gone. ${b.url} does not answer.`);
639
+ })
640
+ .slice(0, MAX_EDITS);
641
+ /** Points each hop of a redirect chain straight at where the chain ends. */
642
+ const chainEdits = (check) => check.redirectChain.slice(0, -1).map((hop, i) => ({
643
+ in: "http",
644
+ action: "replace",
645
+ url: hop.url,
646
+ current: `${statusLine(hop.status)}\nLocation: ${check.redirectChain[i + 1]?.url ?? check.finalUrl}`,
647
+ code: redirectTo(check.finalUrl, hop.status === 307 || hop.status === 308),
648
+ }));
442
649
  /** Internal link targets that are broken, redirect once, or redirect in a chain. */
443
- const linkHealthFindings = (ctx) => {
650
+ const linkHealthFindings = (scope) => {
651
+ const { ctx } = scope;
444
652
  const out = [];
653
+ const brokenKeys = [];
654
+ const redirectedKeys = [];
655
+ const chainChecks = [];
445
656
  /* link health */
446
657
  const broken = [];
447
658
  const redirected = [];
@@ -454,6 +665,7 @@ const linkHealthFindings = (ctx) => {
454
665
  // Chains of two or more hops are handled first, so a first hop here is the only one.
455
666
  const [firstHop] = check.redirectChain;
456
667
  if (check.error || (check.status ?? 0) >= 400) {
668
+ brokenKeys.push({ url: check.url, key: k });
457
669
  broken.push({
458
670
  url: check.url,
459
671
  status: check.status,
@@ -462,6 +674,7 @@ const linkHealthFindings = (ctx) => {
462
674
  });
463
675
  }
464
676
  else if (check.redirectChain.length >= 2) {
677
+ chainChecks.push(check);
465
678
  chains.push({
466
679
  url: check.url,
467
680
  hops: check.redirectChain.length,
@@ -469,6 +682,7 @@ const linkHealthFindings = (ctx) => {
469
682
  });
470
683
  }
471
684
  else if (firstHop) {
685
+ redirectedKeys.push({ key: k, finalUrl: check.finalUrl });
472
686
  redirected.push({
473
687
  url: check.url,
474
688
  finalUrl: check.finalUrl,
@@ -478,21 +692,32 @@ const linkHealthFindings = (ctx) => {
478
692
  }
479
693
  }
480
694
  if (broken.length) {
481
- out.push(finding(m("site.broken-internal-links"), `${broken.length} internal link target(s) are broken.`, { evidence: { count: broken.length, examples: cap(broken, 20) } }));
695
+ out.push(finding(m("site.broken-internal-links"), `${broken.length} internal link target(s) are broken.`, {
696
+ fix: brokenLinkEdits(scope, brokenKeys),
697
+ evidence: { count: broken.length, examples: cap(broken, 20) },
698
+ }));
482
699
  }
483
700
  if (redirected.length) {
484
701
  out.push(finding(m("site.redirected-internal-links"), `${redirected.length} internal link target(s) redirect.`, {
702
+ fix: redirectedKeys
703
+ .flatMap((r) => relinkSources(scope, ctx.linkSources.get(r.key) ?? [], scope.key(r.key), (href, pageUrl) => hrefLike(href, r.finalUrl, pageUrl)))
704
+ .slice(0, MAX_EDITS),
485
705
  evidence: { count: redirected.length, examples: cap(redirected, 20) },
486
706
  }));
487
707
  }
488
708
  if (chains.length) {
489
- out.push(finding(m("site.redirect-chains"), `${chains.length} internal URL(s) have redirect chains.`, { evidence: { count: chains.length, examples: cap(chains) } }));
709
+ out.push(finding(m("site.redirect-chains"), `${chains.length} internal URL(s) have redirect chains.`, {
710
+ fix: chainChecks.flatMap(chainEdits).slice(0, MAX_EDITS),
711
+ evidence: { count: chains.length, examples: cap(chains) },
712
+ }));
490
713
  }
491
714
  return out;
492
715
  };
493
716
  /** Broken external links (when they were probed) and HTML files nothing links to. */
494
- const unreachedFindings = (ctx) => {
717
+ const unreachedFindings = (scope) => {
718
+ const { ctx } = scope;
495
719
  const out = [];
720
+ const start = scope.pagesByKey.get(scope.key(ctx.startUrl));
496
721
  if (ctx.externalChecks) {
497
722
  const dead = [];
498
723
  for (const [u, check] of ctx.externalChecks) {
@@ -506,11 +731,19 @@ const unreachedFindings = (ctx) => {
506
731
  }
507
732
  }
508
733
  if (dead.length) {
509
- out.push(finding(m("site.broken-external-links"), `${dead.length} external link target(s) are broken.`, { evidence: { count: dead.length, examples: cap(dead, 20) } }));
734
+ out.push(finding(m("site.broken-external-links"), `${dead.length} external link target(s) are broken.`, {
735
+ fix: dead
736
+ .flatMap((d) => relinkSources(scope, ctx.externalSources?.get(d.url) ?? [], scope.key(d.url), () => slot("current URL of the linked page"), `Where ${d.url} lives now: search for the page, or link a web archive copy; remove the link if the page is gone.`))
737
+ .slice(0, MAX_EDITS),
738
+ evidence: { count: dead.length, examples: cap(dead, 20) },
739
+ }));
510
740
  }
511
741
  }
512
742
  if (ctx.unlinkedPages?.length) {
513
743
  out.push(finding(m("site.unlinked-pages"), `${ctx.unlinkedPages.length} HTML file(s) are not linked from any other page.`, {
744
+ fix: ctx.unlinkedPages
745
+ .slice(0, MAX_EDITS)
746
+ .map((file) => addLinkEdit(start ? locationOf(start) : ctx.startUrl, ctx.pages.find((p) => (p.file ?? p.url) === file)?.url ?? file, `Words that say what ${file} is about. Link it from the page closest in topic or the navigation; if the file is obsolete, delete it from the build instead.`)),
514
747
  evidence: {
515
748
  count: ctx.unlinkedPages.length,
516
749
  examples: cap(ctx.unlinkedPages),
@@ -524,19 +757,36 @@ const duplicateFindings = ({ key, okPages, isIndexable, }) => {
524
757
  // Pages that name another URL as canonical are consolidated already; only the canonical ones compete.
525
758
  const indexable = okPages.filter((p) => isIndexable(p) &&
526
759
  (!p.page.canonical || key(p.page.canonical) === key(p.url)));
527
- pushDuplicates(out, m("site.duplicate-titles"), indexable, (p) => p.page.title, "title");
528
- pushDuplicates(out, m("site.duplicate-descriptions"), indexable, (p) => p.page.metaDescription, "meta description");
529
- pushDuplicates(out, m("site.duplicate-h1"), indexable, (p) => p.page.h1s[0] ?? null, "H1");
760
+ pushDuplicates(out, m("site.duplicate-titles"), indexable, (p) => p.page.title, "title", retitleEdit);
761
+ pushDuplicates(out, m("site.duplicate-descriptions"), indexable, (p) => p.page.metaDescription, "meta description", redescribeEdit);
762
+ pushDuplicates(out, m("site.duplicate-h1"), indexable, (p) => p.page.h1s[0] ?? null, "H1", reheadEdit);
530
763
  pushDuplicateContent(out, indexable);
531
764
  return out;
532
765
  };
533
766
  /** Pages deep in the site, and how the server answers a made-up URL and the other host names. */
534
- const architectureFindings = ({ ctx, okPages }) => {
767
+ /** The same URL on `origin`, for a redirect to the site's main host or to https. */
768
+ const onOrigin = (url, origin) => {
769
+ try {
770
+ const u = new URL(url);
771
+ // Appended, not resolved: a "//x" path resolved against the origin would name a host.
772
+ return new URL(`${origin}${u.pathname}${u.search}`).href;
773
+ }
774
+ catch {
775
+ return origin;
776
+ }
777
+ };
778
+ const architectureFindings = ({ ctx, key, pagesByKey, okPages, }) => {
535
779
  const out = [];
780
+ const start = pagesByKey.get(key(ctx.startUrl));
781
+ const home = start ? locationOf(start) : ctx.startUrl;
782
+ const { origin } = new URL(ctx.startUrl);
536
783
  /* architecture */
537
784
  const deep = okPages.filter((p) => p.depth > ctx.thresholds.deepPageDepth);
538
785
  if (deep.length) {
539
786
  out.push(finding(m("site.deep-pages"), `${deep.length} page(s) are more than ${ctx.thresholds.deepPageDepth} clicks from the start URL.`, {
787
+ fix: deep
788
+ .slice(0, MAX_EDITS)
789
+ .map((p) => addLinkEdit(home, p.url, `Words that say what ${p.url} is about. Or link it from a category or hub page one or two clicks from the home page.`)),
540
790
  evidence: {
541
791
  count: deep.length,
542
792
  examples: cap(deep.map((p) => ({ url: p.url, depth: p.depth }))),
@@ -547,17 +797,53 @@ const architectureFindings = ({ ctx, okPages }) => {
547
797
  if (ctx.probes.soft404 &&
548
798
  ctx.probes.soft404.status !== null &&
549
799
  ctx.probes.soft404.status < 400) {
550
- out.push(finding(m("site.soft-404"), `${ctx.probes.soft404.url} returned HTTP ${ctx.probes.soft404.status}.`, { evidence: ctx.probes.soft404 }));
800
+ out.push(finding(m("site.soft-404"), `${ctx.probes.soft404.url} returned HTTP ${ctx.probes.soft404.status}.`, {
801
+ fix: [
802
+ {
803
+ in: "http",
804
+ action: "replace",
805
+ url: ctx.probes.soft404.url,
806
+ current: statusLine(ctx.probes.soft404.status),
807
+ code: statusLine(404),
808
+ write: "For every URL no route matches: the catch-all route or the single-page app's fallback should answer 404 (it may keep showing its not-found page).",
809
+ },
810
+ ],
811
+ evidence: ctx.probes.soft404,
812
+ }));
551
813
  }
552
814
  if (ctx.probes.httpRedirect &&
553
815
  ctx.probes.httpRedirect.status !== null &&
554
816
  !ctx.probes.httpRedirect.redirectsToHttps) {
555
- out.push(finding(m("site.http-not-redirected"), `${ctx.probes.httpRedirect.url} returned HTTP ${ctx.probes.httpRedirect.status} without redirecting to https.`, { evidence: ctx.probes.httpRedirect }));
817
+ out.push(finding(m("site.http-not-redirected"), `${ctx.probes.httpRedirect.url} returned HTTP ${ctx.probes.httpRedirect.status} without redirecting to https.`, {
818
+ fix: [
819
+ {
820
+ in: "http",
821
+ action: "replace",
822
+ url: ctx.probes.httpRedirect.url,
823
+ current: statusLine(ctx.probes.httpRedirect.status),
824
+ code: redirectTo(ctx.probes.httpRedirect.url.replace(/^http:/iu, "https:")),
825
+ write: "For every http:// URL, to the same path on https://, at the server or CDN.",
826
+ },
827
+ ],
828
+ evidence: ctx.probes.httpRedirect,
829
+ }));
556
830
  }
557
831
  if (ctx.probes.hostVariant &&
558
832
  ctx.probes.hostVariant.status !== null &&
559
833
  !ctx.probes.hostVariant.redirectsToMain) {
560
- out.push(finding(m("site.host-variant-not-redirected"), `${ctx.probes.hostVariant.url} returned HTTP ${ctx.probes.hostVariant.status} without redirecting to the main host.`, { evidence: ctx.probes.hostVariant }));
834
+ out.push(finding(m("site.host-variant-not-redirected"), `${ctx.probes.hostVariant.url} returned HTTP ${ctx.probes.hostVariant.status} without redirecting to the main host.`, {
835
+ fix: [
836
+ {
837
+ in: "http",
838
+ action: "replace",
839
+ url: ctx.probes.hostVariant.url,
840
+ current: statusLine(ctx.probes.hostVariant.status),
841
+ code: redirectTo(onOrigin(ctx.probes.hostVariant.url, origin)),
842
+ write: `For every URL on this host, to the same path on ${origin}, at the DNS host, server or CDN.`,
843
+ },
844
+ ],
845
+ evidence: ctx.probes.hostVariant,
846
+ }));
561
847
  }
562
848
  return out;
563
849
  };
@@ -565,23 +851,34 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
565
851
  const out = [];
566
852
  /* hreflang return links */
567
853
  const missingReturn = [];
854
+ const returnLinks = [];
568
855
  for (const p of ctx.pages) {
856
+ // A page whose canonical names another URL (a parameter or tracking variant) is folded into
857
+ // that URL, so its hreflang links do not count and nothing should link back to it.
858
+ if (p.page.canonical && key(p.page.canonical) !== key(p.url)) {
859
+ continue;
860
+ }
861
+ // en and x-default often name the same page: one missing return link, not two.
862
+ const seen = new Set();
569
863
  for (const alt of p.page.hreflang) {
570
864
  if (!alt.url) {
571
865
  continue;
572
866
  }
573
867
  const target = pagesByKey.get(key(alt.url));
574
- if (!target || target === p) {
868
+ if (!target || target === p || seen.has(target)) {
575
869
  continue;
576
870
  }
871
+ seen.add(target);
577
872
  const linksBack = target.page.hreflang.some((h) => h.url && key(h.url) === key(p.url));
578
873
  if (!linksBack) {
579
874
  missingReturn.push({ page: p.url, alternate: alt.url });
875
+ returnLinks.push(returnLinkEdit(target, p, key));
580
876
  }
581
877
  }
582
878
  }
583
879
  if (missingReturn.length) {
584
880
  out.push(finding(m("site.hreflang-no-return-link"), `${missingReturn.length} hreflang alternate(s) do not link back.`, {
881
+ fix: returnLinks.slice(0, MAX_EDITS),
585
882
  evidence: {
586
883
  count: missingReturn.length,
587
884
  examples: cap(missingReturn),
@@ -590,6 +887,8 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
590
887
  }
591
888
  if (ctx.crawl.hitPageLimit) {
592
889
  out.push(finding(m("site.crawl-limit-reached"), `Crawl stopped after ${ctx.crawl.pagesCrawled} pages (limit ${ctx.crawl.maxPages}); ${ctx.crawl.pagesDiscovered} URLs were discovered.`, {
890
+ // About the audit, not the site: the change is a larger --max-pages, not an edit.
891
+ fix: [],
593
892
  evidence: {
594
893
  pagesCrawled: ctx.crawl.pagesCrawled,
595
894
  pagesDiscovered: ctx.crawl.pagesDiscovered,
@@ -599,8 +898,39 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
599
898
  }
600
899
  return out;
601
900
  };
901
+ /** The canonical a page should name instead of a broken, redirecting, noindex or chained one. */
902
+ const canonicalRepair = (scope, p, problem) => {
903
+ if (problem.kind === "redirects" || problem.kind === "canonical") {
904
+ return canonicalEdit(p, problem.to ?? p.url);
905
+ }
906
+ if (problem.kind === "noindex") {
907
+ return canonicalEdit(p, p.url, "The page's own URL. If the noindex page is the one meant to rank, remove its noindex and keep this canonical instead.");
908
+ }
909
+ const live = p.page.canonical ? liveSpelling(scope, p.page.canonical) : null;
910
+ return canonicalEdit(p, live ?? p.url, live
911
+ ? undefined
912
+ : "The page's own URL. If another page is the preferred one, use its live URL.");
913
+ };
602
914
  /** What is wrong with a URL that metadata (a canonical, an hreflang link) points to, or null. */
603
- const targetProblem = ({ key, pagesByKey, checksByKey, isIndexable }, url) => {
915
+ /**
916
+ * Where a chain of canonicals ends: the first page that names itself (or nothing), or the last URL
917
+ * before the chain loops or leaves the crawled pages.
918
+ */
919
+ const finalCanonical = ({ key, pagesByKey }, url) => {
920
+ let current = url;
921
+ const seen = new Set([key(url)]);
922
+ for (let hops = 0; hops < 10; hops += 1) {
923
+ const next = pagesByKey.get(key(current))?.page.canonical;
924
+ if (!next || seen.has(key(next))) {
925
+ return current;
926
+ }
927
+ seen.add(key(next));
928
+ current = next;
929
+ }
930
+ return current;
931
+ };
932
+ const targetProblem = (scope, url) => {
933
+ const { key, pagesByKey, checksByKey, isIndexable } = scope;
604
934
  const check = checksByKey.get(key(url));
605
935
  if (check && (check.error || (check.status ?? 0) >= 400)) {
606
936
  return {
@@ -609,7 +939,11 @@ const targetProblem = ({ key, pagesByKey, checksByKey, isIndexable }, url) => {
609
939
  };
610
940
  }
611
941
  if (check && check.redirectChain.length > 0) {
612
- return { kind: "redirects", detail: `redirects to ${check.finalUrl}` };
942
+ return {
943
+ kind: "redirects",
944
+ detail: `redirects to ${check.finalUrl}`,
945
+ to: check.finalUrl,
946
+ };
613
947
  }
614
948
  const target = pagesByKey.get(key(url));
615
949
  if (!target) {
@@ -620,7 +954,11 @@ const targetProblem = ({ key, pagesByKey, checksByKey, isIndexable }, url) => {
620
954
  }
621
955
  const { canonical } = target.page;
622
956
  return canonical && key(canonical) !== key(target.url)
623
- ? { kind: "canonical", detail: `names ${canonical} as canonical` }
957
+ ? {
958
+ kind: "canonical",
959
+ detail: `names ${canonical} as canonical`,
960
+ to: finalCanonical(scope, canonical),
961
+ }
624
962
  : null;
625
963
  };
626
964
  /** Canonicals that point at broken, redirecting, noindex or re-canonicalized URLs. */
@@ -632,6 +970,12 @@ const canonicalGraphFindings = (scope) => {
632
970
  noindex: [],
633
971
  canonical: [],
634
972
  };
973
+ const repairs = {
974
+ broken: [],
975
+ redirects: [],
976
+ noindex: [],
977
+ canonical: [],
978
+ };
635
979
  for (const p of okPages) {
636
980
  const { canonical } = p.page;
637
981
  // A noindex page's canonical carries no weight; the noindex wins.
@@ -645,6 +989,7 @@ const canonicalGraphFindings = (scope) => {
645
989
  canonical,
646
990
  problem: problem.detail,
647
991
  });
992
+ repairs[problem.kind].push(canonicalRepair(scope, p, problem));
648
993
  }
649
994
  }
650
995
  const rules = [
@@ -674,6 +1019,7 @@ const canonicalGraphFindings = (scope) => {
674
1019
  return list.length > 0
675
1020
  ? [
676
1021
  finding(m(id), `${list.length} page(s) ${label}.`, {
1022
+ fix: repairs[kind].slice(0, MAX_EDITS),
677
1023
  evidence: { count: list.length, examples: cap(list) },
678
1024
  }),
679
1025
  ]
@@ -691,7 +1037,7 @@ const isUtilityPage = (url) => {
691
1037
  }
692
1038
  };
693
1039
  /** Noindex pages that other pages of the site link to, most linked first. */
694
- const noindexLinkFindings = ({ ctx, key, okPages, isIndexable, }) => {
1040
+ const noindexLinkFindings = ({ ctx, key, pagesByKey, okPages, isIndexable, }) => {
695
1041
  const linkedFrom = new Map();
696
1042
  for (const [target, sources] of ctx.linkSources) {
697
1043
  const k = key(target);
@@ -713,15 +1059,42 @@ const noindexLinkFindings = ({ ctx, key, okPages, isIndexable, }) => {
713
1059
  .toSorted((a, b) => b.linkedFrom - a.linkedFrom);
714
1060
  return linked.length > 0
715
1061
  ? [
716
- finding(m("site.links-to-noindex"), `${linked.length} noindex page(s) are linked from other pages; the most linked: ${linked[0]?.url} (${linked[0]?.linkedFrom} pages).`, { evidence: { count: linked.length, pages: cap(linked) } }),
1062
+ finding(m("site.links-to-noindex"), `${linked.length} noindex page(s) are linked from other pages; the most linked: ${linked[0]?.url} (${linked[0]?.linkedFrom} pages).`, {
1063
+ fix: linked
1064
+ .flatMap((l) => {
1065
+ const record = pagesByKey.get(key(l.url));
1066
+ return record
1067
+ ? removeNoindexEdits(record, `Only if ${l.url} belongs in search results; leave the noindex on a page kept out on purpose.`)
1068
+ : [];
1069
+ })
1070
+ .slice(0, MAX_EDITS),
1071
+ evidence: { count: linked.length, pages: cap(linked) },
1072
+ }),
717
1073
  ]
718
1074
  : [];
719
1075
  };
1076
+ /**
1077
+ * Points an hreflang link at the canonical, live URL its alternate leads to, or removes it when the
1078
+ * alternate is broken or noindex: an annotation to such a page is ignored anyway.
1079
+ */
1080
+ const hreflangRepair = (p, alt, problem) => {
1081
+ const at = `link[rel="alternate"][hreflang="${alt.lang}"]`;
1082
+ if ((problem.kind === "redirects" || problem.kind === "canonical") &&
1083
+ problem.to) {
1084
+ return onUrl(setAttributeEdit(alt.tag, at, "href", escapeAttribute(problem.to)), locationOf(p));
1085
+ }
1086
+ return removeEdit("html", alt.tag, {
1087
+ url: locationOf(p),
1088
+ at,
1089
+ write: `${alt.url} ${problem.detail}. If that language version should exist, list its live, indexable URL here instead.`,
1090
+ });
1091
+ };
720
1092
  /** hreflang alternates that are broken, redirect, are noindex or canonicalize elsewhere. */
721
1093
  const hreflangTargetFindings = (scope) => {
722
1094
  const { key, okPages } = scope;
723
1095
  const invalid = [];
724
1096
  const seen = new Set();
1097
+ const fix = [];
725
1098
  for (const p of okPages) {
726
1099
  for (const alt of p.page.hreflang) {
727
1100
  if (!alt.url || key(alt.url) === key(p.url)) {
@@ -739,26 +1112,19 @@ const hreflangTargetFindings = (scope) => {
739
1112
  alternate: alt.url,
740
1113
  problem: problem.detail,
741
1114
  });
1115
+ fix.push(hreflangRepair(p, alt, problem));
742
1116
  }
743
1117
  }
744
1118
  }
745
1119
  return invalid.length > 0
746
1120
  ? [
747
- finding(m("site.hreflang-target-invalid"), `${invalid.length} hreflang alternate(s) point to URLs that are broken, redirect, are noindex or are not canonical.`, { evidence: { count: invalid.length, examples: cap(invalid) } }),
1121
+ finding(m("site.hreflang-target-invalid"), `${invalid.length} hreflang alternate(s) point to URLs that are broken, redirect, are noindex or are not canonical.`, {
1122
+ fix: fix.slice(0, MAX_EDITS),
1123
+ evidence: { count: invalid.length, examples: cap(invalid) },
1124
+ }),
748
1125
  ]
749
1126
  : [];
750
1127
  };
751
- /** The URL with its path lower-cased and trailing slashes dropped: one key for every spelling. */
752
- const spellingKey = (url) => {
753
- try {
754
- const u = new URL(url);
755
- const pathname = trimTrailingSlashes(u.pathname.toLowerCase()) || "/";
756
- return `${u.origin}${pathname}${u.search}`;
757
- }
758
- catch {
759
- return url;
760
- }
761
- };
762
1128
  const pathnameOf = (url) => {
763
1129
  try {
764
1130
  return new URL(url).pathname;
@@ -811,8 +1177,19 @@ const urlVariantFindings = (scope) => {
811
1177
  return [];
812
1178
  }
813
1179
  const kinds = [...new Set(duplicates.map((d) => d.kind))];
1180
+ const fix = duplicates.slice(0, MAX_EDITS).map((d) => ({
1181
+ in: "http",
1182
+ action: "replace",
1183
+ url: d.variant,
1184
+ current: statusLine(d.status ?? 200),
1185
+ code: redirectTo(d.url),
1186
+ write: d.kind === "trailing-slash"
1187
+ ? "For every URL of the site, to the one spelling with or without the trailing slash: the trailingSlash setting in Next.js, Nuxt or Astro, or a redirect rule at the server or CDN."
1188
+ : "For every URL spelled in another case, to the spelling the site links to: a redirect rule at the server or CDN.",
1189
+ }));
814
1190
  return [
815
1191
  finding(m("site.url-variant-duplicate"), `${duplicates.length} page(s) also answer at a ${kinds.join(" and a ")} variant of their URL.`, {
1192
+ fix,
816
1193
  evidence: {
817
1194
  count: duplicates.length,
818
1195
  examples: cap(duplicates.map(({ kind, url, variant, canonical }) => ({
@@ -835,7 +1212,16 @@ const withoutQuery = (url) => {
835
1212
  }
836
1213
  };
837
1214
  /** Pages that differ only in their query string, show the same text and do not consolidate. */
838
- const parameterDuplicateFindings = ({ key, okPages, isIndexable, }) => {
1215
+ /** Canonicals from each parameter version to the URL without the parameter, or the first URL. */
1216
+ const parameterCanonicals = ({ key, pagesByKey }, group) => {
1217
+ const target = group.urls.find((u) => !u.includes("?")) ?? group.urls[0] ?? group.path;
1218
+ return group.urls.flatMap((u) => {
1219
+ const record = key(u) === key(target) ? undefined : pagesByKey.get(key(u));
1220
+ return record ? [canonicalEdit(record, target)] : [];
1221
+ });
1222
+ };
1223
+ const parameterDuplicateFindings = (scope) => {
1224
+ const { key, okPages, isIndexable } = scope;
839
1225
  const byPath = new Map();
840
1226
  for (const p of okPages) {
841
1227
  const { canonical } = p.page;
@@ -873,12 +1259,41 @@ const parameterDuplicateFindings = ({ key, okPages, isIndexable, }) => {
873
1259
  }
874
1260
  return groups.length > 0
875
1261
  ? [
876
- finding(m("site.parameter-duplicates"), `${groups.length} path(s) are crawled under several query strings with the same text.`, { evidence: { count: groups.length, groups: cap(groups) } }),
1262
+ finding(m("site.parameter-duplicates"), `${groups.length} path(s) are crawled under several query strings with the same text.`, {
1263
+ fix: groups
1264
+ .flatMap((g) => parameterCanonicals(scope, g))
1265
+ .slice(0, MAX_EDITS),
1266
+ evidence: { count: groups.length, groups: cap(groups) },
1267
+ }),
877
1268
  ]
878
1269
  : [];
879
1270
  };
1271
+ /** Points each <img> that shows a broken image at a slot for the image's working URL. */
1272
+ const brokenImageEdits = ({ key, pagesByKey }, image, pages) => {
1273
+ const write = `The URL the image file is deployed at; or upload the missing file to ${image.url}.`;
1274
+ const edits = [...pages].flatMap((u) => {
1275
+ const record = pagesByKey.get(key(u));
1276
+ return record
1277
+ ? imagesOf(record, image.url).map((img) => onUrl(setAttributeEdit(img.tag, "img", "src", slot("working URL of the image"), write), locationOf(record)))
1278
+ : [];
1279
+ });
1280
+ // An image from srcset or a <picture> source has no <img src> to quote: upload the file instead.
1281
+ return edits.length > 0
1282
+ ? edits
1283
+ : [
1284
+ {
1285
+ in: "asset",
1286
+ action: "add",
1287
+ url: image.url,
1288
+ at: "new file",
1289
+ code: slot("the image file"),
1290
+ write: `Upload the image the pages show here, or change the pages that point to ${image.url}.`,
1291
+ },
1292
+ ];
1293
+ };
880
1294
  /** Images that answer 4xx/5xx or do not load, with the pages that show them. */
881
- const brokenImageFindings = (ctx) => {
1295
+ const brokenImageFindings = (scope) => {
1296
+ const { ctx } = scope;
882
1297
  const broken = [];
883
1298
  for (const [url, check] of ctx.imageChecks ?? []) {
884
1299
  if (check.error || (check.status ?? 0) >= 400) {
@@ -895,7 +1310,12 @@ const brokenImageFindings = (ctx) => {
895
1310
  }
896
1311
  return broken.length > 0
897
1312
  ? [
898
- finding(m("site.broken-images"), `${broken.length} image URL(s) are broken.`, { evidence: { count: broken.length, examples: cap(broken, 20) } }),
1313
+ finding(m("site.broken-images"), `${broken.length} image URL(s) are broken.`, {
1314
+ fix: broken
1315
+ .flatMap((b) => brokenImageEdits(scope, b, ctx.imageSources?.get(b.url) ?? []))
1316
+ .slice(0, MAX_EDITS),
1317
+ evidence: { count: broken.length, examples: cap(broken, 20) },
1318
+ }),
899
1319
  ]
900
1320
  : [];
901
1321
  };
@@ -919,20 +1339,21 @@ export const runSiteRules = (ctx) => {
919
1339
  okPages: ctx.pages.filter((p) => (p.response?.status ?? 200) < 400),
920
1340
  isIndexable: (p) => !robotsDirectives(p.page, p.response).noindex &&
921
1341
  (p.response?.status ?? 200) < 400,
1342
+ linksTo: (p, targetKey) => linksTo(p, targetKey, key),
922
1343
  };
923
1344
  return [
924
- ...robotsFindings(ctx),
1345
+ ...robotsFindings(scope),
925
1346
  ...robotsContentFindings(ctx),
926
1347
  ...blockedResourceFindings(scope),
927
1348
  ...sitemapFindings(scope),
928
1349
  ...sitemapProtocolFindings(ctx),
929
- ...linkHealthFindings(ctx),
930
- ...unreachedFindings(ctx),
1350
+ ...linkHealthFindings(scope),
1351
+ ...unreachedFindings(scope),
931
1352
  ...duplicateFindings(scope),
932
1353
  ...architectureFindings(scope),
933
1354
  ...urlVariantFindings(scope),
934
1355
  ...parameterDuplicateFindings(scope),
935
- ...brokenImageFindings(ctx),
1356
+ ...brokenImageFindings(scope),
936
1357
  ...canonicalGraphFindings(scope),
937
1358
  ...noindexLinkFindings(scope),
938
1359
  ...hreflangTargetFindings(scope),