@rivi-software/agentic-seo-check 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/README.md +29 -5
  2. package/dist/cli.js +20 -0
  3. package/dist/cli.js.map +1 -1
  4. package/dist/core/aeo/audit.d.ts +8 -0
  5. package/dist/core/aeo/audit.js +101 -17
  6. package/dist/core/aeo/audit.js.map +1 -1
  7. package/dist/core/aeo/bots.d.ts +48 -0
  8. package/dist/core/aeo/bots.js +151 -5
  9. package/dist/core/aeo/bots.js.map +1 -1
  10. package/dist/core/aeo/fixes.d.ts +38 -0
  11. package/dist/core/aeo/fixes.js +111 -0
  12. package/dist/core/aeo/fixes.js.map +1 -0
  13. package/dist/core/aeo/site-rules.d.ts +5 -0
  14. package/dist/core/aeo/site-rules.js +293 -10
  15. package/dist/core/aeo/site-rules.js.map +1 -1
  16. package/dist/core/audit.d.ts +8 -0
  17. package/dist/core/audit.js +144 -25
  18. package/dist/core/audit.js.map +1 -1
  19. package/dist/core/crawl.js +43 -4
  20. package/dist/core/crawl.js.map +1 -1
  21. package/dist/core/html/answer.d.ts +5 -2
  22. package/dist/core/html/answer.js +25 -7
  23. package/dist/core/html/answer.js.map +1 -1
  24. package/dist/core/html/dom.d.ts +13 -1
  25. package/dist/core/html/dom.js +30 -2
  26. package/dist/core/html/dom.js.map +1 -1
  27. package/dist/core/html/extract.js +122 -30
  28. package/dist/core/html/extract.js.map +1 -1
  29. package/dist/core/html/head.js +114 -19
  30. package/dist/core/html/head.js.map +1 -1
  31. package/dist/core/html/resources.js +15 -7
  32. package/dist/core/html/resources.js.map +1 -1
  33. package/dist/core/html/structured-data.js +21 -5
  34. package/dist/core/html/structured-data.js.map +1 -1
  35. package/dist/core/lighthouse.d.ts +2 -5
  36. package/dist/core/lighthouse.js +290 -5
  37. package/dist/core/lighthouse.js.map +1 -1
  38. package/dist/core/performance-fixes.d.ts +25 -0
  39. package/dist/core/performance-fixes.js +446 -0
  40. package/dist/core/performance-fixes.js.map +1 -0
  41. package/dist/core/performance-rules.js +56 -8
  42. package/dist/core/performance-rules.js.map +1 -1
  43. package/dist/core/recheck.d.ts +68 -0
  44. package/dist/core/recheck.js +247 -0
  45. package/dist/core/recheck.js.map +1 -0
  46. package/dist/core/render-fixes.d.ts +27 -0
  47. package/dist/core/render-fixes.js +164 -0
  48. package/dist/core/render-fixes.js.map +1 -0
  49. package/dist/core/render-rules.js +60 -21
  50. package/dist/core/render-rules.js.map +1 -1
  51. package/dist/core/report/markdown.js +68 -1
  52. package/dist/core/report/markdown.js.map +1 -1
  53. package/dist/core/robots.d.ts +5 -0
  54. package/dist/core/robots.js +8 -0
  55. package/dist/core/robots.js.map +1 -1
  56. package/dist/core/rules/aeo.js +339 -25
  57. package/dist/core/rules/aeo.js.map +1 -1
  58. package/dist/core/rules/canonical.js +107 -11
  59. package/dist/core/rules/canonical.js.map +1 -1
  60. package/dist/core/rules/content.js +65 -7
  61. package/dist/core/rules/content.js.map +1 -1
  62. package/dist/core/rules/document.js +73 -8
  63. package/dist/core/rules/document.js.map +1 -1
  64. package/dist/core/rules/fix.d.ts +55 -0
  65. package/dist/core/rules/fix.js +159 -0
  66. package/dist/core/rules/fix.js.map +1 -0
  67. package/dist/core/rules/head.js +81 -4
  68. package/dist/core/rules/head.js.map +1 -1
  69. package/dist/core/rules/headings.js +92 -9
  70. package/dist/core/rules/headings.js.map +1 -1
  71. package/dist/core/rules/hreflang.js +79 -9
  72. package/dist/core/rules/hreflang.js.map +1 -1
  73. package/dist/core/rules/images.js +49 -0
  74. package/dist/core/rules/images.js.map +1 -1
  75. package/dist/core/rules/index.d.ts +11 -0
  76. package/dist/core/rules/index.js +21 -1
  77. package/dist/core/rules/index.js.map +1 -1
  78. package/dist/core/rules/indexing.js +108 -1
  79. package/dist/core/rules/indexing.js.map +1 -1
  80. package/dist/core/rules/keyword.js +92 -2
  81. package/dist/core/rules/keyword.js.map +1 -1
  82. package/dist/core/rules/links.js +42 -0
  83. package/dist/core/rules/links.js.map +1 -1
  84. package/dist/core/rules/loading.js +54 -1
  85. package/dist/core/rules/loading.js.map +1 -1
  86. package/dist/core/rules/meta.js +40 -6
  87. package/dist/core/rules/meta.js.map +1 -1
  88. package/dist/core/rules/page-fixes.d.ts +39 -0
  89. package/dist/core/rules/page-fixes.js +107 -0
  90. package/dist/core/rules/page-fixes.js.map +1 -0
  91. package/dist/core/rules/quoted-tags.d.ts +23 -0
  92. package/dist/core/rules/quoted-tags.js +79 -0
  93. package/dist/core/rules/quoted-tags.js.map +1 -0
  94. package/dist/core/rules/response.js +71 -7
  95. package/dist/core/rules/response.js.map +1 -1
  96. package/dist/core/rules/rule.d.ts +17 -4
  97. package/dist/core/rules/rule.js +11 -2
  98. package/dist/core/rules/rule.js.map +1 -1
  99. package/dist/core/rules/social.js +101 -10
  100. package/dist/core/rules/social.js.map +1 -1
  101. package/dist/core/rules/structured-data-fixes.d.ts +33 -0
  102. package/dist/core/rules/structured-data-fixes.js +469 -0
  103. package/dist/core/rules/structured-data-fixes.js.map +1 -0
  104. package/dist/core/rules/structured-data-values.js +242 -31
  105. package/dist/core/rules/structured-data-values.js.map +1 -1
  106. package/dist/core/rules/structured-data.js +171 -4
  107. package/dist/core/rules/structured-data.js.map +1 -1
  108. package/dist/core/rules/title.js +39 -5
  109. package/dist/core/rules/title.js.map +1 -1
  110. package/dist/core/rules/urls.js +166 -0
  111. package/dist/core/rules/urls.js.map +1 -1
  112. package/dist/core/site-fixes.d.ts +111 -0
  113. package/dist/core/site-fixes.js +650 -0
  114. package/dist/core/site-fixes.js.map +1 -0
  115. package/dist/core/site-rules.js +481 -54
  116. package/dist/core/site-rules.js.map +1 -1
  117. package/dist/core/sitemap.d.ts +16 -0
  118. package/dist/core/sitemap.js +34 -0
  119. package/dist/core/sitemap.js.map +1 -1
  120. package/dist/core/types.d.ts +223 -3
  121. package/dist/core/types.js.map +1 -1
  122. package/dist/index.d.ts +2 -0
  123. package/dist/index.js +1 -0
  124. package/dist/index.js.map +1 -1
  125. package/dist/mcp.js +16 -2
  126. package/dist/mcp.js.map +1 -1
  127. package/package.json +1 -1
  128. package/skills/seo-audit/SKILL.md +35 -5
@@ -1,8 +1,11 @@
1
1
  import { normalizeForComparison, shingles, jaccard } from "./html/text.js";
2
2
  import { agentMatcher } from "./robots.js";
3
+ import { escapeAttribute, removeEdit, setAttributeEdit, slot, } from "./rules/fix.js";
3
4
  import { robotsDirectives } from "./rules/indexing.js";
4
5
  import { finding, GOOGLE_DOCS, cap } from "./rules/rule.js";
6
+ import { addLinkEdit, addToSitemapFix, blockedResourcesFix, canonicalEdit, imagesOf, blockingRule, lastmodFix, locationOf, linkSelector, linksTo, nestedIndexFix, hrefLike, onUrl, otherHostFix, redirectTo, redescribeEdit, removeNoindexEdits, returnLinkEdit, reheadEdit, relinkEdit, linesOf, retitleEdit, statusLine, removeFromSitemap, replaceInSitemap, robotsBlocksAllFix, robotsErrorFix, robotsMissingFix, robotsTooLargeFix, robotsUnsupportedFix, MAX_EDITS, sitemapErrorFix, sitemapMissingFix, splitSitemapFix, } from "./site-fixes.js";
5
7
  import { MIN_COMPARABLE_WORDS, SAME_TEXT } from "./site-probes.js";
8
+ import { sitemapLayout } from "./sitemap.js";
6
9
  import { normalizeUrl, normalizePath, trimTrailingSlashes } from "./url.js";
7
10
  const meta = (id, severity, description, recommendation, docs) => ({
8
11
  id,
@@ -85,8 +88,24 @@ const pathKey = (url) => {
85
88
  return url;
86
89
  }
87
90
  };
88
- const pushDuplicates = (out, rule, pages, pick, label) => {
91
+ /** The URL with its path lower-cased and trailing slashes dropped: one key for every spelling. */
92
+ const spellingKey = (url) => {
93
+ try {
94
+ const u = new URL(url);
95
+ const pathname = trimTrailingSlashes(u.pathname.toLowerCase()) || "/";
96
+ return `${u.origin}${pathname}${u.search}`;
97
+ }
98
+ catch {
99
+ return url;
100
+ }
101
+ };
102
+ const pushDuplicates = (out, rule, pages, pick, label, fixFor) => {
89
103
  const groups = new Map();
104
+ // Every page but the first of a group gets a value of its own; the first keeps what it has.
105
+ const others = [];
106
+ // Pages are grouped by a lowercased key, but the evidence quotes the value as the first page
107
+ // wrote it, so a report or pull request shows the text a reviewer would find on the site.
108
+ const written = new Map();
90
109
  for (const p of pages) {
91
110
  const value = pick(p);
92
111
  if (!value) {
@@ -94,6 +113,12 @@ const pushDuplicates = (out, rule, pages, pick, label) => {
94
113
  }
95
114
  const k = normalizeForComparison(value);
96
115
  pushTo(groups, k, p.url);
116
+ if (written.has(k)) {
117
+ others.push(p);
118
+ }
119
+ else {
120
+ written.set(k, value);
121
+ }
97
122
  }
98
123
  const dupes = [...groups.entries()].filter(([, urls]) => urls.length > 1);
99
124
  if (!dupes.length) {
@@ -101,15 +126,44 @@ const pushDuplicates = (out, rule, pages, pick, label) => {
101
126
  }
102
127
  const pagesAffected = dupes.reduce((n, [, urls]) => n + urls.length, 0);
103
128
  out.push(finding(rule, `${dupes.length} ${label}(s) are shared by ${pagesAffected} pages.`, {
129
+ fix: others.flatMap((p) => fixFor(p) ?? []).slice(0, MAX_EDITS),
104
130
  evidence: {
105
- groups: cap(dupes.map(([value, urls]) => ({
106
- value: value.slice(0, 120),
131
+ groups: cap(dupes.map(([k, urls]) => ({
132
+ value: (written.get(k) ?? k).slice(0, 120),
107
133
  pages: cap(urls, 5),
108
134
  count: urls.length,
109
135
  }))),
110
136
  },
111
137
  }));
112
138
  };
139
+ /**
140
+ * The canonical each duplicate should name. The first page of a group is the shallowest the crawl
141
+ * reached: the others name it as canonical. Near-duplicates come in pairs, so a page can be in two
142
+ * groups (a~b and b~c, or a~c and b~c): each page gets one canonical, to a page that keeps its
143
+ * own, so the fix makes no chain and gives no page two.
144
+ */
145
+ const duplicateCanonicalEdits = (groups, byUrl) => {
146
+ const canonicalOf = new Map();
147
+ const roots = new Set();
148
+ const fix = [];
149
+ for (const { pages: [first, ...rest], } of groups) {
150
+ const root = first ? (canonicalOf.get(first) ?? first) : undefined;
151
+ for (const u of rest) {
152
+ const record = byUrl.get(u);
153
+ if (!root ||
154
+ !record ||
155
+ u === root ||
156
+ canonicalOf.has(u) ||
157
+ roots.has(u)) {
158
+ continue;
159
+ }
160
+ canonicalOf.set(u, root);
161
+ roots.add(root);
162
+ fix.push(canonicalEdit(record, root, `If ${u} is meant to be a page of its own, make its text distinct instead.`));
163
+ }
164
+ }
165
+ return fix;
166
+ };
113
167
  const pushDuplicateContent = (out, pages) => {
114
168
  const candidates = pages.filter((p) => p.page.wordCount >= 50);
115
169
  const groups = [];
@@ -150,7 +204,9 @@ const pushDuplicateContent = (out, pages) => {
150
204
  }
151
205
  }
152
206
  if (groups.length) {
153
- out.push(finding(m("site.duplicate-content"), `${groups.length} group(s) of pages have (near-)identical body text.`, { evidence: { groups: cap(groups) } }));
207
+ const byUrl = new Map(candidates.map((p) => [p.url, p]));
208
+ const fix = duplicateCanonicalEdits(groups, byUrl);
209
+ out.push(finding(m("site.duplicate-content"), `${groups.length} group(s) of pages have (near-)identical body text.`, { fix: fix.slice(0, MAX_EDITS), evidence: { groups: cap(groups) } }));
154
210
  }
155
211
  };
156
212
  /** Google reads the first 500 KiB of robots.txt and ignores the rest. */
@@ -158,7 +214,7 @@ const ROBOTS_MAX_BYTES = 512_000;
158
214
  const unsupportedRobotsLines = (text) => {
159
215
  const retired = [];
160
216
  const crawlDelay = [];
161
- for (const [i, raw] of text.split(/\r?\n/u).slice(0, 50_000).entries()) {
217
+ for (const [i, raw] of linesOf(text).slice(0, 50_000).entries()) {
162
218
  const line = raw.replace(/#.*$/u, "").trim();
163
219
  const field = /^(?<field>[a-z-]+)\s*:/iu
164
220
  .exec(line)
@@ -176,10 +232,14 @@ const unsupportedRobotsLines = (text) => {
176
232
  const robotsContentFindings = (ctx) => {
177
233
  const out = [];
178
234
  const bytes = ctx.robots?.bytes ?? 0;
235
+ const text = ctx.robotsText ?? ctx.robots?.text;
236
+ const robotsUrl = ctx.robots?.url ?? new URL("/robots.txt", ctx.startUrl).href;
179
237
  if (bytes > ROBOTS_MAX_BYTES) {
180
- out.push(finding(m("site.robots-txt-too-large"), `robots.txt is ${Math.round(bytes / 1024)} KiB; Google reads only the first 500 KiB.`, { evidence: { url: ctx.robots?.url, bytes } }));
238
+ out.push(finding(m("site.robots-txt-too-large"), `robots.txt is ${Math.round(bytes / 1024)} KiB; Google reads only the first 500 KiB.`, {
239
+ fix: robotsTooLargeFix(robotsUrl, text ?? "", bytes),
240
+ evidence: { url: ctx.robots?.url, bytes },
241
+ }));
181
242
  }
182
- const text = ctx.robotsText ?? ctx.robots?.text;
183
243
  if (!text) {
184
244
  return out;
185
245
  }
@@ -189,6 +249,7 @@ const robotsContentFindings = (ctx) => {
189
249
  ? `robots.txt has ${retired.length} noindex or nofollow line(s), which Google stopped reading in 2019.`
190
250
  : `robots.txt sets crawl-delay, which Google ignores.`, {
191
251
  severity: retired.length > 0 ? "warning" : "info",
252
+ fix: robotsUnsupportedFix(robotsUrl, text),
192
253
  evidence: {
193
254
  url: ctx.robots?.url,
194
255
  retired: cap(retired),
@@ -246,28 +307,44 @@ const blockedResourceFindings = ({ ctx, okPages, isIndexable, }) => {
246
307
  .toSorted((a, b) => b.pages - a.pages);
247
308
  return list.length > 0
248
309
  ? [
249
- finding(m("site.robots-blocks-resources"), `robots.txt keeps Googlebot from ${list.length} script(s) or stylesheet(s) the pages load.`, { evidence: { count: list.length, examples: cap(list) } }),
310
+ finding(m("site.robots-blocks-resources"), `robots.txt keeps Googlebot from ${list.length} script(s) or stylesheet(s) the pages load.`, {
311
+ fix: blockedResourcesFix(robots.url, text, list.map((r) => r.url)),
312
+ evidence: { count: list.length, examples: cap(list) },
313
+ }),
250
314
  ]
251
315
  : [];
252
316
  };
253
317
  /** Sitemaps over the protocol's limits, nested indexes, bad lastmod values and other hosts. */
318
+ /** Parts of at most 45,000 URLs or 45 MB, and at least two. */
319
+ const partsFor = (urls, bytes, perPart) => Math.max(2, Math.ceil((urls ?? 0) / 45_000), Math.ceil((bytes ?? 0) / perPart));
254
320
  const sitemapProtocolFindings = (ctx) => {
255
321
  const { problems } = ctx.sitemaps;
322
+ const layout = sitemapLayout(problems);
323
+ const today = new Date().toISOString().slice(0, 10);
256
324
  const out = [];
257
325
  const over = problems.oversized.filter((o) => o.overLimit);
258
326
  const unread = problems.oversized.filter((o) => !o.overLimit);
259
327
  if (over.length > 0) {
260
- out.push(finding(m("site.sitemap-too-large"), `${over.length} sitemap(s) exceed 50,000 URLs or 50 MB.`, { evidence: { sitemaps: cap(over) } }));
328
+ out.push(finding(m("site.sitemap-too-large"), `${over.length} sitemap(s) exceed 50,000 URLs or 50 MB.`, {
329
+ fix: over.map((o) => splitSitemapFix(o.sitemap, partsFor(o.urls, o.bytes, 45_000_000), "Serve this sitemap index where the large sitemap was, and move its URLs into the files it lists, each under 50,000 URLs and 50 MB.")),
330
+ key: "over-limit",
331
+ evidence: { sitemaps: cap(over) },
332
+ }));
261
333
  }
262
334
  if (unread.length > 0) {
263
335
  out.push(finding(m("site.sitemap-too-large"), `${unread.length} sitemap(s) are larger than the 15 MB this audit reads, so their URLs were not checked.`, {
264
336
  severity: "info",
265
337
  recommendation: "Check that each stays under 50 MB and 50,000 URLs; smaller sitemaps (10,000 URLs or fewer) are also quicker for search engines to fetch again.",
338
+ fix: unread.map((o) => splitSitemapFix(o.sitemap, partsFor(o.urls, o.bytes, 10_000_000), "Serve this sitemap index where the large sitemap was, and move its URLs into the files it lists, 10,000 URLs or fewer each, so search engines (and this audit) can read each one quickly.")),
339
+ key: "unread",
266
340
  evidence: { sitemaps: cap(unread) },
267
341
  }));
268
342
  }
269
343
  if (problems.nestedIndexes.length > 0) {
270
- out.push(finding(m("site.sitemap-index-nested"), `${problems.nestedIndexes.length} sitemap index(es) are listed by another sitemap index.`, { evidence: { indexes: cap(problems.nestedIndexes) } }));
344
+ out.push(finding(m("site.sitemap-index-nested"), `${problems.nestedIndexes.length} sitemap index(es) are listed by another sitemap index.`, {
345
+ fix: problems.nestedIndexes.map((n) => nestedIndexFix(layout, n)),
346
+ evidence: { indexes: cap(problems.nestedIndexes) },
347
+ }));
271
348
  }
272
349
  const examples = [
273
350
  [
@@ -286,22 +363,69 @@ const sitemapProtocolFindings = (ctx) => {
286
363
  "are on another host than the site",
287
364
  ],
288
365
  ];
366
+ const fixes = {
367
+ "site.sitemap-lastmod-invalid": (list) => lastmodFix(layout, list, "invalid", today),
368
+ "site.sitemap-lastmod-future": (list) => lastmodFix(layout, list, "future", today),
369
+ "site.sitemap-other-host": (list) => otherHostFix(layout, list, new URL(ctx.startUrl).origin),
370
+ };
289
371
  for (const [id, list, label] of examples) {
290
372
  if (list.count > 0) {
291
373
  out.push(finding(m(id), `${list.count} sitemap URL(s) ${label}.`, {
374
+ fix: fixes[id](list.examples),
292
375
  evidence: { count: list.count, examples: list.examples },
293
376
  }));
294
377
  }
295
378
  }
296
379
  return out;
297
380
  };
298
- const robotsFindings = (ctx) => {
381
+ /** The sitemap URL a robots.txt should name: the one found, else /sitemap.xml. */
382
+ const sitemapUrlOf = (ctx) => ctx.sitemaps.sources[0] ?? new URL("/sitemap.xml", ctx.startUrl).href;
383
+ /**
384
+ * Keeps blocked pages out of the sitemap and stops crawlers following links to them; unblocking
385
+ * them could expose pages kept private on purpose, so the write names that rule instead.
386
+ */
387
+ const blockedPageEdits = (scope, blocked) => {
388
+ const { ctx, key, pagesByKey } = scope;
389
+ const k = key(blocked);
390
+ const rule = ctx.robots && ctx.robotsText
391
+ ? blockingRule(ctx.robots.url, ctx.robotsText, blocked)
392
+ : null;
393
+ const unblock = `If the page belongs in search results, remove ${rule ? `"${rule}"` : "the rule that blocks it"} from robots.txt instead.`;
394
+ const listed = ctx.sitemaps.urls.find((u) => key(u) === k);
395
+ if (listed) {
396
+ return [
397
+ removeFromSitemap(sitemapLayout(ctx.sitemaps.problems), listed, `robots.txt blocks it, so search engines cannot crawl it. ${unblock}`),
398
+ ];
399
+ }
400
+ const sources = [...ctx.linkSources].flatMap(([target, from]) => key(target) === k ? [...from] : []);
401
+ const edits = sources.flatMap((source) => {
402
+ const record = pagesByKey.get(key(source));
403
+ return record
404
+ ? linksTo(record, k, key).map((link) => onUrl(setAttributeEdit(link.tag, linkSelector(link), "rel", link.rel ? `${link.rel} nofollow` : "nofollow", `So crawlers do not follow it to a page robots.txt blocks. ${unblock}`), locationOf(record)))
405
+ : [];
406
+ });
407
+ if (edits.length > 0 || !rule || !ctx.robots) {
408
+ return edits;
409
+ }
410
+ return [
411
+ {
412
+ in: "robots.txt",
413
+ action: "remove",
414
+ url: ctx.robots.url,
415
+ current: rule,
416
+ write: `Only if ${blocked} belongs in search results; if it is kept out on purpose, leave robots.txt as it is.`,
417
+ },
418
+ ];
419
+ };
420
+ const robotsFindings = (scope) => {
421
+ const { ctx } = scope;
299
422
  const out = [];
300
423
  /* robots.txt */
301
424
  if (ctx.robots) {
302
425
  if (ctx.robots.error ||
303
426
  (ctx.robots.status !== null && ctx.robots.status >= 500)) {
304
427
  out.push(finding(m("site.robots-txt-error"), `robots.txt at ${ctx.robots.url} could not be read (${ctx.robots.error ?? `HTTP ${ctx.robots.status}`}).`, {
428
+ fix: robotsErrorFix(ctx.robots.url, ctx.robots.status, ctx.robots.error),
305
429
  evidence: {
306
430
  url: ctx.robots.url,
307
431
  status: ctx.robots.status,
@@ -312,11 +436,13 @@ const robotsFindings = (ctx) => {
312
436
  else if (!ctx.robots.found) {
313
437
  out.push(finding(m("site.robots-txt-missing"), `No robots.txt at ${ctx.robots.url}.`, {
314
438
  severity: ctx.source === "directory" ? "info" : "warning",
439
+ fix: robotsMissingFix(ctx.robots.url, sitemapUrlOf(ctx)),
315
440
  evidence: { url: ctx.robots.url, status: ctx.robots.status },
316
441
  }));
317
442
  }
318
443
  else if (ctx.robotsAllows && !ctx.robotsAllows(ctx.startUrl)) {
319
444
  out.push(finding(m("site.robots-txt-blocks-all"), "robots.txt blocks the start URL for the crawler user agent.", {
445
+ fix: robotsBlocksAllFix(ctx.robots.url, ctx.robotsText ?? ctx.robots.text ?? "", ctx.startUrl),
320
446
  evidence: {
321
447
  url: ctx.robots.url,
322
448
  robotsTxt: ctx.robots.text?.slice(0, 500),
@@ -326,6 +452,10 @@ const robotsFindings = (ctx) => {
326
452
  }
327
453
  if (ctx.blockedByRobots.length) {
328
454
  out.push(finding(m("site.robots-blocked-pages"), `${ctx.blockedByRobots.length} linked/sitemap URL(s) are disallowed by robots.txt.`, {
455
+ fix: ctx.blockedByRobots
456
+ .slice(0, MAX_EDITS)
457
+ .flatMap((u) => blockedPageEdits(scope, u))
458
+ .slice(0, MAX_EDITS),
329
459
  evidence: {
330
460
  count: ctx.blockedByRobots.length,
331
461
  examples: cap(ctx.blockedByRobots),
@@ -337,6 +467,12 @@ const robotsFindings = (ctx) => {
337
467
  /** Noindex, non-canonical, broken and redirecting URLs listed in the sitemap. */
338
468
  const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
339
469
  const out = [];
470
+ const layout = sitemapLayout(ctx.sitemaps.problems);
471
+ const listed = new Set(ctx.sitemaps.urls.map(key));
472
+ /** Points an entry at `to`, or removes it when the sitemap lists `to` already. */
473
+ const repoint = (loc, to) => listed.has(key(to))
474
+ ? removeFromSitemap(layout, loc, `the sitemap lists ${to} already.`)
475
+ : replaceInSitemap(layout, loc, to);
340
476
  const noindex = [];
341
477
  const nonCanonical = [];
342
478
  const broken = [];
@@ -364,10 +500,14 @@ const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
364
500
  }
365
501
  }
366
502
  if (noindex.length) {
367
- out.push(finding(m("site.sitemap-noindex-page"), `${noindex.length} sitemap URL(s) are noindex.`, { evidence: { count: noindex.length, examples: cap(noindex) } }));
503
+ out.push(finding(m("site.sitemap-noindex-page"), `${noindex.length} sitemap URL(s) are noindex.`, {
504
+ fix: noindex.map((u) => removeFromSitemap(layout, u, "the page is noindex, which may be on purpose. If it belongs in search results, remove its noindex instead.")),
505
+ evidence: { count: noindex.length, examples: cap(noindex) },
506
+ }));
368
507
  }
369
508
  if (nonCanonical.length) {
370
509
  out.push(finding(m("site.sitemap-non-canonical"), `${nonCanonical.length} sitemap URL(s) canonicalise to another URL.`, {
510
+ fix: nonCanonical.map((n) => repoint(n.url, n.canonical)),
371
511
  evidence: {
372
512
  count: nonCanonical.length,
373
513
  examples: cap(nonCanonical),
@@ -375,18 +515,36 @@ const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
375
515
  }));
376
516
  }
377
517
  if (broken.length) {
378
- out.push(finding(m("site.sitemap-broken-url"), `${broken.length} sitemap URL(s) return errors.`, { evidence: { count: broken.length, examples: cap(broken) } }));
518
+ out.push(finding(m("site.sitemap-broken-url"), `${broken.length} sitemap URL(s) return errors.`, {
519
+ fix: broken.map((b) => removeFromSitemap(layout, b.url, `it answers ${b.status === null ? "with no response" : `HTTP ${b.status}`}. If the page should exist, restore it instead.`)),
520
+ evidence: { count: broken.length, examples: cap(broken) },
521
+ }));
379
522
  }
380
523
  if (redirecting.length) {
381
524
  out.push(finding(m("site.sitemap-redirecting-url"), `${redirecting.length} sitemap URL(s) redirect.`, {
525
+ fix: redirecting.map((r) => repoint(r.url, r.finalUrl)),
382
526
  evidence: { count: redirecting.length, examples: cap(redirecting) },
383
527
  }));
384
528
  }
385
529
  return out;
386
530
  };
387
531
  /** Indexable pages missing from the sitemap, and sitemap URLs nothing links to. */
388
- const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
532
+ /**
533
+ * Links an orphan from the home page, or takes it out of the sitemap when it does not load: a link
534
+ * to a broken page would only add a broken link.
535
+ */
536
+ const orphanEdit = (scope, home, url) => {
537
+ const check = scope.checksByKey.get(scope.key(url));
538
+ if (check && (check.error || (check.status ?? 0) >= 400)) {
539
+ return removeFromSitemap(sitemapLayout(scope.ctx.sitemaps.problems), url, `it answers ${check.status === null ? "with no response" : `HTTP ${check.status}`}, and no page links to it.`);
540
+ }
541
+ return addLinkEdit(home, url, `Words that say what ${url} is about. Put the link on the page closest in topic, or in the navigation; if the page is obsolete, remove it from the sitemap instead.`);
542
+ };
543
+ const sitemapCoverageFindings = (scope) => {
544
+ const { ctx, key, pagesByKey, okPages, isIndexable } = scope;
389
545
  const out = [];
546
+ const start = pagesByKey.get(key(ctx.startUrl));
547
+ const home = start ? locationOf(start) : ctx.startUrl;
390
548
  const sitemapKeys = new Set(ctx.sitemaps.urls.map(key));
391
549
  const notInSitemap = okPages
392
550
  .filter((p) => isIndexable(p) &&
@@ -395,6 +553,7 @@ const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
395
553
  .map((p) => p.url);
396
554
  if (notInSitemap.length) {
397
555
  out.push(finding(m("site.not-in-sitemap"), `${notInSitemap.length} crawled indexable page(s) are missing from the sitemap.`, {
556
+ fix: addToSitemapFix(sitemapLayout(ctx.sitemaps.problems), notInSitemap),
398
557
  evidence: {
399
558
  count: notInSitemap.length,
400
559
  examples: cap(notInSitemap),
@@ -409,7 +568,12 @@ const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
409
568
  linked.add(key(ctx.startUrl));
410
569
  const orphans = ctx.sitemaps.urls.filter((u) => !linked.has(key(u)));
411
570
  if (orphans.length) {
412
- out.push(finding(m("site.orphan-pages"), `${orphans.length} sitemap URL(s) are not linked from any crawled page.`, { evidence: { count: orphans.length, examples: cap(orphans) } }));
571
+ out.push(finding(m("site.orphan-pages"), `${orphans.length} sitemap URL(s) are not linked from any crawled page.`, {
572
+ fix: orphans
573
+ .slice(0, MAX_EDITS)
574
+ .map((u) => orphanEdit(scope, home, u)),
575
+ evidence: { count: orphans.length, examples: cap(orphans) },
576
+ }));
413
577
  }
414
578
  }
415
579
  return out;
@@ -418,7 +582,10 @@ const sitemapFindings = (scope) => {
418
582
  const { ctx } = scope;
419
583
  const out = [];
420
584
  if (ctx.sitemaps.errors.length) {
421
- out.push(finding(m("site.sitemap-error"), `${ctx.sitemaps.errors.length} sitemap problem(s).`, { evidence: { errors: cap(ctx.sitemaps.errors) } }));
585
+ out.push(finding(m("site.sitemap-error"), `${ctx.sitemaps.errors.length} sitemap problem(s).`, {
586
+ fix: sitemapErrorFix(ctx.sitemaps.errors.slice(0, 10), new URL(ctx.startUrl).origin),
587
+ evidence: { errors: cap(ctx.sitemaps.errors) },
588
+ }));
422
589
  }
423
590
  // A sitemap too large to read was still found.
424
591
  if (ctx.sitemaps.sources.length === 0 &&
@@ -426,6 +593,11 @@ const sitemapFindings = (scope) => {
426
593
  ctx.sitemaps.problems.oversized.length === 0) {
427
594
  out.push(finding(m("site.sitemap-missing"), "No sitemap was found.", {
428
595
  severity: ctx.source === "directory" ? "info" : "warning",
596
+ fix: sitemapMissingFix(sitemapUrlOf(ctx), scope.okPages
597
+ .filter((p) => scope.isIndexable(p) &&
598
+ (!p.page.canonical ||
599
+ scope.key(p.page.canonical) === scope.key(p.url)))
600
+ .map((p) => p.url), ctx.robots?.found ? ctx.robots.url : null),
429
601
  }));
430
602
  }
431
603
  if (ctx.sitemaps.urlCount > 0) {
@@ -433,9 +605,54 @@ const sitemapFindings = (scope) => {
433
605
  }
434
606
  return out;
435
607
  };
608
+ /**
609
+ * Points every link to `targetKey` on the pages in `sources` at `hrefFor(link, page)`, up to
610
+ * `MAX_EDITS` edits.
611
+ */
612
+ const relinkSources = (scope, sources, targetKey, hrefFor, write) => {
613
+ const edits = [];
614
+ for (const source of sources) {
615
+ const record = scope.pagesByKey.get(scope.key(source));
616
+ for (const link of record ? scope.linksTo(record, targetKey) : []) {
617
+ if (record && edits.length < MAX_EDITS) {
618
+ edits.push(relinkEdit(locationOf(record), link, hrefFor(link.href, record.url), write));
619
+ }
620
+ }
621
+ }
622
+ return edits;
623
+ };
624
+ /** A live page whose URL differs from `url` only in a trailing slash or letter case. */
625
+ const liveSpelling = (scope, url) => {
626
+ const match = scope.okPages.find((p) => spellingKey(p.url) === spellingKey(url) &&
627
+ scope.key(p.url) !== scope.key(url));
628
+ return match?.url ?? null;
629
+ };
630
+ /** Edits for broken internal links: the live spelling of the URL when there is one, else a slot. */
631
+ const brokenLinkEdits = (scope, broken) => broken
632
+ .flatMap((b) => {
633
+ const live = liveSpelling(scope, b.url);
634
+ return relinkSources(scope, scope.ctx.linkSources.get(b.key) ?? [], scope.key(b.url), (href, pageUrl) => live
635
+ ? hrefLike(href, live, pageUrl)
636
+ : slot("URL of the live page this link meant"), live
637
+ ? undefined
638
+ : `The URL of the page the link was meant for; remove the link if that page is gone. ${b.url} does not answer.`);
639
+ })
640
+ .slice(0, MAX_EDITS);
641
+ /** Points each hop of a redirect chain straight at where the chain ends. */
642
+ const chainEdits = (check) => check.redirectChain.slice(0, -1).map((hop, i) => ({
643
+ in: "http",
644
+ action: "replace",
645
+ url: hop.url,
646
+ current: `${statusLine(hop.status)}\nLocation: ${check.redirectChain[i + 1]?.url ?? check.finalUrl}`,
647
+ code: redirectTo(check.finalUrl, hop.status === 307 || hop.status === 308),
648
+ }));
436
649
  /** Internal link targets that are broken, redirect once, or redirect in a chain. */
437
- const linkHealthFindings = (ctx) => {
650
+ const linkHealthFindings = (scope) => {
651
+ const { ctx } = scope;
438
652
  const out = [];
653
+ const brokenKeys = [];
654
+ const redirectedKeys = [];
655
+ const chainChecks = [];
439
656
  /* link health */
440
657
  const broken = [];
441
658
  const redirected = [];
@@ -448,6 +665,7 @@ const linkHealthFindings = (ctx) => {
448
665
  // Chains of two or more hops are handled first, so a first hop here is the only one.
449
666
  const [firstHop] = check.redirectChain;
450
667
  if (check.error || (check.status ?? 0) >= 400) {
668
+ brokenKeys.push({ url: check.url, key: k });
451
669
  broken.push({
452
670
  url: check.url,
453
671
  status: check.status,
@@ -456,6 +674,7 @@ const linkHealthFindings = (ctx) => {
456
674
  });
457
675
  }
458
676
  else if (check.redirectChain.length >= 2) {
677
+ chainChecks.push(check);
459
678
  chains.push({
460
679
  url: check.url,
461
680
  hops: check.redirectChain.length,
@@ -463,6 +682,7 @@ const linkHealthFindings = (ctx) => {
463
682
  });
464
683
  }
465
684
  else if (firstHop) {
685
+ redirectedKeys.push({ key: k, finalUrl: check.finalUrl });
466
686
  redirected.push({
467
687
  url: check.url,
468
688
  finalUrl: check.finalUrl,
@@ -472,21 +692,32 @@ const linkHealthFindings = (ctx) => {
472
692
  }
473
693
  }
474
694
  if (broken.length) {
475
- out.push(finding(m("site.broken-internal-links"), `${broken.length} internal link target(s) are broken.`, { evidence: { count: broken.length, examples: cap(broken, 20) } }));
695
+ out.push(finding(m("site.broken-internal-links"), `${broken.length} internal link target(s) are broken.`, {
696
+ fix: brokenLinkEdits(scope, brokenKeys),
697
+ evidence: { count: broken.length, examples: cap(broken, 20) },
698
+ }));
476
699
  }
477
700
  if (redirected.length) {
478
701
  out.push(finding(m("site.redirected-internal-links"), `${redirected.length} internal link target(s) redirect.`, {
702
+ fix: redirectedKeys
703
+ .flatMap((r) => relinkSources(scope, ctx.linkSources.get(r.key) ?? [], scope.key(r.key), (href, pageUrl) => hrefLike(href, r.finalUrl, pageUrl)))
704
+ .slice(0, MAX_EDITS),
479
705
  evidence: { count: redirected.length, examples: cap(redirected, 20) },
480
706
  }));
481
707
  }
482
708
  if (chains.length) {
483
- out.push(finding(m("site.redirect-chains"), `${chains.length} internal URL(s) have redirect chains.`, { evidence: { count: chains.length, examples: cap(chains) } }));
709
+ out.push(finding(m("site.redirect-chains"), `${chains.length} internal URL(s) have redirect chains.`, {
710
+ fix: chainChecks.flatMap(chainEdits).slice(0, MAX_EDITS),
711
+ evidence: { count: chains.length, examples: cap(chains) },
712
+ }));
484
713
  }
485
714
  return out;
486
715
  };
487
716
  /** Broken external links (when they were probed) and HTML files nothing links to. */
488
- const unreachedFindings = (ctx) => {
717
+ const unreachedFindings = (scope) => {
718
+ const { ctx } = scope;
489
719
  const out = [];
720
+ const start = scope.pagesByKey.get(scope.key(ctx.startUrl));
490
721
  if (ctx.externalChecks) {
491
722
  const dead = [];
492
723
  for (const [u, check] of ctx.externalChecks) {
@@ -500,11 +731,19 @@ const unreachedFindings = (ctx) => {
500
731
  }
501
732
  }
502
733
  if (dead.length) {
503
- out.push(finding(m("site.broken-external-links"), `${dead.length} external link target(s) are broken.`, { evidence: { count: dead.length, examples: cap(dead, 20) } }));
734
+ out.push(finding(m("site.broken-external-links"), `${dead.length} external link target(s) are broken.`, {
735
+ fix: dead
736
+ .flatMap((d) => relinkSources(scope, ctx.externalSources?.get(d.url) ?? [], scope.key(d.url), () => slot("current URL of the linked page"), `Where ${d.url} lives now: search for the page, or link a web archive copy; remove the link if the page is gone.`))
737
+ .slice(0, MAX_EDITS),
738
+ evidence: { count: dead.length, examples: cap(dead, 20) },
739
+ }));
504
740
  }
505
741
  }
506
742
  if (ctx.unlinkedPages?.length) {
507
743
  out.push(finding(m("site.unlinked-pages"), `${ctx.unlinkedPages.length} HTML file(s) are not linked from any other page.`, {
744
+ fix: ctx.unlinkedPages
745
+ .slice(0, MAX_EDITS)
746
+ .map((file) => addLinkEdit(start ? locationOf(start) : ctx.startUrl, ctx.pages.find((p) => (p.file ?? p.url) === file)?.url ?? file, `Words that say what ${file} is about. Link it from the page closest in topic or the navigation; if the file is obsolete, delete it from the build instead.`)),
508
747
  evidence: {
509
748
  count: ctx.unlinkedPages.length,
510
749
  examples: cap(ctx.unlinkedPages),
@@ -518,19 +757,36 @@ const duplicateFindings = ({ key, okPages, isIndexable, }) => {
518
757
  // Pages that name another URL as canonical are consolidated already; only the canonical ones compete.
519
758
  const indexable = okPages.filter((p) => isIndexable(p) &&
520
759
  (!p.page.canonical || key(p.page.canonical) === key(p.url)));
521
- pushDuplicates(out, m("site.duplicate-titles"), indexable, (p) => p.page.title, "title");
522
- pushDuplicates(out, m("site.duplicate-descriptions"), indexable, (p) => p.page.metaDescription, "meta description");
523
- pushDuplicates(out, m("site.duplicate-h1"), indexable, (p) => p.page.h1s[0] ?? null, "H1");
760
+ pushDuplicates(out, m("site.duplicate-titles"), indexable, (p) => p.page.title, "title", retitleEdit);
761
+ pushDuplicates(out, m("site.duplicate-descriptions"), indexable, (p) => p.page.metaDescription, "meta description", redescribeEdit);
762
+ pushDuplicates(out, m("site.duplicate-h1"), indexable, (p) => p.page.h1s[0] ?? null, "H1", reheadEdit);
524
763
  pushDuplicateContent(out, indexable);
525
764
  return out;
526
765
  };
527
766
  /** Pages deep in the site, and how the server answers a made-up URL and the other host names. */
528
- const architectureFindings = ({ ctx, okPages }) => {
767
+ /** The same URL on `origin`, for a redirect to the site's main host or to https. */
768
+ const onOrigin = (url, origin) => {
769
+ try {
770
+ const u = new URL(url);
771
+ // Appended, not resolved: a "//x" path resolved against the origin would name a host.
772
+ return new URL(`${origin}${u.pathname}${u.search}`).href;
773
+ }
774
+ catch {
775
+ return origin;
776
+ }
777
+ };
778
+ const architectureFindings = ({ ctx, key, pagesByKey, okPages, }) => {
529
779
  const out = [];
780
+ const start = pagesByKey.get(key(ctx.startUrl));
781
+ const home = start ? locationOf(start) : ctx.startUrl;
782
+ const { origin } = new URL(ctx.startUrl);
530
783
  /* architecture */
531
784
  const deep = okPages.filter((p) => p.depth > ctx.thresholds.deepPageDepth);
532
785
  if (deep.length) {
533
786
  out.push(finding(m("site.deep-pages"), `${deep.length} page(s) are more than ${ctx.thresholds.deepPageDepth} clicks from the start URL.`, {
787
+ fix: deep
788
+ .slice(0, MAX_EDITS)
789
+ .map((p) => addLinkEdit(home, p.url, `Words that say what ${p.url} is about. Or link it from a category or hub page one or two clicks from the home page.`)),
534
790
  evidence: {
535
791
  count: deep.length,
536
792
  examples: cap(deep.map((p) => ({ url: p.url, depth: p.depth }))),
@@ -541,17 +797,53 @@ const architectureFindings = ({ ctx, okPages }) => {
541
797
  if (ctx.probes.soft404 &&
542
798
  ctx.probes.soft404.status !== null &&
543
799
  ctx.probes.soft404.status < 400) {
544
- out.push(finding(m("site.soft-404"), `${ctx.probes.soft404.url} returned HTTP ${ctx.probes.soft404.status}.`, { evidence: ctx.probes.soft404 }));
800
+ out.push(finding(m("site.soft-404"), `${ctx.probes.soft404.url} returned HTTP ${ctx.probes.soft404.status}.`, {
801
+ fix: [
802
+ {
803
+ in: "http",
804
+ action: "replace",
805
+ url: ctx.probes.soft404.url,
806
+ current: statusLine(ctx.probes.soft404.status),
807
+ code: statusLine(404),
808
+ write: "For every URL no route matches: the catch-all route or the single-page app's fallback should answer 404 (it may keep showing its not-found page).",
809
+ },
810
+ ],
811
+ evidence: ctx.probes.soft404,
812
+ }));
545
813
  }
546
814
  if (ctx.probes.httpRedirect &&
547
815
  ctx.probes.httpRedirect.status !== null &&
548
816
  !ctx.probes.httpRedirect.redirectsToHttps) {
549
- out.push(finding(m("site.http-not-redirected"), `${ctx.probes.httpRedirect.url} returned HTTP ${ctx.probes.httpRedirect.status} without redirecting to https.`, { evidence: ctx.probes.httpRedirect }));
817
+ out.push(finding(m("site.http-not-redirected"), `${ctx.probes.httpRedirect.url} returned HTTP ${ctx.probes.httpRedirect.status} without redirecting to https.`, {
818
+ fix: [
819
+ {
820
+ in: "http",
821
+ action: "replace",
822
+ url: ctx.probes.httpRedirect.url,
823
+ current: statusLine(ctx.probes.httpRedirect.status),
824
+ code: redirectTo(ctx.probes.httpRedirect.url.replace(/^http:/iu, "https:")),
825
+ write: "For every http:// URL, to the same path on https://, at the server or CDN.",
826
+ },
827
+ ],
828
+ evidence: ctx.probes.httpRedirect,
829
+ }));
550
830
  }
551
831
  if (ctx.probes.hostVariant &&
552
832
  ctx.probes.hostVariant.status !== null &&
553
833
  !ctx.probes.hostVariant.redirectsToMain) {
554
- out.push(finding(m("site.host-variant-not-redirected"), `${ctx.probes.hostVariant.url} returned HTTP ${ctx.probes.hostVariant.status} without redirecting to the main host.`, { evidence: ctx.probes.hostVariant }));
834
+ out.push(finding(m("site.host-variant-not-redirected"), `${ctx.probes.hostVariant.url} returned HTTP ${ctx.probes.hostVariant.status} without redirecting to the main host.`, {
835
+ fix: [
836
+ {
837
+ in: "http",
838
+ action: "replace",
839
+ url: ctx.probes.hostVariant.url,
840
+ current: statusLine(ctx.probes.hostVariant.status),
841
+ code: redirectTo(onOrigin(ctx.probes.hostVariant.url, origin)),
842
+ write: `For every URL on this host, to the same path on ${origin}, at the DNS host, server or CDN.`,
843
+ },
844
+ ],
845
+ evidence: ctx.probes.hostVariant,
846
+ }));
555
847
  }
556
848
  return out;
557
849
  };
@@ -559,23 +851,34 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
559
851
  const out = [];
560
852
  /* hreflang return links */
561
853
  const missingReturn = [];
854
+ const returnLinks = [];
562
855
  for (const p of ctx.pages) {
856
+ // A page whose canonical names another URL (a parameter or tracking variant) is folded into
857
+ // that URL, so its hreflang links do not count and nothing should link back to it.
858
+ if (p.page.canonical && key(p.page.canonical) !== key(p.url)) {
859
+ continue;
860
+ }
861
+ // en and x-default often name the same page: one missing return link, not two.
862
+ const seen = new Set();
563
863
  for (const alt of p.page.hreflang) {
564
864
  if (!alt.url) {
565
865
  continue;
566
866
  }
567
867
  const target = pagesByKey.get(key(alt.url));
568
- if (!target || target === p) {
868
+ if (!target || target === p || seen.has(target)) {
569
869
  continue;
570
870
  }
871
+ seen.add(target);
571
872
  const linksBack = target.page.hreflang.some((h) => h.url && key(h.url) === key(p.url));
572
873
  if (!linksBack) {
573
874
  missingReturn.push({ page: p.url, alternate: alt.url });
875
+ returnLinks.push(returnLinkEdit(target, p, key));
574
876
  }
575
877
  }
576
878
  }
577
879
  if (missingReturn.length) {
578
880
  out.push(finding(m("site.hreflang-no-return-link"), `${missingReturn.length} hreflang alternate(s) do not link back.`, {
881
+ fix: returnLinks.slice(0, MAX_EDITS),
579
882
  evidence: {
580
883
  count: missingReturn.length,
581
884
  examples: cap(missingReturn),
@@ -584,6 +887,8 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
584
887
  }
585
888
  if (ctx.crawl.hitPageLimit) {
586
889
  out.push(finding(m("site.crawl-limit-reached"), `Crawl stopped after ${ctx.crawl.pagesCrawled} pages (limit ${ctx.crawl.maxPages}); ${ctx.crawl.pagesDiscovered} URLs were discovered.`, {
890
+ // About the audit, not the site: the change is a larger --max-pages, not an edit.
891
+ fix: [],
587
892
  evidence: {
588
893
  pagesCrawled: ctx.crawl.pagesCrawled,
589
894
  pagesDiscovered: ctx.crawl.pagesDiscovered,
@@ -593,8 +898,39 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
593
898
  }
594
899
  return out;
595
900
  };
901
+ /** The canonical a page should name instead of a broken, redirecting, noindex or chained one. */
902
+ const canonicalRepair = (scope, p, problem) => {
903
+ if (problem.kind === "redirects" || problem.kind === "canonical") {
904
+ return canonicalEdit(p, problem.to ?? p.url);
905
+ }
906
+ if (problem.kind === "noindex") {
907
+ return canonicalEdit(p, p.url, "The page's own URL. If the noindex page is the one meant to rank, remove its noindex and keep this canonical instead.");
908
+ }
909
+ const live = p.page.canonical ? liveSpelling(scope, p.page.canonical) : null;
910
+ return canonicalEdit(p, live ?? p.url, live
911
+ ? undefined
912
+ : "The page's own URL. If another page is the preferred one, use its live URL.");
913
+ };
596
914
  /** What is wrong with a URL that metadata (a canonical, an hreflang link) points to, or null. */
597
- const targetProblem = ({ key, pagesByKey, checksByKey, isIndexable }, url) => {
915
+ /**
916
+ * Where a chain of canonicals ends: the first page that names itself (or nothing), or the last URL
917
+ * before the chain loops or leaves the crawled pages.
918
+ */
919
+ const finalCanonical = ({ key, pagesByKey }, url) => {
920
+ let current = url;
921
+ const seen = new Set([key(url)]);
922
+ for (let hops = 0; hops < 10; hops += 1) {
923
+ const next = pagesByKey.get(key(current))?.page.canonical;
924
+ if (!next || seen.has(key(next))) {
925
+ return current;
926
+ }
927
+ seen.add(key(next));
928
+ current = next;
929
+ }
930
+ return current;
931
+ };
932
+ const targetProblem = (scope, url) => {
933
+ const { key, pagesByKey, checksByKey, isIndexable } = scope;
598
934
  const check = checksByKey.get(key(url));
599
935
  if (check && (check.error || (check.status ?? 0) >= 400)) {
600
936
  return {
@@ -603,7 +939,11 @@ const targetProblem = ({ key, pagesByKey, checksByKey, isIndexable }, url) => {
603
939
  };
604
940
  }
605
941
  if (check && check.redirectChain.length > 0) {
606
- return { kind: "redirects", detail: `redirects to ${check.finalUrl}` };
942
+ return {
943
+ kind: "redirects",
944
+ detail: `redirects to ${check.finalUrl}`,
945
+ to: check.finalUrl,
946
+ };
607
947
  }
608
948
  const target = pagesByKey.get(key(url));
609
949
  if (!target) {
@@ -614,7 +954,11 @@ const targetProblem = ({ key, pagesByKey, checksByKey, isIndexable }, url) => {
614
954
  }
615
955
  const { canonical } = target.page;
616
956
  return canonical && key(canonical) !== key(target.url)
617
- ? { kind: "canonical", detail: `names ${canonical} as canonical` }
957
+ ? {
958
+ kind: "canonical",
959
+ detail: `names ${canonical} as canonical`,
960
+ to: finalCanonical(scope, canonical),
961
+ }
618
962
  : null;
619
963
  };
620
964
  /** Canonicals that point at broken, redirecting, noindex or re-canonicalized URLs. */
@@ -626,6 +970,12 @@ const canonicalGraphFindings = (scope) => {
626
970
  noindex: [],
627
971
  canonical: [],
628
972
  };
973
+ const repairs = {
974
+ broken: [],
975
+ redirects: [],
976
+ noindex: [],
977
+ canonical: [],
978
+ };
629
979
  for (const p of okPages) {
630
980
  const { canonical } = p.page;
631
981
  // A noindex page's canonical carries no weight; the noindex wins.
@@ -639,6 +989,7 @@ const canonicalGraphFindings = (scope) => {
639
989
  canonical,
640
990
  problem: problem.detail,
641
991
  });
992
+ repairs[problem.kind].push(canonicalRepair(scope, p, problem));
642
993
  }
643
994
  }
644
995
  const rules = [
@@ -668,6 +1019,7 @@ const canonicalGraphFindings = (scope) => {
668
1019
  return list.length > 0
669
1020
  ? [
670
1021
  finding(m(id), `${list.length} page(s) ${label}.`, {
1022
+ fix: repairs[kind].slice(0, MAX_EDITS),
671
1023
  evidence: { count: list.length, examples: cap(list) },
672
1024
  }),
673
1025
  ]
@@ -685,7 +1037,7 @@ const isUtilityPage = (url) => {
685
1037
  }
686
1038
  };
687
1039
  /** Noindex pages that other pages of the site link to, most linked first. */
688
- const noindexLinkFindings = ({ ctx, key, okPages, isIndexable, }) => {
1040
+ const noindexLinkFindings = ({ ctx, key, pagesByKey, okPages, isIndexable, }) => {
689
1041
  const linkedFrom = new Map();
690
1042
  for (const [target, sources] of ctx.linkSources) {
691
1043
  const k = key(target);
@@ -707,15 +1059,42 @@ const noindexLinkFindings = ({ ctx, key, okPages, isIndexable, }) => {
707
1059
  .toSorted((a, b) => b.linkedFrom - a.linkedFrom);
708
1060
  return linked.length > 0
709
1061
  ? [
710
- finding(m("site.links-to-noindex"), `${linked.length} noindex page(s) are linked from other pages; the most linked: ${linked[0]?.url} (${linked[0]?.linkedFrom} pages).`, { evidence: { count: linked.length, pages: cap(linked) } }),
1062
+ finding(m("site.links-to-noindex"), `${linked.length} noindex page(s) are linked from other pages; the most linked: ${linked[0]?.url} (${linked[0]?.linkedFrom} pages).`, {
1063
+ fix: linked
1064
+ .flatMap((l) => {
1065
+ const record = pagesByKey.get(key(l.url));
1066
+ return record
1067
+ ? removeNoindexEdits(record, `Only if ${l.url} belongs in search results; leave the noindex on a page kept out on purpose.`)
1068
+ : [];
1069
+ })
1070
+ .slice(0, MAX_EDITS),
1071
+ evidence: { count: linked.length, pages: cap(linked) },
1072
+ }),
711
1073
  ]
712
1074
  : [];
713
1075
  };
1076
+ /**
1077
+ * Points an hreflang link at the canonical, live URL its alternate leads to, or removes it when the
1078
+ * alternate is broken or noindex: an annotation to such a page is ignored anyway.
1079
+ */
1080
+ const hreflangRepair = (p, alt, problem) => {
1081
+ const at = `link[rel="alternate"][hreflang="${alt.lang}"]`;
1082
+ if ((problem.kind === "redirects" || problem.kind === "canonical") &&
1083
+ problem.to) {
1084
+ return onUrl(setAttributeEdit(alt.tag, at, "href", escapeAttribute(problem.to)), locationOf(p));
1085
+ }
1086
+ return removeEdit("html", alt.tag, {
1087
+ url: locationOf(p),
1088
+ at,
1089
+ write: `${alt.url} ${problem.detail}. If that language version should exist, list its live, indexable URL here instead.`,
1090
+ });
1091
+ };
714
1092
  /** hreflang alternates that are broken, redirect, are noindex or canonicalize elsewhere. */
715
1093
  const hreflangTargetFindings = (scope) => {
716
1094
  const { key, okPages } = scope;
717
1095
  const invalid = [];
718
1096
  const seen = new Set();
1097
+ const fix = [];
719
1098
  for (const p of okPages) {
720
1099
  for (const alt of p.page.hreflang) {
721
1100
  if (!alt.url || key(alt.url) === key(p.url)) {
@@ -733,26 +1112,19 @@ const hreflangTargetFindings = (scope) => {
733
1112
  alternate: alt.url,
734
1113
  problem: problem.detail,
735
1114
  });
1115
+ fix.push(hreflangRepair(p, alt, problem));
736
1116
  }
737
1117
  }
738
1118
  }
739
1119
  return invalid.length > 0
740
1120
  ? [
741
- finding(m("site.hreflang-target-invalid"), `${invalid.length} hreflang alternate(s) point to URLs that are broken, redirect, are noindex or are not canonical.`, { evidence: { count: invalid.length, examples: cap(invalid) } }),
1121
+ finding(m("site.hreflang-target-invalid"), `${invalid.length} hreflang alternate(s) point to URLs that are broken, redirect, are noindex or are not canonical.`, {
1122
+ fix: fix.slice(0, MAX_EDITS),
1123
+ evidence: { count: invalid.length, examples: cap(invalid) },
1124
+ }),
742
1125
  ]
743
1126
  : [];
744
1127
  };
745
- /** The URL with its path lower-cased and trailing slashes dropped: one key for every spelling. */
746
- const spellingKey = (url) => {
747
- try {
748
- const u = new URL(url);
749
- const pathname = trimTrailingSlashes(u.pathname.toLowerCase()) || "/";
750
- return `${u.origin}${pathname}${u.search}`;
751
- }
752
- catch {
753
- return url;
754
- }
755
- };
756
1128
  const pathnameOf = (url) => {
757
1129
  try {
758
1130
  return new URL(url).pathname;
@@ -805,8 +1177,19 @@ const urlVariantFindings = (scope) => {
805
1177
  return [];
806
1178
  }
807
1179
  const kinds = [...new Set(duplicates.map((d) => d.kind))];
1180
+ const fix = duplicates.slice(0, MAX_EDITS).map((d) => ({
1181
+ in: "http",
1182
+ action: "replace",
1183
+ url: d.variant,
1184
+ current: statusLine(d.status ?? 200),
1185
+ code: redirectTo(d.url),
1186
+ write: d.kind === "trailing-slash"
1187
+ ? "For every URL of the site, to the one spelling with or without the trailing slash: the trailingSlash setting in Next.js, Nuxt or Astro, or a redirect rule at the server or CDN."
1188
+ : "For every URL spelled in another case, to the spelling the site links to: a redirect rule at the server or CDN.",
1189
+ }));
808
1190
  return [
809
1191
  finding(m("site.url-variant-duplicate"), `${duplicates.length} page(s) also answer at a ${kinds.join(" and a ")} variant of their URL.`, {
1192
+ fix,
810
1193
  evidence: {
811
1194
  count: duplicates.length,
812
1195
  examples: cap(duplicates.map(({ kind, url, variant, canonical }) => ({
@@ -829,7 +1212,16 @@ const withoutQuery = (url) => {
829
1212
  }
830
1213
  };
831
1214
  /** Pages that differ only in their query string, show the same text and do not consolidate. */
832
- const parameterDuplicateFindings = ({ key, okPages, isIndexable, }) => {
1215
+ /** Canonicals from each parameter version to the URL without the parameter, or the first URL. */
1216
+ const parameterCanonicals = ({ key, pagesByKey }, group) => {
1217
+ const target = group.urls.find((u) => !u.includes("?")) ?? group.urls[0] ?? group.path;
1218
+ return group.urls.flatMap((u) => {
1219
+ const record = key(u) === key(target) ? undefined : pagesByKey.get(key(u));
1220
+ return record ? [canonicalEdit(record, target)] : [];
1221
+ });
1222
+ };
1223
+ const parameterDuplicateFindings = (scope) => {
1224
+ const { key, okPages, isIndexable } = scope;
833
1225
  const byPath = new Map();
834
1226
  for (const p of okPages) {
835
1227
  const { canonical } = p.page;
@@ -867,12 +1259,41 @@ const parameterDuplicateFindings = ({ key, okPages, isIndexable, }) => {
867
1259
  }
868
1260
  return groups.length > 0
869
1261
  ? [
870
- finding(m("site.parameter-duplicates"), `${groups.length} path(s) are crawled under several query strings with the same text.`, { evidence: { count: groups.length, groups: cap(groups) } }),
1262
+ finding(m("site.parameter-duplicates"), `${groups.length} path(s) are crawled under several query strings with the same text.`, {
1263
+ fix: groups
1264
+ .flatMap((g) => parameterCanonicals(scope, g))
1265
+ .slice(0, MAX_EDITS),
1266
+ evidence: { count: groups.length, groups: cap(groups) },
1267
+ }),
871
1268
  ]
872
1269
  : [];
873
1270
  };
1271
+ /** Points each <img> that shows a broken image at a slot for the image's working URL. */
1272
+ const brokenImageEdits = ({ key, pagesByKey }, image, pages) => {
1273
+ const write = `The URL the image file is deployed at; or upload the missing file to ${image.url}.`;
1274
+ const edits = [...pages].flatMap((u) => {
1275
+ const record = pagesByKey.get(key(u));
1276
+ return record
1277
+ ? imagesOf(record, image.url).map((img) => onUrl(setAttributeEdit(img.tag, "img", "src", slot("working URL of the image"), write), locationOf(record)))
1278
+ : [];
1279
+ });
1280
+ // An image from srcset or a <picture> source has no <img src> to quote: upload the file instead.
1281
+ return edits.length > 0
1282
+ ? edits
1283
+ : [
1284
+ {
1285
+ in: "asset",
1286
+ action: "add",
1287
+ url: image.url,
1288
+ at: "new file",
1289
+ code: slot("the image file"),
1290
+ write: `Upload the image the pages show here, or change the pages that point to ${image.url}.`,
1291
+ },
1292
+ ];
1293
+ };
874
1294
  /** Images that answer 4xx/5xx or do not load, with the pages that show them. */
875
- const brokenImageFindings = (ctx) => {
1295
+ const brokenImageFindings = (scope) => {
1296
+ const { ctx } = scope;
876
1297
  const broken = [];
877
1298
  for (const [url, check] of ctx.imageChecks ?? []) {
878
1299
  if (check.error || (check.status ?? 0) >= 400) {
@@ -889,7 +1310,12 @@ const brokenImageFindings = (ctx) => {
889
1310
  }
890
1311
  return broken.length > 0
891
1312
  ? [
892
- finding(m("site.broken-images"), `${broken.length} image URL(s) are broken.`, { evidence: { count: broken.length, examples: cap(broken, 20) } }),
1313
+ finding(m("site.broken-images"), `${broken.length} image URL(s) are broken.`, {
1314
+ fix: broken
1315
+ .flatMap((b) => brokenImageEdits(scope, b, ctx.imageSources?.get(b.url) ?? []))
1316
+ .slice(0, MAX_EDITS),
1317
+ evidence: { count: broken.length, examples: cap(broken, 20) },
1318
+ }),
893
1319
  ]
894
1320
  : [];
895
1321
  };
@@ -913,20 +1339,21 @@ export const runSiteRules = (ctx) => {
913
1339
  okPages: ctx.pages.filter((p) => (p.response?.status ?? 200) < 400),
914
1340
  isIndexable: (p) => !robotsDirectives(p.page, p.response).noindex &&
915
1341
  (p.response?.status ?? 200) < 400,
1342
+ linksTo: (p, targetKey) => linksTo(p, targetKey, key),
916
1343
  };
917
1344
  return [
918
- ...robotsFindings(ctx),
1345
+ ...robotsFindings(scope),
919
1346
  ...robotsContentFindings(ctx),
920
1347
  ...blockedResourceFindings(scope),
921
1348
  ...sitemapFindings(scope),
922
1349
  ...sitemapProtocolFindings(ctx),
923
- ...linkHealthFindings(ctx),
924
- ...unreachedFindings(ctx),
1350
+ ...linkHealthFindings(scope),
1351
+ ...unreachedFindings(scope),
925
1352
  ...duplicateFindings(scope),
926
1353
  ...architectureFindings(scope),
927
1354
  ...urlVariantFindings(scope),
928
1355
  ...parameterDuplicateFindings(scope),
929
- ...brokenImageFindings(ctx),
1356
+ ...brokenImageFindings(scope),
930
1357
  ...canonicalGraphFindings(scope),
931
1358
  ...noindexLinkFindings(scope),
932
1359
  ...hreflangTargetFindings(scope),