@rivi-software/agentic-seo-check 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -5
- package/dist/cli.js +20 -0
- package/dist/cli.js.map +1 -1
- package/dist/core/aeo/audit.d.ts +8 -0
- package/dist/core/aeo/audit.js +101 -17
- package/dist/core/aeo/audit.js.map +1 -1
- package/dist/core/aeo/bots.d.ts +48 -0
- package/dist/core/aeo/bots.js +151 -5
- package/dist/core/aeo/bots.js.map +1 -1
- package/dist/core/aeo/fixes.d.ts +38 -0
- package/dist/core/aeo/fixes.js +111 -0
- package/dist/core/aeo/fixes.js.map +1 -0
- package/dist/core/aeo/site-rules.d.ts +5 -0
- package/dist/core/aeo/site-rules.js +293 -10
- package/dist/core/aeo/site-rules.js.map +1 -1
- package/dist/core/audit.d.ts +8 -0
- package/dist/core/audit.js +144 -25
- package/dist/core/audit.js.map +1 -1
- package/dist/core/crawl.js +43 -4
- package/dist/core/crawl.js.map +1 -1
- package/dist/core/html/answer.d.ts +5 -2
- package/dist/core/html/answer.js +25 -7
- package/dist/core/html/answer.js.map +1 -1
- package/dist/core/html/dom.d.ts +13 -1
- package/dist/core/html/dom.js +30 -2
- package/dist/core/html/dom.js.map +1 -1
- package/dist/core/html/extract.js +122 -30
- package/dist/core/html/extract.js.map +1 -1
- package/dist/core/html/head.js +114 -19
- package/dist/core/html/head.js.map +1 -1
- package/dist/core/html/resources.js +15 -7
- package/dist/core/html/resources.js.map +1 -1
- package/dist/core/html/structured-data.js +21 -5
- package/dist/core/html/structured-data.js.map +1 -1
- package/dist/core/lighthouse.d.ts +2 -5
- package/dist/core/lighthouse.js +290 -5
- package/dist/core/lighthouse.js.map +1 -1
- package/dist/core/performance-fixes.d.ts +25 -0
- package/dist/core/performance-fixes.js +446 -0
- package/dist/core/performance-fixes.js.map +1 -0
- package/dist/core/performance-rules.js +56 -8
- package/dist/core/performance-rules.js.map +1 -1
- package/dist/core/recheck.d.ts +68 -0
- package/dist/core/recheck.js +247 -0
- package/dist/core/recheck.js.map +1 -0
- package/dist/core/render-fixes.d.ts +27 -0
- package/dist/core/render-fixes.js +164 -0
- package/dist/core/render-fixes.js.map +1 -0
- package/dist/core/render-rules.js +60 -21
- package/dist/core/render-rules.js.map +1 -1
- package/dist/core/report/markdown.js +68 -1
- package/dist/core/report/markdown.js.map +1 -1
- package/dist/core/robots.d.ts +5 -0
- package/dist/core/robots.js +8 -0
- package/dist/core/robots.js.map +1 -1
- package/dist/core/rules/aeo.js +339 -25
- package/dist/core/rules/aeo.js.map +1 -1
- package/dist/core/rules/canonical.js +107 -11
- package/dist/core/rules/canonical.js.map +1 -1
- package/dist/core/rules/content.js +65 -7
- package/dist/core/rules/content.js.map +1 -1
- package/dist/core/rules/document.js +73 -8
- package/dist/core/rules/document.js.map +1 -1
- package/dist/core/rules/fix.d.ts +55 -0
- package/dist/core/rules/fix.js +159 -0
- package/dist/core/rules/fix.js.map +1 -0
- package/dist/core/rules/head.js +81 -4
- package/dist/core/rules/head.js.map +1 -1
- package/dist/core/rules/headings.js +92 -9
- package/dist/core/rules/headings.js.map +1 -1
- package/dist/core/rules/hreflang.js +79 -9
- package/dist/core/rules/hreflang.js.map +1 -1
- package/dist/core/rules/images.js +49 -0
- package/dist/core/rules/images.js.map +1 -1
- package/dist/core/rules/index.d.ts +11 -0
- package/dist/core/rules/index.js +21 -1
- package/dist/core/rules/index.js.map +1 -1
- package/dist/core/rules/indexing.js +108 -1
- package/dist/core/rules/indexing.js.map +1 -1
- package/dist/core/rules/keyword.js +92 -2
- package/dist/core/rules/keyword.js.map +1 -1
- package/dist/core/rules/links.js +42 -0
- package/dist/core/rules/links.js.map +1 -1
- package/dist/core/rules/loading.js +54 -1
- package/dist/core/rules/loading.js.map +1 -1
- package/dist/core/rules/meta.js +40 -6
- package/dist/core/rules/meta.js.map +1 -1
- package/dist/core/rules/page-fixes.d.ts +39 -0
- package/dist/core/rules/page-fixes.js +107 -0
- package/dist/core/rules/page-fixes.js.map +1 -0
- package/dist/core/rules/quoted-tags.d.ts +23 -0
- package/dist/core/rules/quoted-tags.js +79 -0
- package/dist/core/rules/quoted-tags.js.map +1 -0
- package/dist/core/rules/response.js +71 -7
- package/dist/core/rules/response.js.map +1 -1
- package/dist/core/rules/rule.d.ts +17 -4
- package/dist/core/rules/rule.js +11 -2
- package/dist/core/rules/rule.js.map +1 -1
- package/dist/core/rules/social.js +101 -10
- package/dist/core/rules/social.js.map +1 -1
- package/dist/core/rules/structured-data-fixes.d.ts +33 -0
- package/dist/core/rules/structured-data-fixes.js +469 -0
- package/dist/core/rules/structured-data-fixes.js.map +1 -0
- package/dist/core/rules/structured-data-values.js +242 -31
- package/dist/core/rules/structured-data-values.js.map +1 -1
- package/dist/core/rules/structured-data.js +171 -4
- package/dist/core/rules/structured-data.js.map +1 -1
- package/dist/core/rules/title.js +39 -5
- package/dist/core/rules/title.js.map +1 -1
- package/dist/core/rules/urls.js +166 -0
- package/dist/core/rules/urls.js.map +1 -1
- package/dist/core/site-fixes.d.ts +111 -0
- package/dist/core/site-fixes.js +650 -0
- package/dist/core/site-fixes.js.map +1 -0
- package/dist/core/site-rules.js +481 -54
- package/dist/core/site-rules.js.map +1 -1
- package/dist/core/sitemap.d.ts +16 -0
- package/dist/core/sitemap.js +34 -0
- package/dist/core/sitemap.js.map +1 -1
- package/dist/core/types.d.ts +223 -3
- package/dist/core/types.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +16 -2
- package/dist/mcp.js.map +1 -1
- package/package.json +1 -1
- package/skills/seo-audit/SKILL.md +35 -5
package/dist/core/site-rules.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { normalizeForComparison, shingles, jaccard } from "./html/text.js";
|
|
2
2
|
import { agentMatcher } from "./robots.js";
|
|
3
|
+
import { escapeAttribute, removeEdit, setAttributeEdit, slot, } from "./rules/fix.js";
|
|
3
4
|
import { robotsDirectives } from "./rules/indexing.js";
|
|
4
5
|
import { finding, GOOGLE_DOCS, cap } from "./rules/rule.js";
|
|
6
|
+
import { addLinkEdit, addToSitemapFix, blockedResourcesFix, canonicalEdit, imagesOf, blockingRule, lastmodFix, locationOf, linkSelector, linksTo, nestedIndexFix, hrefLike, onUrl, otherHostFix, redirectTo, redescribeEdit, removeNoindexEdits, returnLinkEdit, reheadEdit, relinkEdit, linesOf, retitleEdit, statusLine, removeFromSitemap, replaceInSitemap, robotsBlocksAllFix, robotsErrorFix, robotsMissingFix, robotsTooLargeFix, robotsUnsupportedFix, MAX_EDITS, sitemapErrorFix, sitemapMissingFix, splitSitemapFix, } from "./site-fixes.js";
|
|
5
7
|
import { MIN_COMPARABLE_WORDS, SAME_TEXT } from "./site-probes.js";
|
|
8
|
+
import { sitemapLayout } from "./sitemap.js";
|
|
6
9
|
import { normalizeUrl, normalizePath, trimTrailingSlashes } from "./url.js";
|
|
7
10
|
const meta = (id, severity, description, recommendation, docs) => ({
|
|
8
11
|
id,
|
|
@@ -85,8 +88,24 @@ const pathKey = (url) => {
|
|
|
85
88
|
return url;
|
|
86
89
|
}
|
|
87
90
|
};
|
|
88
|
-
|
|
91
|
+
/** The URL with its path lower-cased and trailing slashes dropped: one key for every spelling. */
|
|
92
|
+
const spellingKey = (url) => {
|
|
93
|
+
try {
|
|
94
|
+
const u = new URL(url);
|
|
95
|
+
const pathname = trimTrailingSlashes(u.pathname.toLowerCase()) || "/";
|
|
96
|
+
return `${u.origin}${pathname}${u.search}`;
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return url;
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
const pushDuplicates = (out, rule, pages, pick, label, fixFor) => {
|
|
89
103
|
const groups = new Map();
|
|
104
|
+
// Every page but the first of a group gets a value of its own; the first keeps what it has.
|
|
105
|
+
const others = [];
|
|
106
|
+
// Pages are grouped by a lowercased key, but the evidence quotes the value as the first page
|
|
107
|
+
// wrote it, so a report or pull request shows the text a reviewer would find on the site.
|
|
108
|
+
const written = new Map();
|
|
90
109
|
for (const p of pages) {
|
|
91
110
|
const value = pick(p);
|
|
92
111
|
if (!value) {
|
|
@@ -94,6 +113,12 @@ const pushDuplicates = (out, rule, pages, pick, label) => {
|
|
|
94
113
|
}
|
|
95
114
|
const k = normalizeForComparison(value);
|
|
96
115
|
pushTo(groups, k, p.url);
|
|
116
|
+
if (written.has(k)) {
|
|
117
|
+
others.push(p);
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
written.set(k, value);
|
|
121
|
+
}
|
|
97
122
|
}
|
|
98
123
|
const dupes = [...groups.entries()].filter(([, urls]) => urls.length > 1);
|
|
99
124
|
if (!dupes.length) {
|
|
@@ -101,15 +126,44 @@ const pushDuplicates = (out, rule, pages, pick, label) => {
|
|
|
101
126
|
}
|
|
102
127
|
const pagesAffected = dupes.reduce((n, [, urls]) => n + urls.length, 0);
|
|
103
128
|
out.push(finding(rule, `${dupes.length} ${label}(s) are shared by ${pagesAffected} pages.`, {
|
|
129
|
+
fix: others.flatMap((p) => fixFor(p) ?? []).slice(0, MAX_EDITS),
|
|
104
130
|
evidence: {
|
|
105
|
-
groups: cap(dupes.map(([
|
|
106
|
-
value:
|
|
131
|
+
groups: cap(dupes.map(([k, urls]) => ({
|
|
132
|
+
value: (written.get(k) ?? k).slice(0, 120),
|
|
107
133
|
pages: cap(urls, 5),
|
|
108
134
|
count: urls.length,
|
|
109
135
|
}))),
|
|
110
136
|
},
|
|
111
137
|
}));
|
|
112
138
|
};
|
|
139
|
+
/**
|
|
140
|
+
* The canonical each duplicate should name. The first page of a group is the shallowest the crawl
|
|
141
|
+
* reached: the others name it as canonical. Near-duplicates come in pairs, so a page can be in two
|
|
142
|
+
* groups (a~b and b~c, or a~c and b~c): each page gets one canonical, to a page that keeps its
|
|
143
|
+
* own, so the fix makes no chain and gives no page two.
|
|
144
|
+
*/
|
|
145
|
+
const duplicateCanonicalEdits = (groups, byUrl) => {
|
|
146
|
+
const canonicalOf = new Map();
|
|
147
|
+
const roots = new Set();
|
|
148
|
+
const fix = [];
|
|
149
|
+
for (const { pages: [first, ...rest], } of groups) {
|
|
150
|
+
const root = first ? (canonicalOf.get(first) ?? first) : undefined;
|
|
151
|
+
for (const u of rest) {
|
|
152
|
+
const record = byUrl.get(u);
|
|
153
|
+
if (!root ||
|
|
154
|
+
!record ||
|
|
155
|
+
u === root ||
|
|
156
|
+
canonicalOf.has(u) ||
|
|
157
|
+
roots.has(u)) {
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
canonicalOf.set(u, root);
|
|
161
|
+
roots.add(root);
|
|
162
|
+
fix.push(canonicalEdit(record, root, `If ${u} is meant to be a page of its own, make its text distinct instead.`));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return fix;
|
|
166
|
+
};
|
|
113
167
|
const pushDuplicateContent = (out, pages) => {
|
|
114
168
|
const candidates = pages.filter((p) => p.page.wordCount >= 50);
|
|
115
169
|
const groups = [];
|
|
@@ -150,7 +204,9 @@ const pushDuplicateContent = (out, pages) => {
|
|
|
150
204
|
}
|
|
151
205
|
}
|
|
152
206
|
if (groups.length) {
|
|
153
|
-
|
|
207
|
+
const byUrl = new Map(candidates.map((p) => [p.url, p]));
|
|
208
|
+
const fix = duplicateCanonicalEdits(groups, byUrl);
|
|
209
|
+
out.push(finding(m("site.duplicate-content"), `${groups.length} group(s) of pages have (near-)identical body text.`, { fix: fix.slice(0, MAX_EDITS), evidence: { groups: cap(groups) } }));
|
|
154
210
|
}
|
|
155
211
|
};
|
|
156
212
|
/** Google reads the first 500 KiB of robots.txt and ignores the rest. */
|
|
@@ -158,7 +214,7 @@ const ROBOTS_MAX_BYTES = 512_000;
|
|
|
158
214
|
const unsupportedRobotsLines = (text) => {
|
|
159
215
|
const retired = [];
|
|
160
216
|
const crawlDelay = [];
|
|
161
|
-
for (const [i, raw] of text
|
|
217
|
+
for (const [i, raw] of linesOf(text).slice(0, 50_000).entries()) {
|
|
162
218
|
const line = raw.replace(/#.*$/u, "").trim();
|
|
163
219
|
const field = /^(?<field>[a-z-]+)\s*:/iu
|
|
164
220
|
.exec(line)
|
|
@@ -176,10 +232,14 @@ const unsupportedRobotsLines = (text) => {
|
|
|
176
232
|
const robotsContentFindings = (ctx) => {
|
|
177
233
|
const out = [];
|
|
178
234
|
const bytes = ctx.robots?.bytes ?? 0;
|
|
235
|
+
const text = ctx.robotsText ?? ctx.robots?.text;
|
|
236
|
+
const robotsUrl = ctx.robots?.url ?? new URL("/robots.txt", ctx.startUrl).href;
|
|
179
237
|
if (bytes > ROBOTS_MAX_BYTES) {
|
|
180
|
-
out.push(finding(m("site.robots-txt-too-large"), `robots.txt is ${Math.round(bytes / 1024)} KiB; Google reads only the first 500 KiB.`, {
|
|
238
|
+
out.push(finding(m("site.robots-txt-too-large"), `robots.txt is ${Math.round(bytes / 1024)} KiB; Google reads only the first 500 KiB.`, {
|
|
239
|
+
fix: robotsTooLargeFix(robotsUrl, text ?? "", bytes),
|
|
240
|
+
evidence: { url: ctx.robots?.url, bytes },
|
|
241
|
+
}));
|
|
181
242
|
}
|
|
182
|
-
const text = ctx.robotsText ?? ctx.robots?.text;
|
|
183
243
|
if (!text) {
|
|
184
244
|
return out;
|
|
185
245
|
}
|
|
@@ -189,6 +249,7 @@ const robotsContentFindings = (ctx) => {
|
|
|
189
249
|
? `robots.txt has ${retired.length} noindex or nofollow line(s), which Google stopped reading in 2019.`
|
|
190
250
|
: `robots.txt sets crawl-delay, which Google ignores.`, {
|
|
191
251
|
severity: retired.length > 0 ? "warning" : "info",
|
|
252
|
+
fix: robotsUnsupportedFix(robotsUrl, text),
|
|
192
253
|
evidence: {
|
|
193
254
|
url: ctx.robots?.url,
|
|
194
255
|
retired: cap(retired),
|
|
@@ -246,28 +307,44 @@ const blockedResourceFindings = ({ ctx, okPages, isIndexable, }) => {
|
|
|
246
307
|
.toSorted((a, b) => b.pages - a.pages);
|
|
247
308
|
return list.length > 0
|
|
248
309
|
? [
|
|
249
|
-
finding(m("site.robots-blocks-resources"), `robots.txt keeps Googlebot from ${list.length} script(s) or stylesheet(s) the pages load.`, {
|
|
310
|
+
finding(m("site.robots-blocks-resources"), `robots.txt keeps Googlebot from ${list.length} script(s) or stylesheet(s) the pages load.`, {
|
|
311
|
+
fix: blockedResourcesFix(robots.url, text, list.map((r) => r.url)),
|
|
312
|
+
evidence: { count: list.length, examples: cap(list) },
|
|
313
|
+
}),
|
|
250
314
|
]
|
|
251
315
|
: [];
|
|
252
316
|
};
|
|
253
317
|
/** Sitemaps over the protocol's limits, nested indexes, bad lastmod values and other hosts. */
|
|
318
|
+
/** Parts of at most 45,000 URLs or 45 MB, and at least two. */
|
|
319
|
+
const partsFor = (urls, bytes, perPart) => Math.max(2, Math.ceil((urls ?? 0) / 45_000), Math.ceil((bytes ?? 0) / perPart));
|
|
254
320
|
const sitemapProtocolFindings = (ctx) => {
|
|
255
321
|
const { problems } = ctx.sitemaps;
|
|
322
|
+
const layout = sitemapLayout(problems);
|
|
323
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
256
324
|
const out = [];
|
|
257
325
|
const over = problems.oversized.filter((o) => o.overLimit);
|
|
258
326
|
const unread = problems.oversized.filter((o) => !o.overLimit);
|
|
259
327
|
if (over.length > 0) {
|
|
260
|
-
out.push(finding(m("site.sitemap-too-large"), `${over.length} sitemap(s) exceed 50,000 URLs or 50 MB.`, {
|
|
328
|
+
out.push(finding(m("site.sitemap-too-large"), `${over.length} sitemap(s) exceed 50,000 URLs or 50 MB.`, {
|
|
329
|
+
fix: over.map((o) => splitSitemapFix(o.sitemap, partsFor(o.urls, o.bytes, 45_000_000), "Serve this sitemap index where the large sitemap was, and move its URLs into the files it lists, each under 50,000 URLs and 50 MB.")),
|
|
330
|
+
key: "over-limit",
|
|
331
|
+
evidence: { sitemaps: cap(over) },
|
|
332
|
+
}));
|
|
261
333
|
}
|
|
262
334
|
if (unread.length > 0) {
|
|
263
335
|
out.push(finding(m("site.sitemap-too-large"), `${unread.length} sitemap(s) are larger than the 15 MB this audit reads, so their URLs were not checked.`, {
|
|
264
336
|
severity: "info",
|
|
265
337
|
recommendation: "Check that each stays under 50 MB and 50,000 URLs; smaller sitemaps (10,000 URLs or fewer) are also quicker for search engines to fetch again.",
|
|
338
|
+
fix: unread.map((o) => splitSitemapFix(o.sitemap, partsFor(o.urls, o.bytes, 10_000_000), "Serve this sitemap index where the large sitemap was, and move its URLs into the files it lists, 10,000 URLs or fewer each, so search engines (and this audit) can read each one quickly.")),
|
|
339
|
+
key: "unread",
|
|
266
340
|
evidence: { sitemaps: cap(unread) },
|
|
267
341
|
}));
|
|
268
342
|
}
|
|
269
343
|
if (problems.nestedIndexes.length > 0) {
|
|
270
|
-
out.push(finding(m("site.sitemap-index-nested"), `${problems.nestedIndexes.length} sitemap index(es) are listed by another sitemap index.`, {
|
|
344
|
+
out.push(finding(m("site.sitemap-index-nested"), `${problems.nestedIndexes.length} sitemap index(es) are listed by another sitemap index.`, {
|
|
345
|
+
fix: problems.nestedIndexes.map((n) => nestedIndexFix(layout, n)),
|
|
346
|
+
evidence: { indexes: cap(problems.nestedIndexes) },
|
|
347
|
+
}));
|
|
271
348
|
}
|
|
272
349
|
const examples = [
|
|
273
350
|
[
|
|
@@ -286,22 +363,69 @@ const sitemapProtocolFindings = (ctx) => {
|
|
|
286
363
|
"are on another host than the site",
|
|
287
364
|
],
|
|
288
365
|
];
|
|
366
|
+
const fixes = {
|
|
367
|
+
"site.sitemap-lastmod-invalid": (list) => lastmodFix(layout, list, "invalid", today),
|
|
368
|
+
"site.sitemap-lastmod-future": (list) => lastmodFix(layout, list, "future", today),
|
|
369
|
+
"site.sitemap-other-host": (list) => otherHostFix(layout, list, new URL(ctx.startUrl).origin),
|
|
370
|
+
};
|
|
289
371
|
for (const [id, list, label] of examples) {
|
|
290
372
|
if (list.count > 0) {
|
|
291
373
|
out.push(finding(m(id), `${list.count} sitemap URL(s) ${label}.`, {
|
|
374
|
+
fix: fixes[id](list.examples),
|
|
292
375
|
evidence: { count: list.count, examples: list.examples },
|
|
293
376
|
}));
|
|
294
377
|
}
|
|
295
378
|
}
|
|
296
379
|
return out;
|
|
297
380
|
};
|
|
298
|
-
|
|
381
|
+
/** The sitemap URL a robots.txt should name: the one found, else /sitemap.xml. */
|
|
382
|
+
const sitemapUrlOf = (ctx) => ctx.sitemaps.sources[0] ?? new URL("/sitemap.xml", ctx.startUrl).href;
|
|
383
|
+
/**
|
|
384
|
+
* Keeps blocked pages out of the sitemap and stops crawlers following links to them; unblocking
|
|
385
|
+
* them could expose pages kept private on purpose, so the write names that rule instead.
|
|
386
|
+
*/
|
|
387
|
+
const blockedPageEdits = (scope, blocked) => {
|
|
388
|
+
const { ctx, key, pagesByKey } = scope;
|
|
389
|
+
const k = key(blocked);
|
|
390
|
+
const rule = ctx.robots && ctx.robotsText
|
|
391
|
+
? blockingRule(ctx.robots.url, ctx.robotsText, blocked)
|
|
392
|
+
: null;
|
|
393
|
+
const unblock = `If the page belongs in search results, remove ${rule ? `"${rule}"` : "the rule that blocks it"} from robots.txt instead.`;
|
|
394
|
+
const listed = ctx.sitemaps.urls.find((u) => key(u) === k);
|
|
395
|
+
if (listed) {
|
|
396
|
+
return [
|
|
397
|
+
removeFromSitemap(sitemapLayout(ctx.sitemaps.problems), listed, `robots.txt blocks it, so search engines cannot crawl it. ${unblock}`),
|
|
398
|
+
];
|
|
399
|
+
}
|
|
400
|
+
const sources = [...ctx.linkSources].flatMap(([target, from]) => key(target) === k ? [...from] : []);
|
|
401
|
+
const edits = sources.flatMap((source) => {
|
|
402
|
+
const record = pagesByKey.get(key(source));
|
|
403
|
+
return record
|
|
404
|
+
? linksTo(record, k, key).map((link) => onUrl(setAttributeEdit(link.tag, linkSelector(link), "rel", link.rel ? `${link.rel} nofollow` : "nofollow", `So crawlers do not follow it to a page robots.txt blocks. ${unblock}`), locationOf(record)))
|
|
405
|
+
: [];
|
|
406
|
+
});
|
|
407
|
+
if (edits.length > 0 || !rule || !ctx.robots) {
|
|
408
|
+
return edits;
|
|
409
|
+
}
|
|
410
|
+
return [
|
|
411
|
+
{
|
|
412
|
+
in: "robots.txt",
|
|
413
|
+
action: "remove",
|
|
414
|
+
url: ctx.robots.url,
|
|
415
|
+
current: rule,
|
|
416
|
+
write: `Only if ${blocked} belongs in search results; if it is kept out on purpose, leave robots.txt as it is.`,
|
|
417
|
+
},
|
|
418
|
+
];
|
|
419
|
+
};
|
|
420
|
+
const robotsFindings = (scope) => {
|
|
421
|
+
const { ctx } = scope;
|
|
299
422
|
const out = [];
|
|
300
423
|
/* robots.txt */
|
|
301
424
|
if (ctx.robots) {
|
|
302
425
|
if (ctx.robots.error ||
|
|
303
426
|
(ctx.robots.status !== null && ctx.robots.status >= 500)) {
|
|
304
427
|
out.push(finding(m("site.robots-txt-error"), `robots.txt at ${ctx.robots.url} could not be read (${ctx.robots.error ?? `HTTP ${ctx.robots.status}`}).`, {
|
|
428
|
+
fix: robotsErrorFix(ctx.robots.url, ctx.robots.status, ctx.robots.error),
|
|
305
429
|
evidence: {
|
|
306
430
|
url: ctx.robots.url,
|
|
307
431
|
status: ctx.robots.status,
|
|
@@ -312,11 +436,13 @@ const robotsFindings = (ctx) => {
|
|
|
312
436
|
else if (!ctx.robots.found) {
|
|
313
437
|
out.push(finding(m("site.robots-txt-missing"), `No robots.txt at ${ctx.robots.url}.`, {
|
|
314
438
|
severity: ctx.source === "directory" ? "info" : "warning",
|
|
439
|
+
fix: robotsMissingFix(ctx.robots.url, sitemapUrlOf(ctx)),
|
|
315
440
|
evidence: { url: ctx.robots.url, status: ctx.robots.status },
|
|
316
441
|
}));
|
|
317
442
|
}
|
|
318
443
|
else if (ctx.robotsAllows && !ctx.robotsAllows(ctx.startUrl)) {
|
|
319
444
|
out.push(finding(m("site.robots-txt-blocks-all"), "robots.txt blocks the start URL for the crawler user agent.", {
|
|
445
|
+
fix: robotsBlocksAllFix(ctx.robots.url, ctx.robotsText ?? ctx.robots.text ?? "", ctx.startUrl),
|
|
320
446
|
evidence: {
|
|
321
447
|
url: ctx.robots.url,
|
|
322
448
|
robotsTxt: ctx.robots.text?.slice(0, 500),
|
|
@@ -326,6 +452,10 @@ const robotsFindings = (ctx) => {
|
|
|
326
452
|
}
|
|
327
453
|
if (ctx.blockedByRobots.length) {
|
|
328
454
|
out.push(finding(m("site.robots-blocked-pages"), `${ctx.blockedByRobots.length} linked/sitemap URL(s) are disallowed by robots.txt.`, {
|
|
455
|
+
fix: ctx.blockedByRobots
|
|
456
|
+
.slice(0, MAX_EDITS)
|
|
457
|
+
.flatMap((u) => blockedPageEdits(scope, u))
|
|
458
|
+
.slice(0, MAX_EDITS),
|
|
329
459
|
evidence: {
|
|
330
460
|
count: ctx.blockedByRobots.length,
|
|
331
461
|
examples: cap(ctx.blockedByRobots),
|
|
@@ -337,6 +467,12 @@ const robotsFindings = (ctx) => {
|
|
|
337
467
|
/** Noindex, non-canonical, broken and redirecting URLs listed in the sitemap. */
|
|
338
468
|
const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
|
|
339
469
|
const out = [];
|
|
470
|
+
const layout = sitemapLayout(ctx.sitemaps.problems);
|
|
471
|
+
const listed = new Set(ctx.sitemaps.urls.map(key));
|
|
472
|
+
/** Points an entry at `to`, or removes it when the sitemap lists `to` already. */
|
|
473
|
+
const repoint = (loc, to) => listed.has(key(to))
|
|
474
|
+
? removeFromSitemap(layout, loc, `the sitemap lists ${to} already.`)
|
|
475
|
+
: replaceInSitemap(layout, loc, to);
|
|
340
476
|
const noindex = [];
|
|
341
477
|
const nonCanonical = [];
|
|
342
478
|
const broken = [];
|
|
@@ -364,10 +500,14 @@ const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
|
|
|
364
500
|
}
|
|
365
501
|
}
|
|
366
502
|
if (noindex.length) {
|
|
367
|
-
out.push(finding(m("site.sitemap-noindex-page"), `${noindex.length} sitemap URL(s) are noindex.`, {
|
|
503
|
+
out.push(finding(m("site.sitemap-noindex-page"), `${noindex.length} sitemap URL(s) are noindex.`, {
|
|
504
|
+
fix: noindex.map((u) => removeFromSitemap(layout, u, "the page is noindex, which may be on purpose. If it belongs in search results, remove its noindex instead.")),
|
|
505
|
+
evidence: { count: noindex.length, examples: cap(noindex) },
|
|
506
|
+
}));
|
|
368
507
|
}
|
|
369
508
|
if (nonCanonical.length) {
|
|
370
509
|
out.push(finding(m("site.sitemap-non-canonical"), `${nonCanonical.length} sitemap URL(s) canonicalise to another URL.`, {
|
|
510
|
+
fix: nonCanonical.map((n) => repoint(n.url, n.canonical)),
|
|
371
511
|
evidence: {
|
|
372
512
|
count: nonCanonical.length,
|
|
373
513
|
examples: cap(nonCanonical),
|
|
@@ -375,18 +515,36 @@ const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
|
|
|
375
515
|
}));
|
|
376
516
|
}
|
|
377
517
|
if (broken.length) {
|
|
378
|
-
out.push(finding(m("site.sitemap-broken-url"), `${broken.length} sitemap URL(s) return errors.`, {
|
|
518
|
+
out.push(finding(m("site.sitemap-broken-url"), `${broken.length} sitemap URL(s) return errors.`, {
|
|
519
|
+
fix: broken.map((b) => removeFromSitemap(layout, b.url, `it answers ${b.status === null ? "with no response" : `HTTP ${b.status}`}. If the page should exist, restore it instead.`)),
|
|
520
|
+
evidence: { count: broken.length, examples: cap(broken) },
|
|
521
|
+
}));
|
|
379
522
|
}
|
|
380
523
|
if (redirecting.length) {
|
|
381
524
|
out.push(finding(m("site.sitemap-redirecting-url"), `${redirecting.length} sitemap URL(s) redirect.`, {
|
|
525
|
+
fix: redirecting.map((r) => repoint(r.url, r.finalUrl)),
|
|
382
526
|
evidence: { count: redirecting.length, examples: cap(redirecting) },
|
|
383
527
|
}));
|
|
384
528
|
}
|
|
385
529
|
return out;
|
|
386
530
|
};
|
|
387
531
|
/** Indexable pages missing from the sitemap, and sitemap URLs nothing links to. */
|
|
388
|
-
|
|
532
|
+
/**
|
|
533
|
+
* Links an orphan from the home page, or takes it out of the sitemap when it does not load: a link
|
|
534
|
+
* to a broken page would only add a broken link.
|
|
535
|
+
*/
|
|
536
|
+
const orphanEdit = (scope, home, url) => {
|
|
537
|
+
const check = scope.checksByKey.get(scope.key(url));
|
|
538
|
+
if (check && (check.error || (check.status ?? 0) >= 400)) {
|
|
539
|
+
return removeFromSitemap(sitemapLayout(scope.ctx.sitemaps.problems), url, `it answers ${check.status === null ? "with no response" : `HTTP ${check.status}`}, and no page links to it.`);
|
|
540
|
+
}
|
|
541
|
+
return addLinkEdit(home, url, `Words that say what ${url} is about. Put the link on the page closest in topic, or in the navigation; if the page is obsolete, remove it from the sitemap instead.`);
|
|
542
|
+
};
|
|
543
|
+
const sitemapCoverageFindings = (scope) => {
|
|
544
|
+
const { ctx, key, pagesByKey, okPages, isIndexable } = scope;
|
|
389
545
|
const out = [];
|
|
546
|
+
const start = pagesByKey.get(key(ctx.startUrl));
|
|
547
|
+
const home = start ? locationOf(start) : ctx.startUrl;
|
|
390
548
|
const sitemapKeys = new Set(ctx.sitemaps.urls.map(key));
|
|
391
549
|
const notInSitemap = okPages
|
|
392
550
|
.filter((p) => isIndexable(p) &&
|
|
@@ -395,6 +553,7 @@ const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
|
|
|
395
553
|
.map((p) => p.url);
|
|
396
554
|
if (notInSitemap.length) {
|
|
397
555
|
out.push(finding(m("site.not-in-sitemap"), `${notInSitemap.length} crawled indexable page(s) are missing from the sitemap.`, {
|
|
556
|
+
fix: addToSitemapFix(sitemapLayout(ctx.sitemaps.problems), notInSitemap),
|
|
398
557
|
evidence: {
|
|
399
558
|
count: notInSitemap.length,
|
|
400
559
|
examples: cap(notInSitemap),
|
|
@@ -409,7 +568,12 @@ const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
|
|
|
409
568
|
linked.add(key(ctx.startUrl));
|
|
410
569
|
const orphans = ctx.sitemaps.urls.filter((u) => !linked.has(key(u)));
|
|
411
570
|
if (orphans.length) {
|
|
412
|
-
out.push(finding(m("site.orphan-pages"), `${orphans.length} sitemap URL(s) are not linked from any crawled page.`, {
|
|
571
|
+
out.push(finding(m("site.orphan-pages"), `${orphans.length} sitemap URL(s) are not linked from any crawled page.`, {
|
|
572
|
+
fix: orphans
|
|
573
|
+
.slice(0, MAX_EDITS)
|
|
574
|
+
.map((u) => orphanEdit(scope, home, u)),
|
|
575
|
+
evidence: { count: orphans.length, examples: cap(orphans) },
|
|
576
|
+
}));
|
|
413
577
|
}
|
|
414
578
|
}
|
|
415
579
|
return out;
|
|
@@ -418,7 +582,10 @@ const sitemapFindings = (scope) => {
|
|
|
418
582
|
const { ctx } = scope;
|
|
419
583
|
const out = [];
|
|
420
584
|
if (ctx.sitemaps.errors.length) {
|
|
421
|
-
out.push(finding(m("site.sitemap-error"), `${ctx.sitemaps.errors.length} sitemap problem(s).`, {
|
|
585
|
+
out.push(finding(m("site.sitemap-error"), `${ctx.sitemaps.errors.length} sitemap problem(s).`, {
|
|
586
|
+
fix: sitemapErrorFix(ctx.sitemaps.errors.slice(0, 10), new URL(ctx.startUrl).origin),
|
|
587
|
+
evidence: { errors: cap(ctx.sitemaps.errors) },
|
|
588
|
+
}));
|
|
422
589
|
}
|
|
423
590
|
// A sitemap too large to read was still found.
|
|
424
591
|
if (ctx.sitemaps.sources.length === 0 &&
|
|
@@ -426,6 +593,11 @@ const sitemapFindings = (scope) => {
|
|
|
426
593
|
ctx.sitemaps.problems.oversized.length === 0) {
|
|
427
594
|
out.push(finding(m("site.sitemap-missing"), "No sitemap was found.", {
|
|
428
595
|
severity: ctx.source === "directory" ? "info" : "warning",
|
|
596
|
+
fix: sitemapMissingFix(sitemapUrlOf(ctx), scope.okPages
|
|
597
|
+
.filter((p) => scope.isIndexable(p) &&
|
|
598
|
+
(!p.page.canonical ||
|
|
599
|
+
scope.key(p.page.canonical) === scope.key(p.url)))
|
|
600
|
+
.map((p) => p.url), ctx.robots?.found ? ctx.robots.url : null),
|
|
429
601
|
}));
|
|
430
602
|
}
|
|
431
603
|
if (ctx.sitemaps.urlCount > 0) {
|
|
@@ -433,9 +605,54 @@ const sitemapFindings = (scope) => {
|
|
|
433
605
|
}
|
|
434
606
|
return out;
|
|
435
607
|
};
|
|
608
|
+
/**
|
|
609
|
+
* Points every link to `targetKey` on the pages in `sources` at `hrefFor(link, page)`, up to
|
|
610
|
+
* `MAX_EDITS` edits.
|
|
611
|
+
*/
|
|
612
|
+
const relinkSources = (scope, sources, targetKey, hrefFor, write) => {
|
|
613
|
+
const edits = [];
|
|
614
|
+
for (const source of sources) {
|
|
615
|
+
const record = scope.pagesByKey.get(scope.key(source));
|
|
616
|
+
for (const link of record ? scope.linksTo(record, targetKey) : []) {
|
|
617
|
+
if (record && edits.length < MAX_EDITS) {
|
|
618
|
+
edits.push(relinkEdit(locationOf(record), link, hrefFor(link.href, record.url), write));
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
return edits;
|
|
623
|
+
};
|
|
624
|
+
/** A live page whose URL differs from `url` only in a trailing slash or letter case. */
|
|
625
|
+
const liveSpelling = (scope, url) => {
|
|
626
|
+
const match = scope.okPages.find((p) => spellingKey(p.url) === spellingKey(url) &&
|
|
627
|
+
scope.key(p.url) !== scope.key(url));
|
|
628
|
+
return match?.url ?? null;
|
|
629
|
+
};
|
|
630
|
+
/** Edits for broken internal links: the live spelling of the URL when there is one, else a slot. */
|
|
631
|
+
const brokenLinkEdits = (scope, broken) => broken
|
|
632
|
+
.flatMap((b) => {
|
|
633
|
+
const live = liveSpelling(scope, b.url);
|
|
634
|
+
return relinkSources(scope, scope.ctx.linkSources.get(b.key) ?? [], scope.key(b.url), (href, pageUrl) => live
|
|
635
|
+
? hrefLike(href, live, pageUrl)
|
|
636
|
+
: slot("URL of the live page this link meant"), live
|
|
637
|
+
? undefined
|
|
638
|
+
: `The URL of the page the link was meant for; remove the link if that page is gone. ${b.url} does not answer.`);
|
|
639
|
+
})
|
|
640
|
+
.slice(0, MAX_EDITS);
|
|
641
|
+
/** Points each hop of a redirect chain straight at where the chain ends. */
|
|
642
|
+
const chainEdits = (check) => check.redirectChain.slice(0, -1).map((hop, i) => ({
|
|
643
|
+
in: "http",
|
|
644
|
+
action: "replace",
|
|
645
|
+
url: hop.url,
|
|
646
|
+
current: `${statusLine(hop.status)}\nLocation: ${check.redirectChain[i + 1]?.url ?? check.finalUrl}`,
|
|
647
|
+
code: redirectTo(check.finalUrl, hop.status === 307 || hop.status === 308),
|
|
648
|
+
}));
|
|
436
649
|
/** Internal link targets that are broken, redirect once, or redirect in a chain. */
|
|
437
|
-
const linkHealthFindings = (
|
|
650
|
+
const linkHealthFindings = (scope) => {
|
|
651
|
+
const { ctx } = scope;
|
|
438
652
|
const out = [];
|
|
653
|
+
const brokenKeys = [];
|
|
654
|
+
const redirectedKeys = [];
|
|
655
|
+
const chainChecks = [];
|
|
439
656
|
/* link health */
|
|
440
657
|
const broken = [];
|
|
441
658
|
const redirected = [];
|
|
@@ -448,6 +665,7 @@ const linkHealthFindings = (ctx) => {
|
|
|
448
665
|
// Chains of two or more hops are handled first, so a first hop here is the only one.
|
|
449
666
|
const [firstHop] = check.redirectChain;
|
|
450
667
|
if (check.error || (check.status ?? 0) >= 400) {
|
|
668
|
+
brokenKeys.push({ url: check.url, key: k });
|
|
451
669
|
broken.push({
|
|
452
670
|
url: check.url,
|
|
453
671
|
status: check.status,
|
|
@@ -456,6 +674,7 @@ const linkHealthFindings = (ctx) => {
|
|
|
456
674
|
});
|
|
457
675
|
}
|
|
458
676
|
else if (check.redirectChain.length >= 2) {
|
|
677
|
+
chainChecks.push(check);
|
|
459
678
|
chains.push({
|
|
460
679
|
url: check.url,
|
|
461
680
|
hops: check.redirectChain.length,
|
|
@@ -463,6 +682,7 @@ const linkHealthFindings = (ctx) => {
|
|
|
463
682
|
});
|
|
464
683
|
}
|
|
465
684
|
else if (firstHop) {
|
|
685
|
+
redirectedKeys.push({ key: k, finalUrl: check.finalUrl });
|
|
466
686
|
redirected.push({
|
|
467
687
|
url: check.url,
|
|
468
688
|
finalUrl: check.finalUrl,
|
|
@@ -472,21 +692,32 @@ const linkHealthFindings = (ctx) => {
|
|
|
472
692
|
}
|
|
473
693
|
}
|
|
474
694
|
if (broken.length) {
|
|
475
|
-
out.push(finding(m("site.broken-internal-links"), `${broken.length} internal link target(s) are broken.`, {
|
|
695
|
+
out.push(finding(m("site.broken-internal-links"), `${broken.length} internal link target(s) are broken.`, {
|
|
696
|
+
fix: brokenLinkEdits(scope, brokenKeys),
|
|
697
|
+
evidence: { count: broken.length, examples: cap(broken, 20) },
|
|
698
|
+
}));
|
|
476
699
|
}
|
|
477
700
|
if (redirected.length) {
|
|
478
701
|
out.push(finding(m("site.redirected-internal-links"), `${redirected.length} internal link target(s) redirect.`, {
|
|
702
|
+
fix: redirectedKeys
|
|
703
|
+
.flatMap((r) => relinkSources(scope, ctx.linkSources.get(r.key) ?? [], scope.key(r.key), (href, pageUrl) => hrefLike(href, r.finalUrl, pageUrl)))
|
|
704
|
+
.slice(0, MAX_EDITS),
|
|
479
705
|
evidence: { count: redirected.length, examples: cap(redirected, 20) },
|
|
480
706
|
}));
|
|
481
707
|
}
|
|
482
708
|
if (chains.length) {
|
|
483
|
-
out.push(finding(m("site.redirect-chains"), `${chains.length} internal URL(s) have redirect chains.`, {
|
|
709
|
+
out.push(finding(m("site.redirect-chains"), `${chains.length} internal URL(s) have redirect chains.`, {
|
|
710
|
+
fix: chainChecks.flatMap(chainEdits).slice(0, MAX_EDITS),
|
|
711
|
+
evidence: { count: chains.length, examples: cap(chains) },
|
|
712
|
+
}));
|
|
484
713
|
}
|
|
485
714
|
return out;
|
|
486
715
|
};
|
|
487
716
|
/** Broken external links (when they were probed) and HTML files nothing links to. */
|
|
488
|
-
const unreachedFindings = (
|
|
717
|
+
const unreachedFindings = (scope) => {
|
|
718
|
+
const { ctx } = scope;
|
|
489
719
|
const out = [];
|
|
720
|
+
const start = scope.pagesByKey.get(scope.key(ctx.startUrl));
|
|
490
721
|
if (ctx.externalChecks) {
|
|
491
722
|
const dead = [];
|
|
492
723
|
for (const [u, check] of ctx.externalChecks) {
|
|
@@ -500,11 +731,19 @@ const unreachedFindings = (ctx) => {
|
|
|
500
731
|
}
|
|
501
732
|
}
|
|
502
733
|
if (dead.length) {
|
|
503
|
-
out.push(finding(m("site.broken-external-links"), `${dead.length} external link target(s) are broken.`, {
|
|
734
|
+
out.push(finding(m("site.broken-external-links"), `${dead.length} external link target(s) are broken.`, {
|
|
735
|
+
fix: dead
|
|
736
|
+
.flatMap((d) => relinkSources(scope, ctx.externalSources?.get(d.url) ?? [], scope.key(d.url), () => slot("current URL of the linked page"), `Where ${d.url} lives now: search for the page, or link a web archive copy; remove the link if the page is gone.`))
|
|
737
|
+
.slice(0, MAX_EDITS),
|
|
738
|
+
evidence: { count: dead.length, examples: cap(dead, 20) },
|
|
739
|
+
}));
|
|
504
740
|
}
|
|
505
741
|
}
|
|
506
742
|
if (ctx.unlinkedPages?.length) {
|
|
507
743
|
out.push(finding(m("site.unlinked-pages"), `${ctx.unlinkedPages.length} HTML file(s) are not linked from any other page.`, {
|
|
744
|
+
fix: ctx.unlinkedPages
|
|
745
|
+
.slice(0, MAX_EDITS)
|
|
746
|
+
.map((file) => addLinkEdit(start ? locationOf(start) : ctx.startUrl, ctx.pages.find((p) => (p.file ?? p.url) === file)?.url ?? file, `Words that say what ${file} is about. Link it from the page closest in topic or the navigation; if the file is obsolete, delete it from the build instead.`)),
|
|
508
747
|
evidence: {
|
|
509
748
|
count: ctx.unlinkedPages.length,
|
|
510
749
|
examples: cap(ctx.unlinkedPages),
|
|
@@ -518,19 +757,36 @@ const duplicateFindings = ({ key, okPages, isIndexable, }) => {
|
|
|
518
757
|
// Pages that name another URL as canonical are consolidated already; only the canonical ones compete.
|
|
519
758
|
const indexable = okPages.filter((p) => isIndexable(p) &&
|
|
520
759
|
(!p.page.canonical || key(p.page.canonical) === key(p.url)));
|
|
521
|
-
pushDuplicates(out, m("site.duplicate-titles"), indexable, (p) => p.page.title, "title");
|
|
522
|
-
pushDuplicates(out, m("site.duplicate-descriptions"), indexable, (p) => p.page.metaDescription, "meta description");
|
|
523
|
-
pushDuplicates(out, m("site.duplicate-h1"), indexable, (p) => p.page.h1s[0] ?? null, "H1");
|
|
760
|
+
pushDuplicates(out, m("site.duplicate-titles"), indexable, (p) => p.page.title, "title", retitleEdit);
|
|
761
|
+
pushDuplicates(out, m("site.duplicate-descriptions"), indexable, (p) => p.page.metaDescription, "meta description", redescribeEdit);
|
|
762
|
+
pushDuplicates(out, m("site.duplicate-h1"), indexable, (p) => p.page.h1s[0] ?? null, "H1", reheadEdit);
|
|
524
763
|
pushDuplicateContent(out, indexable);
|
|
525
764
|
return out;
|
|
526
765
|
};
|
|
527
766
|
/** Pages deep in the site, and how the server answers a made-up URL and the other host names. */
|
|
528
|
-
|
|
767
|
+
/** The same URL on `origin`, for a redirect to the site's main host or to https. */
|
|
768
|
+
const onOrigin = (url, origin) => {
|
|
769
|
+
try {
|
|
770
|
+
const u = new URL(url);
|
|
771
|
+
// Appended, not resolved: a "//x" path resolved against the origin would name a host.
|
|
772
|
+
return new URL(`${origin}${u.pathname}${u.search}`).href;
|
|
773
|
+
}
|
|
774
|
+
catch {
|
|
775
|
+
return origin;
|
|
776
|
+
}
|
|
777
|
+
};
|
|
778
|
+
const architectureFindings = ({ ctx, key, pagesByKey, okPages, }) => {
|
|
529
779
|
const out = [];
|
|
780
|
+
const start = pagesByKey.get(key(ctx.startUrl));
|
|
781
|
+
const home = start ? locationOf(start) : ctx.startUrl;
|
|
782
|
+
const { origin } = new URL(ctx.startUrl);
|
|
530
783
|
/* architecture */
|
|
531
784
|
const deep = okPages.filter((p) => p.depth > ctx.thresholds.deepPageDepth);
|
|
532
785
|
if (deep.length) {
|
|
533
786
|
out.push(finding(m("site.deep-pages"), `${deep.length} page(s) are more than ${ctx.thresholds.deepPageDepth} clicks from the start URL.`, {
|
|
787
|
+
fix: deep
|
|
788
|
+
.slice(0, MAX_EDITS)
|
|
789
|
+
.map((p) => addLinkEdit(home, p.url, `Words that say what ${p.url} is about. Or link it from a category or hub page one or two clicks from the home page.`)),
|
|
534
790
|
evidence: {
|
|
535
791
|
count: deep.length,
|
|
536
792
|
examples: cap(deep.map((p) => ({ url: p.url, depth: p.depth }))),
|
|
@@ -541,17 +797,53 @@ const architectureFindings = ({ ctx, okPages }) => {
|
|
|
541
797
|
if (ctx.probes.soft404 &&
|
|
542
798
|
ctx.probes.soft404.status !== null &&
|
|
543
799
|
ctx.probes.soft404.status < 400) {
|
|
544
|
-
out.push(finding(m("site.soft-404"), `${ctx.probes.soft404.url} returned HTTP ${ctx.probes.soft404.status}.`, {
|
|
800
|
+
out.push(finding(m("site.soft-404"), `${ctx.probes.soft404.url} returned HTTP ${ctx.probes.soft404.status}.`, {
|
|
801
|
+
fix: [
|
|
802
|
+
{
|
|
803
|
+
in: "http",
|
|
804
|
+
action: "replace",
|
|
805
|
+
url: ctx.probes.soft404.url,
|
|
806
|
+
current: statusLine(ctx.probes.soft404.status),
|
|
807
|
+
code: statusLine(404),
|
|
808
|
+
write: "For every URL no route matches: the catch-all route or the single-page app's fallback should answer 404 (it may keep showing its not-found page).",
|
|
809
|
+
},
|
|
810
|
+
],
|
|
811
|
+
evidence: ctx.probes.soft404,
|
|
812
|
+
}));
|
|
545
813
|
}
|
|
546
814
|
if (ctx.probes.httpRedirect &&
|
|
547
815
|
ctx.probes.httpRedirect.status !== null &&
|
|
548
816
|
!ctx.probes.httpRedirect.redirectsToHttps) {
|
|
549
|
-
out.push(finding(m("site.http-not-redirected"), `${ctx.probes.httpRedirect.url} returned HTTP ${ctx.probes.httpRedirect.status} without redirecting to https.`, {
|
|
817
|
+
out.push(finding(m("site.http-not-redirected"), `${ctx.probes.httpRedirect.url} returned HTTP ${ctx.probes.httpRedirect.status} without redirecting to https.`, {
|
|
818
|
+
fix: [
|
|
819
|
+
{
|
|
820
|
+
in: "http",
|
|
821
|
+
action: "replace",
|
|
822
|
+
url: ctx.probes.httpRedirect.url,
|
|
823
|
+
current: statusLine(ctx.probes.httpRedirect.status),
|
|
824
|
+
code: redirectTo(ctx.probes.httpRedirect.url.replace(/^http:/iu, "https:")),
|
|
825
|
+
write: "For every http:// URL, to the same path on https://, at the server or CDN.",
|
|
826
|
+
},
|
|
827
|
+
],
|
|
828
|
+
evidence: ctx.probes.httpRedirect,
|
|
829
|
+
}));
|
|
550
830
|
}
|
|
551
831
|
if (ctx.probes.hostVariant &&
|
|
552
832
|
ctx.probes.hostVariant.status !== null &&
|
|
553
833
|
!ctx.probes.hostVariant.redirectsToMain) {
|
|
554
|
-
out.push(finding(m("site.host-variant-not-redirected"), `${ctx.probes.hostVariant.url} returned HTTP ${ctx.probes.hostVariant.status} without redirecting to the main host.`, {
|
|
834
|
+
out.push(finding(m("site.host-variant-not-redirected"), `${ctx.probes.hostVariant.url} returned HTTP ${ctx.probes.hostVariant.status} without redirecting to the main host.`, {
|
|
835
|
+
fix: [
|
|
836
|
+
{
|
|
837
|
+
in: "http",
|
|
838
|
+
action: "replace",
|
|
839
|
+
url: ctx.probes.hostVariant.url,
|
|
840
|
+
current: statusLine(ctx.probes.hostVariant.status),
|
|
841
|
+
code: redirectTo(onOrigin(ctx.probes.hostVariant.url, origin)),
|
|
842
|
+
write: `For every URL on this host, to the same path on ${origin}, at the DNS host, server or CDN.`,
|
|
843
|
+
},
|
|
844
|
+
],
|
|
845
|
+
evidence: ctx.probes.hostVariant,
|
|
846
|
+
}));
|
|
555
847
|
}
|
|
556
848
|
return out;
|
|
557
849
|
};
|
|
@@ -559,23 +851,34 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
|
|
|
559
851
|
const out = [];
|
|
560
852
|
/* hreflang return links */
|
|
561
853
|
const missingReturn = [];
|
|
854
|
+
const returnLinks = [];
|
|
562
855
|
for (const p of ctx.pages) {
|
|
856
|
+
// A page whose canonical names another URL (a parameter or tracking variant) is folded into
|
|
857
|
+
// that URL, so its hreflang links do not count and nothing should link back to it.
|
|
858
|
+
if (p.page.canonical && key(p.page.canonical) !== key(p.url)) {
|
|
859
|
+
continue;
|
|
860
|
+
}
|
|
861
|
+
// en and x-default often name the same page: one missing return link, not two.
|
|
862
|
+
const seen = new Set();
|
|
563
863
|
for (const alt of p.page.hreflang) {
|
|
564
864
|
if (!alt.url) {
|
|
565
865
|
continue;
|
|
566
866
|
}
|
|
567
867
|
const target = pagesByKey.get(key(alt.url));
|
|
568
|
-
if (!target || target === p) {
|
|
868
|
+
if (!target || target === p || seen.has(target)) {
|
|
569
869
|
continue;
|
|
570
870
|
}
|
|
871
|
+
seen.add(target);
|
|
571
872
|
const linksBack = target.page.hreflang.some((h) => h.url && key(h.url) === key(p.url));
|
|
572
873
|
if (!linksBack) {
|
|
573
874
|
missingReturn.push({ page: p.url, alternate: alt.url });
|
|
875
|
+
returnLinks.push(returnLinkEdit(target, p, key));
|
|
574
876
|
}
|
|
575
877
|
}
|
|
576
878
|
}
|
|
577
879
|
if (missingReturn.length) {
|
|
578
880
|
out.push(finding(m("site.hreflang-no-return-link"), `${missingReturn.length} hreflang alternate(s) do not link back.`, {
|
|
881
|
+
fix: returnLinks.slice(0, MAX_EDITS),
|
|
579
882
|
evidence: {
|
|
580
883
|
count: missingReturn.length,
|
|
581
884
|
examples: cap(missingReturn),
|
|
@@ -584,6 +887,8 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
|
|
|
584
887
|
}
|
|
585
888
|
if (ctx.crawl.hitPageLimit) {
|
|
586
889
|
out.push(finding(m("site.crawl-limit-reached"), `Crawl stopped after ${ctx.crawl.pagesCrawled} pages (limit ${ctx.crawl.maxPages}); ${ctx.crawl.pagesDiscovered} URLs were discovered.`, {
|
|
890
|
+
// About the audit, not the site: the change is a larger --max-pages, not an edit.
|
|
891
|
+
fix: [],
|
|
587
892
|
evidence: {
|
|
588
893
|
pagesCrawled: ctx.crawl.pagesCrawled,
|
|
589
894
|
pagesDiscovered: ctx.crawl.pagesDiscovered,
|
|
@@ -593,8 +898,39 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
|
|
|
593
898
|
}
|
|
594
899
|
return out;
|
|
595
900
|
};
|
|
901
|
+
/** The canonical a page should name instead of a broken, redirecting, noindex or chained one. */
|
|
902
|
+
const canonicalRepair = (scope, p, problem) => {
|
|
903
|
+
if (problem.kind === "redirects" || problem.kind === "canonical") {
|
|
904
|
+
return canonicalEdit(p, problem.to ?? p.url);
|
|
905
|
+
}
|
|
906
|
+
if (problem.kind === "noindex") {
|
|
907
|
+
return canonicalEdit(p, p.url, "The page's own URL. If the noindex page is the one meant to rank, remove its noindex and keep this canonical instead.");
|
|
908
|
+
}
|
|
909
|
+
const live = p.page.canonical ? liveSpelling(scope, p.page.canonical) : null;
|
|
910
|
+
return canonicalEdit(p, live ?? p.url, live
|
|
911
|
+
? undefined
|
|
912
|
+
: "The page's own URL. If another page is the preferred one, use its live URL.");
|
|
913
|
+
};
|
|
596
914
|
/** What is wrong with a URL that metadata (a canonical, an hreflang link) points to, or null. */
|
|
597
|
-
|
|
915
|
+
/**
|
|
916
|
+
* Where a chain of canonicals ends: the first page that names itself (or nothing), or the last URL
|
|
917
|
+
* before the chain loops or leaves the crawled pages.
|
|
918
|
+
*/
|
|
919
|
+
const finalCanonical = ({ key, pagesByKey }, url) => {
|
|
920
|
+
let current = url;
|
|
921
|
+
const seen = new Set([key(url)]);
|
|
922
|
+
for (let hops = 0; hops < 10; hops += 1) {
|
|
923
|
+
const next = pagesByKey.get(key(current))?.page.canonical;
|
|
924
|
+
if (!next || seen.has(key(next))) {
|
|
925
|
+
return current;
|
|
926
|
+
}
|
|
927
|
+
seen.add(key(next));
|
|
928
|
+
current = next;
|
|
929
|
+
}
|
|
930
|
+
return current;
|
|
931
|
+
};
|
|
932
|
+
const targetProblem = (scope, url) => {
|
|
933
|
+
const { key, pagesByKey, checksByKey, isIndexable } = scope;
|
|
598
934
|
const check = checksByKey.get(key(url));
|
|
599
935
|
if (check && (check.error || (check.status ?? 0) >= 400)) {
|
|
600
936
|
return {
|
|
@@ -603,7 +939,11 @@ const targetProblem = ({ key, pagesByKey, checksByKey, isIndexable }, url) => {
|
|
|
603
939
|
};
|
|
604
940
|
}
|
|
605
941
|
if (check && check.redirectChain.length > 0) {
|
|
606
|
-
return {
|
|
942
|
+
return {
|
|
943
|
+
kind: "redirects",
|
|
944
|
+
detail: `redirects to ${check.finalUrl}`,
|
|
945
|
+
to: check.finalUrl,
|
|
946
|
+
};
|
|
607
947
|
}
|
|
608
948
|
const target = pagesByKey.get(key(url));
|
|
609
949
|
if (!target) {
|
|
@@ -614,7 +954,11 @@ const targetProblem = ({ key, pagesByKey, checksByKey, isIndexable }, url) => {
|
|
|
614
954
|
}
|
|
615
955
|
const { canonical } = target.page;
|
|
616
956
|
return canonical && key(canonical) !== key(target.url)
|
|
617
|
-
? {
|
|
957
|
+
? {
|
|
958
|
+
kind: "canonical",
|
|
959
|
+
detail: `names ${canonical} as canonical`,
|
|
960
|
+
to: finalCanonical(scope, canonical),
|
|
961
|
+
}
|
|
618
962
|
: null;
|
|
619
963
|
};
|
|
620
964
|
/** Canonicals that point at broken, redirecting, noindex or re-canonicalized URLs. */
|
|
@@ -626,6 +970,12 @@ const canonicalGraphFindings = (scope) => {
|
|
|
626
970
|
noindex: [],
|
|
627
971
|
canonical: [],
|
|
628
972
|
};
|
|
973
|
+
const repairs = {
|
|
974
|
+
broken: [],
|
|
975
|
+
redirects: [],
|
|
976
|
+
noindex: [],
|
|
977
|
+
canonical: [],
|
|
978
|
+
};
|
|
629
979
|
for (const p of okPages) {
|
|
630
980
|
const { canonical } = p.page;
|
|
631
981
|
// A noindex page's canonical carries no weight; the noindex wins.
|
|
@@ -639,6 +989,7 @@ const canonicalGraphFindings = (scope) => {
|
|
|
639
989
|
canonical,
|
|
640
990
|
problem: problem.detail,
|
|
641
991
|
});
|
|
992
|
+
repairs[problem.kind].push(canonicalRepair(scope, p, problem));
|
|
642
993
|
}
|
|
643
994
|
}
|
|
644
995
|
const rules = [
|
|
@@ -668,6 +1019,7 @@ const canonicalGraphFindings = (scope) => {
|
|
|
668
1019
|
return list.length > 0
|
|
669
1020
|
? [
|
|
670
1021
|
finding(m(id), `${list.length} page(s) ${label}.`, {
|
|
1022
|
+
fix: repairs[kind].slice(0, MAX_EDITS),
|
|
671
1023
|
evidence: { count: list.length, examples: cap(list) },
|
|
672
1024
|
}),
|
|
673
1025
|
]
|
|
@@ -685,7 +1037,7 @@ const isUtilityPage = (url) => {
|
|
|
685
1037
|
}
|
|
686
1038
|
};
|
|
687
1039
|
/** Noindex pages that other pages of the site link to, most linked first. */
|
|
688
|
-
const noindexLinkFindings = ({ ctx, key, okPages, isIndexable, }) => {
|
|
1040
|
+
const noindexLinkFindings = ({ ctx, key, pagesByKey, okPages, isIndexable, }) => {
|
|
689
1041
|
const linkedFrom = new Map();
|
|
690
1042
|
for (const [target, sources] of ctx.linkSources) {
|
|
691
1043
|
const k = key(target);
|
|
@@ -707,15 +1059,42 @@ const noindexLinkFindings = ({ ctx, key, okPages, isIndexable, }) => {
|
|
|
707
1059
|
.toSorted((a, b) => b.linkedFrom - a.linkedFrom);
|
|
708
1060
|
return linked.length > 0
|
|
709
1061
|
? [
|
|
710
|
-
finding(m("site.links-to-noindex"), `${linked.length} noindex page(s) are linked from other pages; the most linked: ${linked[0]?.url} (${linked[0]?.linkedFrom} pages).`, {
|
|
1062
|
+
finding(m("site.links-to-noindex"), `${linked.length} noindex page(s) are linked from other pages; the most linked: ${linked[0]?.url} (${linked[0]?.linkedFrom} pages).`, {
|
|
1063
|
+
fix: linked
|
|
1064
|
+
.flatMap((l) => {
|
|
1065
|
+
const record = pagesByKey.get(key(l.url));
|
|
1066
|
+
return record
|
|
1067
|
+
? removeNoindexEdits(record, `Only if ${l.url} belongs in search results; leave the noindex on a page kept out on purpose.`)
|
|
1068
|
+
: [];
|
|
1069
|
+
})
|
|
1070
|
+
.slice(0, MAX_EDITS),
|
|
1071
|
+
evidence: { count: linked.length, pages: cap(linked) },
|
|
1072
|
+
}),
|
|
711
1073
|
]
|
|
712
1074
|
: [];
|
|
713
1075
|
};
|
|
1076
|
+
/**
|
|
1077
|
+
* Points an hreflang link at the canonical, live URL its alternate leads to, or removes it when the
|
|
1078
|
+
* alternate is broken or noindex: an annotation to such a page is ignored anyway.
|
|
1079
|
+
*/
|
|
1080
|
+
const hreflangRepair = (p, alt, problem) => {
|
|
1081
|
+
const at = `link[rel="alternate"][hreflang="${alt.lang}"]`;
|
|
1082
|
+
if ((problem.kind === "redirects" || problem.kind === "canonical") &&
|
|
1083
|
+
problem.to) {
|
|
1084
|
+
return onUrl(setAttributeEdit(alt.tag, at, "href", escapeAttribute(problem.to)), locationOf(p));
|
|
1085
|
+
}
|
|
1086
|
+
return removeEdit("html", alt.tag, {
|
|
1087
|
+
url: locationOf(p),
|
|
1088
|
+
at,
|
|
1089
|
+
write: `${alt.url} ${problem.detail}. If that language version should exist, list its live, indexable URL here instead.`,
|
|
1090
|
+
});
|
|
1091
|
+
};
|
|
714
1092
|
/** hreflang alternates that are broken, redirect, are noindex or canonicalize elsewhere. */
|
|
715
1093
|
const hreflangTargetFindings = (scope) => {
|
|
716
1094
|
const { key, okPages } = scope;
|
|
717
1095
|
const invalid = [];
|
|
718
1096
|
const seen = new Set();
|
|
1097
|
+
const fix = [];
|
|
719
1098
|
for (const p of okPages) {
|
|
720
1099
|
for (const alt of p.page.hreflang) {
|
|
721
1100
|
if (!alt.url || key(alt.url) === key(p.url)) {
|
|
@@ -733,26 +1112,19 @@ const hreflangTargetFindings = (scope) => {
|
|
|
733
1112
|
alternate: alt.url,
|
|
734
1113
|
problem: problem.detail,
|
|
735
1114
|
});
|
|
1115
|
+
fix.push(hreflangRepair(p, alt, problem));
|
|
736
1116
|
}
|
|
737
1117
|
}
|
|
738
1118
|
}
|
|
739
1119
|
return invalid.length > 0
|
|
740
1120
|
? [
|
|
741
|
-
finding(m("site.hreflang-target-invalid"), `${invalid.length} hreflang alternate(s) point to URLs that are broken, redirect, are noindex or are not canonical.`, {
|
|
1121
|
+
finding(m("site.hreflang-target-invalid"), `${invalid.length} hreflang alternate(s) point to URLs that are broken, redirect, are noindex or are not canonical.`, {
|
|
1122
|
+
fix: fix.slice(0, MAX_EDITS),
|
|
1123
|
+
evidence: { count: invalid.length, examples: cap(invalid) },
|
|
1124
|
+
}),
|
|
742
1125
|
]
|
|
743
1126
|
: [];
|
|
744
1127
|
};
|
|
745
|
-
/** The URL with its path lower-cased and trailing slashes dropped: one key for every spelling. */
|
|
746
|
-
const spellingKey = (url) => {
|
|
747
|
-
try {
|
|
748
|
-
const u = new URL(url);
|
|
749
|
-
const pathname = trimTrailingSlashes(u.pathname.toLowerCase()) || "/";
|
|
750
|
-
return `${u.origin}${pathname}${u.search}`;
|
|
751
|
-
}
|
|
752
|
-
catch {
|
|
753
|
-
return url;
|
|
754
|
-
}
|
|
755
|
-
};
|
|
756
1128
|
const pathnameOf = (url) => {
|
|
757
1129
|
try {
|
|
758
1130
|
return new URL(url).pathname;
|
|
@@ -805,8 +1177,19 @@ const urlVariantFindings = (scope) => {
|
|
|
805
1177
|
return [];
|
|
806
1178
|
}
|
|
807
1179
|
const kinds = [...new Set(duplicates.map((d) => d.kind))];
|
|
1180
|
+
const fix = duplicates.slice(0, MAX_EDITS).map((d) => ({
|
|
1181
|
+
in: "http",
|
|
1182
|
+
action: "replace",
|
|
1183
|
+
url: d.variant,
|
|
1184
|
+
current: statusLine(d.status ?? 200),
|
|
1185
|
+
code: redirectTo(d.url),
|
|
1186
|
+
write: d.kind === "trailing-slash"
|
|
1187
|
+
? "For every URL of the site, to the one spelling with or without the trailing slash: the trailingSlash setting in Next.js, Nuxt or Astro, or a redirect rule at the server or CDN."
|
|
1188
|
+
: "For every URL spelled in another case, to the spelling the site links to: a redirect rule at the server or CDN.",
|
|
1189
|
+
}));
|
|
808
1190
|
return [
|
|
809
1191
|
finding(m("site.url-variant-duplicate"), `${duplicates.length} page(s) also answer at a ${kinds.join(" and a ")} variant of their URL.`, {
|
|
1192
|
+
fix,
|
|
810
1193
|
evidence: {
|
|
811
1194
|
count: duplicates.length,
|
|
812
1195
|
examples: cap(duplicates.map(({ kind, url, variant, canonical }) => ({
|
|
@@ -829,7 +1212,16 @@ const withoutQuery = (url) => {
|
|
|
829
1212
|
}
|
|
830
1213
|
};
|
|
831
1214
|
/** Pages that differ only in their query string, show the same text and do not consolidate. */
|
|
832
|
-
|
|
1215
|
+
/** Canonicals from each parameter version to the URL without the parameter, or the first URL. */
|
|
1216
|
+
const parameterCanonicals = ({ key, pagesByKey }, group) => {
|
|
1217
|
+
const target = group.urls.find((u) => !u.includes("?")) ?? group.urls[0] ?? group.path;
|
|
1218
|
+
return group.urls.flatMap((u) => {
|
|
1219
|
+
const record = key(u) === key(target) ? undefined : pagesByKey.get(key(u));
|
|
1220
|
+
return record ? [canonicalEdit(record, target)] : [];
|
|
1221
|
+
});
|
|
1222
|
+
};
|
|
1223
|
+
const parameterDuplicateFindings = (scope) => {
|
|
1224
|
+
const { key, okPages, isIndexable } = scope;
|
|
833
1225
|
const byPath = new Map();
|
|
834
1226
|
for (const p of okPages) {
|
|
835
1227
|
const { canonical } = p.page;
|
|
@@ -867,12 +1259,41 @@ const parameterDuplicateFindings = ({ key, okPages, isIndexable, }) => {
|
|
|
867
1259
|
}
|
|
868
1260
|
return groups.length > 0
|
|
869
1261
|
? [
|
|
870
|
-
finding(m("site.parameter-duplicates"), `${groups.length} path(s) are crawled under several query strings with the same text.`, {
|
|
1262
|
+
finding(m("site.parameter-duplicates"), `${groups.length} path(s) are crawled under several query strings with the same text.`, {
|
|
1263
|
+
fix: groups
|
|
1264
|
+
.flatMap((g) => parameterCanonicals(scope, g))
|
|
1265
|
+
.slice(0, MAX_EDITS),
|
|
1266
|
+
evidence: { count: groups.length, groups: cap(groups) },
|
|
1267
|
+
}),
|
|
871
1268
|
]
|
|
872
1269
|
: [];
|
|
873
1270
|
};
|
|
1271
|
+
/** Points each <img> that shows a broken image at a slot for the image's working URL. */
|
|
1272
|
+
const brokenImageEdits = ({ key, pagesByKey }, image, pages) => {
|
|
1273
|
+
const write = `The URL the image file is deployed at; or upload the missing file to ${image.url}.`;
|
|
1274
|
+
const edits = [...pages].flatMap((u) => {
|
|
1275
|
+
const record = pagesByKey.get(key(u));
|
|
1276
|
+
return record
|
|
1277
|
+
? imagesOf(record, image.url).map((img) => onUrl(setAttributeEdit(img.tag, "img", "src", slot("working URL of the image"), write), locationOf(record)))
|
|
1278
|
+
: [];
|
|
1279
|
+
});
|
|
1280
|
+
// An image from srcset or a <picture> source has no <img src> to quote: upload the file instead.
|
|
1281
|
+
return edits.length > 0
|
|
1282
|
+
? edits
|
|
1283
|
+
: [
|
|
1284
|
+
{
|
|
1285
|
+
in: "asset",
|
|
1286
|
+
action: "add",
|
|
1287
|
+
url: image.url,
|
|
1288
|
+
at: "new file",
|
|
1289
|
+
code: slot("the image file"),
|
|
1290
|
+
write: `Upload the image the pages show here, or change the pages that point to ${image.url}.`,
|
|
1291
|
+
},
|
|
1292
|
+
];
|
|
1293
|
+
};
|
|
874
1294
|
/** Images that answer 4xx/5xx or do not load, with the pages that show them. */
|
|
875
|
-
const brokenImageFindings = (
|
|
1295
|
+
const brokenImageFindings = (scope) => {
|
|
1296
|
+
const { ctx } = scope;
|
|
876
1297
|
const broken = [];
|
|
877
1298
|
for (const [url, check] of ctx.imageChecks ?? []) {
|
|
878
1299
|
if (check.error || (check.status ?? 0) >= 400) {
|
|
@@ -889,7 +1310,12 @@ const brokenImageFindings = (ctx) => {
|
|
|
889
1310
|
}
|
|
890
1311
|
return broken.length > 0
|
|
891
1312
|
? [
|
|
892
|
-
finding(m("site.broken-images"), `${broken.length} image URL(s) are broken.`, {
|
|
1313
|
+
finding(m("site.broken-images"), `${broken.length} image URL(s) are broken.`, {
|
|
1314
|
+
fix: broken
|
|
1315
|
+
.flatMap((b) => brokenImageEdits(scope, b, ctx.imageSources?.get(b.url) ?? []))
|
|
1316
|
+
.slice(0, MAX_EDITS),
|
|
1317
|
+
evidence: { count: broken.length, examples: cap(broken, 20) },
|
|
1318
|
+
}),
|
|
893
1319
|
]
|
|
894
1320
|
: [];
|
|
895
1321
|
};
|
|
@@ -913,20 +1339,21 @@ export const runSiteRules = (ctx) => {
|
|
|
913
1339
|
okPages: ctx.pages.filter((p) => (p.response?.status ?? 200) < 400),
|
|
914
1340
|
isIndexable: (p) => !robotsDirectives(p.page, p.response).noindex &&
|
|
915
1341
|
(p.response?.status ?? 200) < 400,
|
|
1342
|
+
linksTo: (p, targetKey) => linksTo(p, targetKey, key),
|
|
916
1343
|
};
|
|
917
1344
|
return [
|
|
918
|
-
...robotsFindings(
|
|
1345
|
+
...robotsFindings(scope),
|
|
919
1346
|
...robotsContentFindings(ctx),
|
|
920
1347
|
...blockedResourceFindings(scope),
|
|
921
1348
|
...sitemapFindings(scope),
|
|
922
1349
|
...sitemapProtocolFindings(ctx),
|
|
923
|
-
...linkHealthFindings(
|
|
924
|
-
...unreachedFindings(
|
|
1350
|
+
...linkHealthFindings(scope),
|
|
1351
|
+
...unreachedFindings(scope),
|
|
925
1352
|
...duplicateFindings(scope),
|
|
926
1353
|
...architectureFindings(scope),
|
|
927
1354
|
...urlVariantFindings(scope),
|
|
928
1355
|
...parameterDuplicateFindings(scope),
|
|
929
|
-
...brokenImageFindings(
|
|
1356
|
+
...brokenImageFindings(scope),
|
|
930
1357
|
...canonicalGraphFindings(scope),
|
|
931
1358
|
...noindexLinkFindings(scope),
|
|
932
1359
|
...hreflangTargetFindings(scope),
|