@rivi-software/agentic-seo-check 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -7
- package/dist/cli.js +28 -0
- package/dist/cli.js.map +1 -1
- package/dist/core/aeo/audit.d.ts +8 -0
- package/dist/core/aeo/audit.js +101 -17
- package/dist/core/aeo/audit.js.map +1 -1
- package/dist/core/aeo/bots.d.ts +48 -0
- package/dist/core/aeo/bots.js +151 -5
- package/dist/core/aeo/bots.js.map +1 -1
- package/dist/core/aeo/fixes.d.ts +38 -0
- package/dist/core/aeo/fixes.js +111 -0
- package/dist/core/aeo/fixes.js.map +1 -0
- package/dist/core/aeo/site-rules.d.ts +5 -0
- package/dist/core/aeo/site-rules.js +293 -10
- package/dist/core/aeo/site-rules.js.map +1 -1
- package/dist/core/audit.d.ts +8 -0
- package/dist/core/audit.js +144 -25
- package/dist/core/audit.js.map +1 -1
- package/dist/core/crawl.js +43 -4
- package/dist/core/crawl.js.map +1 -1
- package/dist/core/html/answer.d.ts +5 -2
- package/dist/core/html/answer.js +25 -7
- package/dist/core/html/answer.js.map +1 -1
- package/dist/core/html/dom.d.ts +13 -1
- package/dist/core/html/dom.js +30 -2
- package/dist/core/html/dom.js.map +1 -1
- package/dist/core/html/extract.js +122 -30
- package/dist/core/html/extract.js.map +1 -1
- package/dist/core/html/head.js +114 -19
- package/dist/core/html/head.js.map +1 -1
- package/dist/core/html/resources.js +15 -7
- package/dist/core/html/resources.js.map +1 -1
- package/dist/core/html/structured-data.js +21 -5
- package/dist/core/html/structured-data.js.map +1 -1
- package/dist/core/lighthouse.d.ts +5 -6
- package/dist/core/lighthouse.js +297 -7
- package/dist/core/lighthouse.js.map +1 -1
- package/dist/core/performance-fixes.d.ts +25 -0
- package/dist/core/performance-fixes.js +446 -0
- package/dist/core/performance-fixes.js.map +1 -0
- package/dist/core/performance-rules.js +56 -8
- package/dist/core/performance-rules.js.map +1 -1
- package/dist/core/recheck.d.ts +68 -0
- package/dist/core/recheck.js +247 -0
- package/dist/core/recheck.js.map +1 -0
- package/dist/core/render-fixes.d.ts +27 -0
- package/dist/core/render-fixes.js +164 -0
- package/dist/core/render-fixes.js.map +1 -0
- package/dist/core/render-rules.js +60 -21
- package/dist/core/render-rules.js.map +1 -1
- package/dist/core/render.d.ts +9 -1
- package/dist/core/render.js +36 -2
- package/dist/core/render.js.map +1 -1
- package/dist/core/report/markdown.js +68 -1
- package/dist/core/report/markdown.js.map +1 -1
- package/dist/core/robots.d.ts +5 -0
- package/dist/core/robots.js +8 -0
- package/dist/core/robots.js.map +1 -1
- package/dist/core/rules/aeo.js +339 -25
- package/dist/core/rules/aeo.js.map +1 -1
- package/dist/core/rules/canonical.js +107 -11
- package/dist/core/rules/canonical.js.map +1 -1
- package/dist/core/rules/content.js +65 -7
- package/dist/core/rules/content.js.map +1 -1
- package/dist/core/rules/document.js +73 -8
- package/dist/core/rules/document.js.map +1 -1
- package/dist/core/rules/fix.d.ts +55 -0
- package/dist/core/rules/fix.js +159 -0
- package/dist/core/rules/fix.js.map +1 -0
- package/dist/core/rules/head.js +81 -4
- package/dist/core/rules/head.js.map +1 -1
- package/dist/core/rules/headings.js +92 -9
- package/dist/core/rules/headings.js.map +1 -1
- package/dist/core/rules/hreflang.js +79 -9
- package/dist/core/rules/hreflang.js.map +1 -1
- package/dist/core/rules/images.js +49 -0
- package/dist/core/rules/images.js.map +1 -1
- package/dist/core/rules/index.d.ts +11 -0
- package/dist/core/rules/index.js +21 -1
- package/dist/core/rules/index.js.map +1 -1
- package/dist/core/rules/indexing.js +108 -1
- package/dist/core/rules/indexing.js.map +1 -1
- package/dist/core/rules/keyword.js +91 -1
- package/dist/core/rules/keyword.js.map +1 -1
- package/dist/core/rules/links.js +42 -0
- package/dist/core/rules/links.js.map +1 -1
- package/dist/core/rules/loading.js +54 -1
- package/dist/core/rules/loading.js.map +1 -1
- package/dist/core/rules/meta.js +36 -5
- package/dist/core/rules/meta.js.map +1 -1
- package/dist/core/rules/page-fixes.d.ts +39 -0
- package/dist/core/rules/page-fixes.js +107 -0
- package/dist/core/rules/page-fixes.js.map +1 -0
- package/dist/core/rules/quoted-tags.d.ts +23 -0
- package/dist/core/rules/quoted-tags.js +79 -0
- package/dist/core/rules/quoted-tags.js.map +1 -0
- package/dist/core/rules/response.js +71 -7
- package/dist/core/rules/response.js.map +1 -1
- package/dist/core/rules/rule.d.ts +17 -4
- package/dist/core/rules/rule.js +11 -2
- package/dist/core/rules/rule.js.map +1 -1
- package/dist/core/rules/social.js +101 -10
- package/dist/core/rules/social.js.map +1 -1
- package/dist/core/rules/structured-data-fixes.d.ts +33 -0
- package/dist/core/rules/structured-data-fixes.js +469 -0
- package/dist/core/rules/structured-data-fixes.js.map +1 -0
- package/dist/core/rules/structured-data-values.js +242 -31
- package/dist/core/rules/structured-data-values.js.map +1 -1
- package/dist/core/rules/structured-data.js +171 -4
- package/dist/core/rules/structured-data.js.map +1 -1
- package/dist/core/rules/title.js +39 -5
- package/dist/core/rules/title.js.map +1 -1
- package/dist/core/rules/urls.js +166 -0
- package/dist/core/rules/urls.js.map +1 -1
- package/dist/core/site-fixes.d.ts +111 -0
- package/dist/core/site-fixes.js +650 -0
- package/dist/core/site-fixes.js.map +1 -0
- package/dist/core/site-rules.js +474 -53
- package/dist/core/site-rules.js.map +1 -1
- package/dist/core/sitemap.d.ts +16 -0
- package/dist/core/sitemap.js +34 -0
- package/dist/core/sitemap.js.map +1 -1
- package/dist/core/types.d.ts +223 -3
- package/dist/core/types.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +16 -2
- package/dist/mcp.js.map +1 -1
- package/package.json +5 -7
- package/skills/seo-audit/SKILL.md +23 -6
package/dist/core/site-rules.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { normalizeForComparison, shingles, jaccard } from "./html/text.js";
|
|
2
2
|
import { agentMatcher } from "./robots.js";
|
|
3
|
+
import { escapeAttribute, removeEdit, setAttributeEdit, slot, } from "./rules/fix.js";
|
|
3
4
|
import { robotsDirectives } from "./rules/indexing.js";
|
|
4
5
|
import { finding, GOOGLE_DOCS, cap } from "./rules/rule.js";
|
|
6
|
+
import { addLinkEdit, addToSitemapFix, blockedResourcesFix, canonicalEdit, imagesOf, blockingRule, lastmodFix, locationOf, linkSelector, linksTo, nestedIndexFix, hrefLike, onUrl, otherHostFix, redirectTo, redescribeEdit, removeNoindexEdits, returnLinkEdit, reheadEdit, relinkEdit, linesOf, retitleEdit, statusLine, removeFromSitemap, replaceInSitemap, robotsBlocksAllFix, robotsErrorFix, robotsMissingFix, robotsTooLargeFix, robotsUnsupportedFix, MAX_EDITS, sitemapErrorFix, sitemapMissingFix, splitSitemapFix, } from "./site-fixes.js";
|
|
5
7
|
import { MIN_COMPARABLE_WORDS, SAME_TEXT } from "./site-probes.js";
|
|
8
|
+
import { sitemapLayout } from "./sitemap.js";
|
|
6
9
|
import { normalizeUrl, normalizePath, trimTrailingSlashes } from "./url.js";
|
|
7
10
|
const meta = (id, severity, description, recommendation, docs) => ({
|
|
8
11
|
id,
|
|
@@ -85,8 +88,21 @@ const pathKey = (url) => {
|
|
|
85
88
|
return url;
|
|
86
89
|
}
|
|
87
90
|
};
|
|
88
|
-
|
|
91
|
+
/** The URL with its path lower-cased and trailing slashes dropped: one key for every spelling. */
|
|
92
|
+
const spellingKey = (url) => {
|
|
93
|
+
try {
|
|
94
|
+
const u = new URL(url);
|
|
95
|
+
const pathname = trimTrailingSlashes(u.pathname.toLowerCase()) || "/";
|
|
96
|
+
return `${u.origin}${pathname}${u.search}`;
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return url;
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
const pushDuplicates = (out, rule, pages, pick, label, fixFor) => {
|
|
89
103
|
const groups = new Map();
|
|
104
|
+
// Every page but the first of a group gets a value of its own; the first keeps what it has.
|
|
105
|
+
const others = [];
|
|
90
106
|
// Pages are grouped by a lowercased key, but the evidence quotes the value as the first page
|
|
91
107
|
// wrote it, so a report or pull request shows the text a reviewer would find on the site.
|
|
92
108
|
const written = new Map();
|
|
@@ -97,7 +113,10 @@ const pushDuplicates = (out, rule, pages, pick, label) => {
|
|
|
97
113
|
}
|
|
98
114
|
const k = normalizeForComparison(value);
|
|
99
115
|
pushTo(groups, k, p.url);
|
|
100
|
-
if (
|
|
116
|
+
if (written.has(k)) {
|
|
117
|
+
others.push(p);
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
101
120
|
written.set(k, value);
|
|
102
121
|
}
|
|
103
122
|
}
|
|
@@ -107,6 +126,7 @@ const pushDuplicates = (out, rule, pages, pick, label) => {
|
|
|
107
126
|
}
|
|
108
127
|
const pagesAffected = dupes.reduce((n, [, urls]) => n + urls.length, 0);
|
|
109
128
|
out.push(finding(rule, `${dupes.length} ${label}(s) are shared by ${pagesAffected} pages.`, {
|
|
129
|
+
fix: others.flatMap((p) => fixFor(p) ?? []).slice(0, MAX_EDITS),
|
|
110
130
|
evidence: {
|
|
111
131
|
groups: cap(dupes.map(([k, urls]) => ({
|
|
112
132
|
value: (written.get(k) ?? k).slice(0, 120),
|
|
@@ -116,6 +136,34 @@ const pushDuplicates = (out, rule, pages, pick, label) => {
|
|
|
116
136
|
},
|
|
117
137
|
}));
|
|
118
138
|
};
|
|
139
|
+
/**
|
|
140
|
+
* The canonical each duplicate should name. The first page of a group is the shallowest the crawl
|
|
141
|
+
* reached: the others name it as canonical. Near-duplicates come in pairs, so a page can be in two
|
|
142
|
+
* groups (a~b and b~c, or a~c and b~c): each page gets one canonical, to a page that keeps its
|
|
143
|
+
* own, so the fix makes no chain and gives no page two.
|
|
144
|
+
*/
|
|
145
|
+
const duplicateCanonicalEdits = (groups, byUrl) => {
|
|
146
|
+
const canonicalOf = new Map();
|
|
147
|
+
const roots = new Set();
|
|
148
|
+
const fix = [];
|
|
149
|
+
for (const { pages: [first, ...rest], } of groups) {
|
|
150
|
+
const root = first ? (canonicalOf.get(first) ?? first) : undefined;
|
|
151
|
+
for (const u of rest) {
|
|
152
|
+
const record = byUrl.get(u);
|
|
153
|
+
if (!root ||
|
|
154
|
+
!record ||
|
|
155
|
+
u === root ||
|
|
156
|
+
canonicalOf.has(u) ||
|
|
157
|
+
roots.has(u)) {
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
canonicalOf.set(u, root);
|
|
161
|
+
roots.add(root);
|
|
162
|
+
fix.push(canonicalEdit(record, root, `If ${u} is meant to be a page of its own, make its text distinct instead.`));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return fix;
|
|
166
|
+
};
|
|
119
167
|
const pushDuplicateContent = (out, pages) => {
|
|
120
168
|
const candidates = pages.filter((p) => p.page.wordCount >= 50);
|
|
121
169
|
const groups = [];
|
|
@@ -156,7 +204,9 @@ const pushDuplicateContent = (out, pages) => {
|
|
|
156
204
|
}
|
|
157
205
|
}
|
|
158
206
|
if (groups.length) {
|
|
159
|
-
|
|
207
|
+
const byUrl = new Map(candidates.map((p) => [p.url, p]));
|
|
208
|
+
const fix = duplicateCanonicalEdits(groups, byUrl);
|
|
209
|
+
out.push(finding(m("site.duplicate-content"), `${groups.length} group(s) of pages have (near-)identical body text.`, { fix: fix.slice(0, MAX_EDITS), evidence: { groups: cap(groups) } }));
|
|
160
210
|
}
|
|
161
211
|
};
|
|
162
212
|
/** Google reads the first 500 KiB of robots.txt and ignores the rest. */
|
|
@@ -164,7 +214,7 @@ const ROBOTS_MAX_BYTES = 512_000;
|
|
|
164
214
|
const unsupportedRobotsLines = (text) => {
|
|
165
215
|
const retired = [];
|
|
166
216
|
const crawlDelay = [];
|
|
167
|
-
for (const [i, raw] of text
|
|
217
|
+
for (const [i, raw] of linesOf(text).slice(0, 50_000).entries()) {
|
|
168
218
|
const line = raw.replace(/#.*$/u, "").trim();
|
|
169
219
|
const field = /^(?<field>[a-z-]+)\s*:/iu
|
|
170
220
|
.exec(line)
|
|
@@ -182,10 +232,14 @@ const unsupportedRobotsLines = (text) => {
|
|
|
182
232
|
const robotsContentFindings = (ctx) => {
|
|
183
233
|
const out = [];
|
|
184
234
|
const bytes = ctx.robots?.bytes ?? 0;
|
|
235
|
+
const text = ctx.robotsText ?? ctx.robots?.text;
|
|
236
|
+
const robotsUrl = ctx.robots?.url ?? new URL("/robots.txt", ctx.startUrl).href;
|
|
185
237
|
if (bytes > ROBOTS_MAX_BYTES) {
|
|
186
|
-
out.push(finding(m("site.robots-txt-too-large"), `robots.txt is ${Math.round(bytes / 1024)} KiB; Google reads only the first 500 KiB.`, {
|
|
238
|
+
out.push(finding(m("site.robots-txt-too-large"), `robots.txt is ${Math.round(bytes / 1024)} KiB; Google reads only the first 500 KiB.`, {
|
|
239
|
+
fix: robotsTooLargeFix(robotsUrl, text ?? "", bytes),
|
|
240
|
+
evidence: { url: ctx.robots?.url, bytes },
|
|
241
|
+
}));
|
|
187
242
|
}
|
|
188
|
-
const text = ctx.robotsText ?? ctx.robots?.text;
|
|
189
243
|
if (!text) {
|
|
190
244
|
return out;
|
|
191
245
|
}
|
|
@@ -195,6 +249,7 @@ const robotsContentFindings = (ctx) => {
|
|
|
195
249
|
? `robots.txt has ${retired.length} noindex or nofollow line(s), which Google stopped reading in 2019.`
|
|
196
250
|
: `robots.txt sets crawl-delay, which Google ignores.`, {
|
|
197
251
|
severity: retired.length > 0 ? "warning" : "info",
|
|
252
|
+
fix: robotsUnsupportedFix(robotsUrl, text),
|
|
198
253
|
evidence: {
|
|
199
254
|
url: ctx.robots?.url,
|
|
200
255
|
retired: cap(retired),
|
|
@@ -252,28 +307,44 @@ const blockedResourceFindings = ({ ctx, okPages, isIndexable, }) => {
|
|
|
252
307
|
.toSorted((a, b) => b.pages - a.pages);
|
|
253
308
|
return list.length > 0
|
|
254
309
|
? [
|
|
255
|
-
finding(m("site.robots-blocks-resources"), `robots.txt keeps Googlebot from ${list.length} script(s) or stylesheet(s) the pages load.`, {
|
|
310
|
+
finding(m("site.robots-blocks-resources"), `robots.txt keeps Googlebot from ${list.length} script(s) or stylesheet(s) the pages load.`, {
|
|
311
|
+
fix: blockedResourcesFix(robots.url, text, list.map((r) => r.url)),
|
|
312
|
+
evidence: { count: list.length, examples: cap(list) },
|
|
313
|
+
}),
|
|
256
314
|
]
|
|
257
315
|
: [];
|
|
258
316
|
};
|
|
259
317
|
/** Sitemaps over the protocol's limits, nested indexes, bad lastmod values and other hosts. */
|
|
318
|
+
/** Parts of at most 45,000 URLs or 45 MB, and at least two. */
|
|
319
|
+
const partsFor = (urls, bytes, perPart) => Math.max(2, Math.ceil((urls ?? 0) / 45_000), Math.ceil((bytes ?? 0) / perPart));
|
|
260
320
|
const sitemapProtocolFindings = (ctx) => {
|
|
261
321
|
const { problems } = ctx.sitemaps;
|
|
322
|
+
const layout = sitemapLayout(problems);
|
|
323
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
262
324
|
const out = [];
|
|
263
325
|
const over = problems.oversized.filter((o) => o.overLimit);
|
|
264
326
|
const unread = problems.oversized.filter((o) => !o.overLimit);
|
|
265
327
|
if (over.length > 0) {
|
|
266
|
-
out.push(finding(m("site.sitemap-too-large"), `${over.length} sitemap(s) exceed 50,000 URLs or 50 MB.`, {
|
|
328
|
+
out.push(finding(m("site.sitemap-too-large"), `${over.length} sitemap(s) exceed 50,000 URLs or 50 MB.`, {
|
|
329
|
+
fix: over.map((o) => splitSitemapFix(o.sitemap, partsFor(o.urls, o.bytes, 45_000_000), "Serve this sitemap index where the large sitemap was, and move its URLs into the files it lists, each under 50,000 URLs and 50 MB.")),
|
|
330
|
+
key: "over-limit",
|
|
331
|
+
evidence: { sitemaps: cap(over) },
|
|
332
|
+
}));
|
|
267
333
|
}
|
|
268
334
|
if (unread.length > 0) {
|
|
269
335
|
out.push(finding(m("site.sitemap-too-large"), `${unread.length} sitemap(s) are larger than the 15 MB this audit reads, so their URLs were not checked.`, {
|
|
270
336
|
severity: "info",
|
|
271
337
|
recommendation: "Check that each stays under 50 MB and 50,000 URLs; smaller sitemaps (10,000 URLs or fewer) are also quicker for search engines to fetch again.",
|
|
338
|
+
fix: unread.map((o) => splitSitemapFix(o.sitemap, partsFor(o.urls, o.bytes, 10_000_000), "Serve this sitemap index where the large sitemap was, and move its URLs into the files it lists, 10,000 URLs or fewer each, so search engines (and this audit) can read each one quickly.")),
|
|
339
|
+
key: "unread",
|
|
272
340
|
evidence: { sitemaps: cap(unread) },
|
|
273
341
|
}));
|
|
274
342
|
}
|
|
275
343
|
if (problems.nestedIndexes.length > 0) {
|
|
276
|
-
out.push(finding(m("site.sitemap-index-nested"), `${problems.nestedIndexes.length} sitemap index(es) are listed by another sitemap index.`, {
|
|
344
|
+
out.push(finding(m("site.sitemap-index-nested"), `${problems.nestedIndexes.length} sitemap index(es) are listed by another sitemap index.`, {
|
|
345
|
+
fix: problems.nestedIndexes.map((n) => nestedIndexFix(layout, n)),
|
|
346
|
+
evidence: { indexes: cap(problems.nestedIndexes) },
|
|
347
|
+
}));
|
|
277
348
|
}
|
|
278
349
|
const examples = [
|
|
279
350
|
[
|
|
@@ -292,22 +363,69 @@ const sitemapProtocolFindings = (ctx) => {
|
|
|
292
363
|
"are on another host than the site",
|
|
293
364
|
],
|
|
294
365
|
];
|
|
366
|
+
const fixes = {
|
|
367
|
+
"site.sitemap-lastmod-invalid": (list) => lastmodFix(layout, list, "invalid", today),
|
|
368
|
+
"site.sitemap-lastmod-future": (list) => lastmodFix(layout, list, "future", today),
|
|
369
|
+
"site.sitemap-other-host": (list) => otherHostFix(layout, list, new URL(ctx.startUrl).origin),
|
|
370
|
+
};
|
|
295
371
|
for (const [id, list, label] of examples) {
|
|
296
372
|
if (list.count > 0) {
|
|
297
373
|
out.push(finding(m(id), `${list.count} sitemap URL(s) ${label}.`, {
|
|
374
|
+
fix: fixes[id](list.examples),
|
|
298
375
|
evidence: { count: list.count, examples: list.examples },
|
|
299
376
|
}));
|
|
300
377
|
}
|
|
301
378
|
}
|
|
302
379
|
return out;
|
|
303
380
|
};
|
|
304
|
-
|
|
381
|
+
/** The sitemap URL a robots.txt should name: the one found, else /sitemap.xml. */
|
|
382
|
+
const sitemapUrlOf = (ctx) => ctx.sitemaps.sources[0] ?? new URL("/sitemap.xml", ctx.startUrl).href;
|
|
383
|
+
/**
|
|
384
|
+
* Keeps blocked pages out of the sitemap and stops crawlers following links to them; unblocking
|
|
385
|
+
* them could expose pages kept private on purpose, so the write names that rule instead.
|
|
386
|
+
*/
|
|
387
|
+
const blockedPageEdits = (scope, blocked) => {
|
|
388
|
+
const { ctx, key, pagesByKey } = scope;
|
|
389
|
+
const k = key(blocked);
|
|
390
|
+
const rule = ctx.robots && ctx.robotsText
|
|
391
|
+
? blockingRule(ctx.robots.url, ctx.robotsText, blocked)
|
|
392
|
+
: null;
|
|
393
|
+
const unblock = `If the page belongs in search results, remove ${rule ? `"${rule}"` : "the rule that blocks it"} from robots.txt instead.`;
|
|
394
|
+
const listed = ctx.sitemaps.urls.find((u) => key(u) === k);
|
|
395
|
+
if (listed) {
|
|
396
|
+
return [
|
|
397
|
+
removeFromSitemap(sitemapLayout(ctx.sitemaps.problems), listed, `robots.txt blocks it, so search engines cannot crawl it. ${unblock}`),
|
|
398
|
+
];
|
|
399
|
+
}
|
|
400
|
+
const sources = [...ctx.linkSources].flatMap(([target, from]) => key(target) === k ? [...from] : []);
|
|
401
|
+
const edits = sources.flatMap((source) => {
|
|
402
|
+
const record = pagesByKey.get(key(source));
|
|
403
|
+
return record
|
|
404
|
+
? linksTo(record, k, key).map((link) => onUrl(setAttributeEdit(link.tag, linkSelector(link), "rel", link.rel ? `${link.rel} nofollow` : "nofollow", `So crawlers do not follow it to a page robots.txt blocks. ${unblock}`), locationOf(record)))
|
|
405
|
+
: [];
|
|
406
|
+
});
|
|
407
|
+
if (edits.length > 0 || !rule || !ctx.robots) {
|
|
408
|
+
return edits;
|
|
409
|
+
}
|
|
410
|
+
return [
|
|
411
|
+
{
|
|
412
|
+
in: "robots.txt",
|
|
413
|
+
action: "remove",
|
|
414
|
+
url: ctx.robots.url,
|
|
415
|
+
current: rule,
|
|
416
|
+
write: `Only if ${blocked} belongs in search results; if it is kept out on purpose, leave robots.txt as it is.`,
|
|
417
|
+
},
|
|
418
|
+
];
|
|
419
|
+
};
|
|
420
|
+
const robotsFindings = (scope) => {
|
|
421
|
+
const { ctx } = scope;
|
|
305
422
|
const out = [];
|
|
306
423
|
/* robots.txt */
|
|
307
424
|
if (ctx.robots) {
|
|
308
425
|
if (ctx.robots.error ||
|
|
309
426
|
(ctx.robots.status !== null && ctx.robots.status >= 500)) {
|
|
310
427
|
out.push(finding(m("site.robots-txt-error"), `robots.txt at ${ctx.robots.url} could not be read (${ctx.robots.error ?? `HTTP ${ctx.robots.status}`}).`, {
|
|
428
|
+
fix: robotsErrorFix(ctx.robots.url, ctx.robots.status, ctx.robots.error),
|
|
311
429
|
evidence: {
|
|
312
430
|
url: ctx.robots.url,
|
|
313
431
|
status: ctx.robots.status,
|
|
@@ -318,11 +436,13 @@ const robotsFindings = (ctx) => {
|
|
|
318
436
|
else if (!ctx.robots.found) {
|
|
319
437
|
out.push(finding(m("site.robots-txt-missing"), `No robots.txt at ${ctx.robots.url}.`, {
|
|
320
438
|
severity: ctx.source === "directory" ? "info" : "warning",
|
|
439
|
+
fix: robotsMissingFix(ctx.robots.url, sitemapUrlOf(ctx)),
|
|
321
440
|
evidence: { url: ctx.robots.url, status: ctx.robots.status },
|
|
322
441
|
}));
|
|
323
442
|
}
|
|
324
443
|
else if (ctx.robotsAllows && !ctx.robotsAllows(ctx.startUrl)) {
|
|
325
444
|
out.push(finding(m("site.robots-txt-blocks-all"), "robots.txt blocks the start URL for the crawler user agent.", {
|
|
445
|
+
fix: robotsBlocksAllFix(ctx.robots.url, ctx.robotsText ?? ctx.robots.text ?? "", ctx.startUrl),
|
|
326
446
|
evidence: {
|
|
327
447
|
url: ctx.robots.url,
|
|
328
448
|
robotsTxt: ctx.robots.text?.slice(0, 500),
|
|
@@ -332,6 +452,10 @@ const robotsFindings = (ctx) => {
|
|
|
332
452
|
}
|
|
333
453
|
if (ctx.blockedByRobots.length) {
|
|
334
454
|
out.push(finding(m("site.robots-blocked-pages"), `${ctx.blockedByRobots.length} linked/sitemap URL(s) are disallowed by robots.txt.`, {
|
|
455
|
+
fix: ctx.blockedByRobots
|
|
456
|
+
.slice(0, MAX_EDITS)
|
|
457
|
+
.flatMap((u) => blockedPageEdits(scope, u))
|
|
458
|
+
.slice(0, MAX_EDITS),
|
|
335
459
|
evidence: {
|
|
336
460
|
count: ctx.blockedByRobots.length,
|
|
337
461
|
examples: cap(ctx.blockedByRobots),
|
|
@@ -343,6 +467,12 @@ const robotsFindings = (ctx) => {
|
|
|
343
467
|
/** Noindex, non-canonical, broken and redirecting URLs listed in the sitemap. */
|
|
344
468
|
const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
|
|
345
469
|
const out = [];
|
|
470
|
+
const layout = sitemapLayout(ctx.sitemaps.problems);
|
|
471
|
+
const listed = new Set(ctx.sitemaps.urls.map(key));
|
|
472
|
+
/** Points an entry at `to`, or removes it when the sitemap lists `to` already. */
|
|
473
|
+
const repoint = (loc, to) => listed.has(key(to))
|
|
474
|
+
? removeFromSitemap(layout, loc, `the sitemap lists ${to} already.`)
|
|
475
|
+
: replaceInSitemap(layout, loc, to);
|
|
346
476
|
const noindex = [];
|
|
347
477
|
const nonCanonical = [];
|
|
348
478
|
const broken = [];
|
|
@@ -370,10 +500,14 @@ const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
|
|
|
370
500
|
}
|
|
371
501
|
}
|
|
372
502
|
if (noindex.length) {
|
|
373
|
-
out.push(finding(m("site.sitemap-noindex-page"), `${noindex.length} sitemap URL(s) are noindex.`, {
|
|
503
|
+
out.push(finding(m("site.sitemap-noindex-page"), `${noindex.length} sitemap URL(s) are noindex.`, {
|
|
504
|
+
fix: noindex.map((u) => removeFromSitemap(layout, u, "the page is noindex, which may be on purpose. If it belongs in search results, remove its noindex instead.")),
|
|
505
|
+
evidence: { count: noindex.length, examples: cap(noindex) },
|
|
506
|
+
}));
|
|
374
507
|
}
|
|
375
508
|
if (nonCanonical.length) {
|
|
376
509
|
out.push(finding(m("site.sitemap-non-canonical"), `${nonCanonical.length} sitemap URL(s) canonicalise to another URL.`, {
|
|
510
|
+
fix: nonCanonical.map((n) => repoint(n.url, n.canonical)),
|
|
377
511
|
evidence: {
|
|
378
512
|
count: nonCanonical.length,
|
|
379
513
|
examples: cap(nonCanonical),
|
|
@@ -381,18 +515,36 @@ const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
|
|
|
381
515
|
}));
|
|
382
516
|
}
|
|
383
517
|
if (broken.length) {
|
|
384
|
-
out.push(finding(m("site.sitemap-broken-url"), `${broken.length} sitemap URL(s) return errors.`, {
|
|
518
|
+
out.push(finding(m("site.sitemap-broken-url"), `${broken.length} sitemap URL(s) return errors.`, {
|
|
519
|
+
fix: broken.map((b) => removeFromSitemap(layout, b.url, `it answers ${b.status === null ? "with no response" : `HTTP ${b.status}`}. If the page should exist, restore it instead.`)),
|
|
520
|
+
evidence: { count: broken.length, examples: cap(broken) },
|
|
521
|
+
}));
|
|
385
522
|
}
|
|
386
523
|
if (redirecting.length) {
|
|
387
524
|
out.push(finding(m("site.sitemap-redirecting-url"), `${redirecting.length} sitemap URL(s) redirect.`, {
|
|
525
|
+
fix: redirecting.map((r) => repoint(r.url, r.finalUrl)),
|
|
388
526
|
evidence: { count: redirecting.length, examples: cap(redirecting) },
|
|
389
527
|
}));
|
|
390
528
|
}
|
|
391
529
|
return out;
|
|
392
530
|
};
|
|
393
531
|
/** Indexable pages missing from the sitemap, and sitemap URLs nothing links to. */
|
|
394
|
-
|
|
532
|
+
/**
|
|
533
|
+
* Links an orphan from the home page, or takes it out of the sitemap when it does not load: a link
|
|
534
|
+
* to a broken page would only add a broken link.
|
|
535
|
+
*/
|
|
536
|
+
const orphanEdit = (scope, home, url) => {
|
|
537
|
+
const check = scope.checksByKey.get(scope.key(url));
|
|
538
|
+
if (check && (check.error || (check.status ?? 0) >= 400)) {
|
|
539
|
+
return removeFromSitemap(sitemapLayout(scope.ctx.sitemaps.problems), url, `it answers ${check.status === null ? "with no response" : `HTTP ${check.status}`}, and no page links to it.`);
|
|
540
|
+
}
|
|
541
|
+
return addLinkEdit(home, url, `Words that say what ${url} is about. Put the link on the page closest in topic, or in the navigation; if the page is obsolete, remove it from the sitemap instead.`);
|
|
542
|
+
};
|
|
543
|
+
const sitemapCoverageFindings = (scope) => {
|
|
544
|
+
const { ctx, key, pagesByKey, okPages, isIndexable } = scope;
|
|
395
545
|
const out = [];
|
|
546
|
+
const start = pagesByKey.get(key(ctx.startUrl));
|
|
547
|
+
const home = start ? locationOf(start) : ctx.startUrl;
|
|
396
548
|
const sitemapKeys = new Set(ctx.sitemaps.urls.map(key));
|
|
397
549
|
const notInSitemap = okPages
|
|
398
550
|
.filter((p) => isIndexable(p) &&
|
|
@@ -401,6 +553,7 @@ const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
|
|
|
401
553
|
.map((p) => p.url);
|
|
402
554
|
if (notInSitemap.length) {
|
|
403
555
|
out.push(finding(m("site.not-in-sitemap"), `${notInSitemap.length} crawled indexable page(s) are missing from the sitemap.`, {
|
|
556
|
+
fix: addToSitemapFix(sitemapLayout(ctx.sitemaps.problems), notInSitemap),
|
|
404
557
|
evidence: {
|
|
405
558
|
count: notInSitemap.length,
|
|
406
559
|
examples: cap(notInSitemap),
|
|
@@ -415,7 +568,12 @@ const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
|
|
|
415
568
|
linked.add(key(ctx.startUrl));
|
|
416
569
|
const orphans = ctx.sitemaps.urls.filter((u) => !linked.has(key(u)));
|
|
417
570
|
if (orphans.length) {
|
|
418
|
-
out.push(finding(m("site.orphan-pages"), `${orphans.length} sitemap URL(s) are not linked from any crawled page.`, {
|
|
571
|
+
out.push(finding(m("site.orphan-pages"), `${orphans.length} sitemap URL(s) are not linked from any crawled page.`, {
|
|
572
|
+
fix: orphans
|
|
573
|
+
.slice(0, MAX_EDITS)
|
|
574
|
+
.map((u) => orphanEdit(scope, home, u)),
|
|
575
|
+
evidence: { count: orphans.length, examples: cap(orphans) },
|
|
576
|
+
}));
|
|
419
577
|
}
|
|
420
578
|
}
|
|
421
579
|
return out;
|
|
@@ -424,7 +582,10 @@ const sitemapFindings = (scope) => {
|
|
|
424
582
|
const { ctx } = scope;
|
|
425
583
|
const out = [];
|
|
426
584
|
if (ctx.sitemaps.errors.length) {
|
|
427
|
-
out.push(finding(m("site.sitemap-error"), `${ctx.sitemaps.errors.length} sitemap problem(s).`, {
|
|
585
|
+
out.push(finding(m("site.sitemap-error"), `${ctx.sitemaps.errors.length} sitemap problem(s).`, {
|
|
586
|
+
fix: sitemapErrorFix(ctx.sitemaps.errors.slice(0, 10), new URL(ctx.startUrl).origin),
|
|
587
|
+
evidence: { errors: cap(ctx.sitemaps.errors) },
|
|
588
|
+
}));
|
|
428
589
|
}
|
|
429
590
|
// A sitemap too large to read was still found.
|
|
430
591
|
if (ctx.sitemaps.sources.length === 0 &&
|
|
@@ -432,6 +593,11 @@ const sitemapFindings = (scope) => {
|
|
|
432
593
|
ctx.sitemaps.problems.oversized.length === 0) {
|
|
433
594
|
out.push(finding(m("site.sitemap-missing"), "No sitemap was found.", {
|
|
434
595
|
severity: ctx.source === "directory" ? "info" : "warning",
|
|
596
|
+
fix: sitemapMissingFix(sitemapUrlOf(ctx), scope.okPages
|
|
597
|
+
.filter((p) => scope.isIndexable(p) &&
|
|
598
|
+
(!p.page.canonical ||
|
|
599
|
+
scope.key(p.page.canonical) === scope.key(p.url)))
|
|
600
|
+
.map((p) => p.url), ctx.robots?.found ? ctx.robots.url : null),
|
|
435
601
|
}));
|
|
436
602
|
}
|
|
437
603
|
if (ctx.sitemaps.urlCount > 0) {
|
|
@@ -439,9 +605,54 @@ const sitemapFindings = (scope) => {
|
|
|
439
605
|
}
|
|
440
606
|
return out;
|
|
441
607
|
};
|
|
608
|
+
/**
|
|
609
|
+
* Points every link to `targetKey` on the pages in `sources` at `hrefFor(link, page)`, up to
|
|
610
|
+
* `MAX_EDITS` edits.
|
|
611
|
+
*/
|
|
612
|
+
const relinkSources = (scope, sources, targetKey, hrefFor, write) => {
|
|
613
|
+
const edits = [];
|
|
614
|
+
for (const source of sources) {
|
|
615
|
+
const record = scope.pagesByKey.get(scope.key(source));
|
|
616
|
+
for (const link of record ? scope.linksTo(record, targetKey) : []) {
|
|
617
|
+
if (record && edits.length < MAX_EDITS) {
|
|
618
|
+
edits.push(relinkEdit(locationOf(record), link, hrefFor(link.href, record.url), write));
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
return edits;
|
|
623
|
+
};
|
|
624
|
+
/** A live page whose URL differs from `url` only in a trailing slash or letter case. */
|
|
625
|
+
const liveSpelling = (scope, url) => {
|
|
626
|
+
const match = scope.okPages.find((p) => spellingKey(p.url) === spellingKey(url) &&
|
|
627
|
+
scope.key(p.url) !== scope.key(url));
|
|
628
|
+
return match?.url ?? null;
|
|
629
|
+
};
|
|
630
|
+
/** Edits for broken internal links: the live spelling of the URL when there is one, else a slot. */
|
|
631
|
+
const brokenLinkEdits = (scope, broken) => broken
|
|
632
|
+
.flatMap((b) => {
|
|
633
|
+
const live = liveSpelling(scope, b.url);
|
|
634
|
+
return relinkSources(scope, scope.ctx.linkSources.get(b.key) ?? [], scope.key(b.url), (href, pageUrl) => live
|
|
635
|
+
? hrefLike(href, live, pageUrl)
|
|
636
|
+
: slot("URL of the live page this link meant"), live
|
|
637
|
+
? undefined
|
|
638
|
+
: `The URL of the page the link was meant for; remove the link if that page is gone. ${b.url} does not answer.`);
|
|
639
|
+
})
|
|
640
|
+
.slice(0, MAX_EDITS);
|
|
641
|
+
/** Points each hop of a redirect chain straight at where the chain ends. */
|
|
642
|
+
const chainEdits = (check) => check.redirectChain.slice(0, -1).map((hop, i) => ({
|
|
643
|
+
in: "http",
|
|
644
|
+
action: "replace",
|
|
645
|
+
url: hop.url,
|
|
646
|
+
current: `${statusLine(hop.status)}\nLocation: ${check.redirectChain[i + 1]?.url ?? check.finalUrl}`,
|
|
647
|
+
code: redirectTo(check.finalUrl, hop.status === 307 || hop.status === 308),
|
|
648
|
+
}));
|
|
442
649
|
/** Internal link targets that are broken, redirect once, or redirect in a chain. */
|
|
443
|
-
const linkHealthFindings = (
|
|
650
|
+
const linkHealthFindings = (scope) => {
|
|
651
|
+
const { ctx } = scope;
|
|
444
652
|
const out = [];
|
|
653
|
+
const brokenKeys = [];
|
|
654
|
+
const redirectedKeys = [];
|
|
655
|
+
const chainChecks = [];
|
|
445
656
|
/* link health */
|
|
446
657
|
const broken = [];
|
|
447
658
|
const redirected = [];
|
|
@@ -454,6 +665,7 @@ const linkHealthFindings = (ctx) => {
|
|
|
454
665
|
// Chains of two or more hops are handled first, so a first hop here is the only one.
|
|
455
666
|
const [firstHop] = check.redirectChain;
|
|
456
667
|
if (check.error || (check.status ?? 0) >= 400) {
|
|
668
|
+
brokenKeys.push({ url: check.url, key: k });
|
|
457
669
|
broken.push({
|
|
458
670
|
url: check.url,
|
|
459
671
|
status: check.status,
|
|
@@ -462,6 +674,7 @@ const linkHealthFindings = (ctx) => {
|
|
|
462
674
|
});
|
|
463
675
|
}
|
|
464
676
|
else if (check.redirectChain.length >= 2) {
|
|
677
|
+
chainChecks.push(check);
|
|
465
678
|
chains.push({
|
|
466
679
|
url: check.url,
|
|
467
680
|
hops: check.redirectChain.length,
|
|
@@ -469,6 +682,7 @@ const linkHealthFindings = (ctx) => {
|
|
|
469
682
|
});
|
|
470
683
|
}
|
|
471
684
|
else if (firstHop) {
|
|
685
|
+
redirectedKeys.push({ key: k, finalUrl: check.finalUrl });
|
|
472
686
|
redirected.push({
|
|
473
687
|
url: check.url,
|
|
474
688
|
finalUrl: check.finalUrl,
|
|
@@ -478,21 +692,32 @@ const linkHealthFindings = (ctx) => {
|
|
|
478
692
|
}
|
|
479
693
|
}
|
|
480
694
|
if (broken.length) {
|
|
481
|
-
out.push(finding(m("site.broken-internal-links"), `${broken.length} internal link target(s) are broken.`, {
|
|
695
|
+
out.push(finding(m("site.broken-internal-links"), `${broken.length} internal link target(s) are broken.`, {
|
|
696
|
+
fix: brokenLinkEdits(scope, brokenKeys),
|
|
697
|
+
evidence: { count: broken.length, examples: cap(broken, 20) },
|
|
698
|
+
}));
|
|
482
699
|
}
|
|
483
700
|
if (redirected.length) {
|
|
484
701
|
out.push(finding(m("site.redirected-internal-links"), `${redirected.length} internal link target(s) redirect.`, {
|
|
702
|
+
fix: redirectedKeys
|
|
703
|
+
.flatMap((r) => relinkSources(scope, ctx.linkSources.get(r.key) ?? [], scope.key(r.key), (href, pageUrl) => hrefLike(href, r.finalUrl, pageUrl)))
|
|
704
|
+
.slice(0, MAX_EDITS),
|
|
485
705
|
evidence: { count: redirected.length, examples: cap(redirected, 20) },
|
|
486
706
|
}));
|
|
487
707
|
}
|
|
488
708
|
if (chains.length) {
|
|
489
|
-
out.push(finding(m("site.redirect-chains"), `${chains.length} internal URL(s) have redirect chains.`, {
|
|
709
|
+
out.push(finding(m("site.redirect-chains"), `${chains.length} internal URL(s) have redirect chains.`, {
|
|
710
|
+
fix: chainChecks.flatMap(chainEdits).slice(0, MAX_EDITS),
|
|
711
|
+
evidence: { count: chains.length, examples: cap(chains) },
|
|
712
|
+
}));
|
|
490
713
|
}
|
|
491
714
|
return out;
|
|
492
715
|
};
|
|
493
716
|
/** Broken external links (when they were probed) and HTML files nothing links to. */
|
|
494
|
-
const unreachedFindings = (
|
|
717
|
+
const unreachedFindings = (scope) => {
|
|
718
|
+
const { ctx } = scope;
|
|
495
719
|
const out = [];
|
|
720
|
+
const start = scope.pagesByKey.get(scope.key(ctx.startUrl));
|
|
496
721
|
if (ctx.externalChecks) {
|
|
497
722
|
const dead = [];
|
|
498
723
|
for (const [u, check] of ctx.externalChecks) {
|
|
@@ -506,11 +731,19 @@ const unreachedFindings = (ctx) => {
|
|
|
506
731
|
}
|
|
507
732
|
}
|
|
508
733
|
if (dead.length) {
|
|
509
|
-
out.push(finding(m("site.broken-external-links"), `${dead.length} external link target(s) are broken.`, {
|
|
734
|
+
out.push(finding(m("site.broken-external-links"), `${dead.length} external link target(s) are broken.`, {
|
|
735
|
+
fix: dead
|
|
736
|
+
.flatMap((d) => relinkSources(scope, ctx.externalSources?.get(d.url) ?? [], scope.key(d.url), () => slot("current URL of the linked page"), `Where ${d.url} lives now: search for the page, or link a web archive copy; remove the link if the page is gone.`))
|
|
737
|
+
.slice(0, MAX_EDITS),
|
|
738
|
+
evidence: { count: dead.length, examples: cap(dead, 20) },
|
|
739
|
+
}));
|
|
510
740
|
}
|
|
511
741
|
}
|
|
512
742
|
if (ctx.unlinkedPages?.length) {
|
|
513
743
|
out.push(finding(m("site.unlinked-pages"), `${ctx.unlinkedPages.length} HTML file(s) are not linked from any other page.`, {
|
|
744
|
+
fix: ctx.unlinkedPages
|
|
745
|
+
.slice(0, MAX_EDITS)
|
|
746
|
+
.map((file) => addLinkEdit(start ? locationOf(start) : ctx.startUrl, ctx.pages.find((p) => (p.file ?? p.url) === file)?.url ?? file, `Words that say what ${file} is about. Link it from the page closest in topic or the navigation; if the file is obsolete, delete it from the build instead.`)),
|
|
514
747
|
evidence: {
|
|
515
748
|
count: ctx.unlinkedPages.length,
|
|
516
749
|
examples: cap(ctx.unlinkedPages),
|
|
@@ -524,19 +757,36 @@ const duplicateFindings = ({ key, okPages, isIndexable, }) => {
|
|
|
524
757
|
// Pages that name another URL as canonical are consolidated already; only the canonical ones compete.
|
|
525
758
|
const indexable = okPages.filter((p) => isIndexable(p) &&
|
|
526
759
|
(!p.page.canonical || key(p.page.canonical) === key(p.url)));
|
|
527
|
-
pushDuplicates(out, m("site.duplicate-titles"), indexable, (p) => p.page.title, "title");
|
|
528
|
-
pushDuplicates(out, m("site.duplicate-descriptions"), indexable, (p) => p.page.metaDescription, "meta description");
|
|
529
|
-
pushDuplicates(out, m("site.duplicate-h1"), indexable, (p) => p.page.h1s[0] ?? null, "H1");
|
|
760
|
+
pushDuplicates(out, m("site.duplicate-titles"), indexable, (p) => p.page.title, "title", retitleEdit);
|
|
761
|
+
pushDuplicates(out, m("site.duplicate-descriptions"), indexable, (p) => p.page.metaDescription, "meta description", redescribeEdit);
|
|
762
|
+
pushDuplicates(out, m("site.duplicate-h1"), indexable, (p) => p.page.h1s[0] ?? null, "H1", reheadEdit);
|
|
530
763
|
pushDuplicateContent(out, indexable);
|
|
531
764
|
return out;
|
|
532
765
|
};
|
|
533
766
|
/** Pages deep in the site, and how the server answers a made-up URL and the other host names. */
|
|
534
|
-
|
|
767
|
+
/** The same URL on `origin`, for a redirect to the site's main host or to https. */
|
|
768
|
+
const onOrigin = (url, origin) => {
|
|
769
|
+
try {
|
|
770
|
+
const u = new URL(url);
|
|
771
|
+
// Appended, not resolved: a "//x" path resolved against the origin would name a host.
|
|
772
|
+
return new URL(`${origin}${u.pathname}${u.search}`).href;
|
|
773
|
+
}
|
|
774
|
+
catch {
|
|
775
|
+
return origin;
|
|
776
|
+
}
|
|
777
|
+
};
|
|
778
|
+
const architectureFindings = ({ ctx, key, pagesByKey, okPages, }) => {
|
|
535
779
|
const out = [];
|
|
780
|
+
const start = pagesByKey.get(key(ctx.startUrl));
|
|
781
|
+
const home = start ? locationOf(start) : ctx.startUrl;
|
|
782
|
+
const { origin } = new URL(ctx.startUrl);
|
|
536
783
|
/* architecture */
|
|
537
784
|
const deep = okPages.filter((p) => p.depth > ctx.thresholds.deepPageDepth);
|
|
538
785
|
if (deep.length) {
|
|
539
786
|
out.push(finding(m("site.deep-pages"), `${deep.length} page(s) are more than ${ctx.thresholds.deepPageDepth} clicks from the start URL.`, {
|
|
787
|
+
fix: deep
|
|
788
|
+
.slice(0, MAX_EDITS)
|
|
789
|
+
.map((p) => addLinkEdit(home, p.url, `Words that say what ${p.url} is about. Or link it from a category or hub page one or two clicks from the home page.`)),
|
|
540
790
|
evidence: {
|
|
541
791
|
count: deep.length,
|
|
542
792
|
examples: cap(deep.map((p) => ({ url: p.url, depth: p.depth }))),
|
|
@@ -547,17 +797,53 @@ const architectureFindings = ({ ctx, okPages }) => {
|
|
|
547
797
|
if (ctx.probes.soft404 &&
|
|
548
798
|
ctx.probes.soft404.status !== null &&
|
|
549
799
|
ctx.probes.soft404.status < 400) {
|
|
550
|
-
out.push(finding(m("site.soft-404"), `${ctx.probes.soft404.url} returned HTTP ${ctx.probes.soft404.status}.`, {
|
|
800
|
+
out.push(finding(m("site.soft-404"), `${ctx.probes.soft404.url} returned HTTP ${ctx.probes.soft404.status}.`, {
|
|
801
|
+
fix: [
|
|
802
|
+
{
|
|
803
|
+
in: "http",
|
|
804
|
+
action: "replace",
|
|
805
|
+
url: ctx.probes.soft404.url,
|
|
806
|
+
current: statusLine(ctx.probes.soft404.status),
|
|
807
|
+
code: statusLine(404),
|
|
808
|
+
write: "For every URL no route matches: the catch-all route or the single-page app's fallback should answer 404 (it may keep showing its not-found page).",
|
|
809
|
+
},
|
|
810
|
+
],
|
|
811
|
+
evidence: ctx.probes.soft404,
|
|
812
|
+
}));
|
|
551
813
|
}
|
|
552
814
|
if (ctx.probes.httpRedirect &&
|
|
553
815
|
ctx.probes.httpRedirect.status !== null &&
|
|
554
816
|
!ctx.probes.httpRedirect.redirectsToHttps) {
|
|
555
|
-
out.push(finding(m("site.http-not-redirected"), `${ctx.probes.httpRedirect.url} returned HTTP ${ctx.probes.httpRedirect.status} without redirecting to https.`, {
|
|
817
|
+
out.push(finding(m("site.http-not-redirected"), `${ctx.probes.httpRedirect.url} returned HTTP ${ctx.probes.httpRedirect.status} without redirecting to https.`, {
|
|
818
|
+
fix: [
|
|
819
|
+
{
|
|
820
|
+
in: "http",
|
|
821
|
+
action: "replace",
|
|
822
|
+
url: ctx.probes.httpRedirect.url,
|
|
823
|
+
current: statusLine(ctx.probes.httpRedirect.status),
|
|
824
|
+
code: redirectTo(ctx.probes.httpRedirect.url.replace(/^http:/iu, "https:")),
|
|
825
|
+
write: "For every http:// URL, to the same path on https://, at the server or CDN.",
|
|
826
|
+
},
|
|
827
|
+
],
|
|
828
|
+
evidence: ctx.probes.httpRedirect,
|
|
829
|
+
}));
|
|
556
830
|
}
|
|
557
831
|
if (ctx.probes.hostVariant &&
|
|
558
832
|
ctx.probes.hostVariant.status !== null &&
|
|
559
833
|
!ctx.probes.hostVariant.redirectsToMain) {
|
|
560
|
-
out.push(finding(m("site.host-variant-not-redirected"), `${ctx.probes.hostVariant.url} returned HTTP ${ctx.probes.hostVariant.status} without redirecting to the main host.`, {
|
|
834
|
+
out.push(finding(m("site.host-variant-not-redirected"), `${ctx.probes.hostVariant.url} returned HTTP ${ctx.probes.hostVariant.status} without redirecting to the main host.`, {
|
|
835
|
+
fix: [
|
|
836
|
+
{
|
|
837
|
+
in: "http",
|
|
838
|
+
action: "replace",
|
|
839
|
+
url: ctx.probes.hostVariant.url,
|
|
840
|
+
current: statusLine(ctx.probes.hostVariant.status),
|
|
841
|
+
code: redirectTo(onOrigin(ctx.probes.hostVariant.url, origin)),
|
|
842
|
+
write: `For every URL on this host, to the same path on ${origin}, at the DNS host, server or CDN.`,
|
|
843
|
+
},
|
|
844
|
+
],
|
|
845
|
+
evidence: ctx.probes.hostVariant,
|
|
846
|
+
}));
|
|
561
847
|
}
|
|
562
848
|
return out;
|
|
563
849
|
};
|
|
@@ -565,23 +851,34 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
|
|
|
565
851
|
const out = [];
|
|
566
852
|
/* hreflang return links */
|
|
567
853
|
const missingReturn = [];
|
|
854
|
+
const returnLinks = [];
|
|
568
855
|
for (const p of ctx.pages) {
|
|
856
|
+
// A page whose canonical names another URL (a parameter or tracking variant) is folded into
|
|
857
|
+
// that URL, so its hreflang links do not count and nothing should link back to it.
|
|
858
|
+
if (p.page.canonical && key(p.page.canonical) !== key(p.url)) {
|
|
859
|
+
continue;
|
|
860
|
+
}
|
|
861
|
+
// en and x-default often name the same page: one missing return link, not two.
|
|
862
|
+
const seen = new Set();
|
|
569
863
|
for (const alt of p.page.hreflang) {
|
|
570
864
|
if (!alt.url) {
|
|
571
865
|
continue;
|
|
572
866
|
}
|
|
573
867
|
const target = pagesByKey.get(key(alt.url));
|
|
574
|
-
if (!target || target === p) {
|
|
868
|
+
if (!target || target === p || seen.has(target)) {
|
|
575
869
|
continue;
|
|
576
870
|
}
|
|
871
|
+
seen.add(target);
|
|
577
872
|
const linksBack = target.page.hreflang.some((h) => h.url && key(h.url) === key(p.url));
|
|
578
873
|
if (!linksBack) {
|
|
579
874
|
missingReturn.push({ page: p.url, alternate: alt.url });
|
|
875
|
+
returnLinks.push(returnLinkEdit(target, p, key));
|
|
580
876
|
}
|
|
581
877
|
}
|
|
582
878
|
}
|
|
583
879
|
if (missingReturn.length) {
|
|
584
880
|
out.push(finding(m("site.hreflang-no-return-link"), `${missingReturn.length} hreflang alternate(s) do not link back.`, {
|
|
881
|
+
fix: returnLinks.slice(0, MAX_EDITS),
|
|
585
882
|
evidence: {
|
|
586
883
|
count: missingReturn.length,
|
|
587
884
|
examples: cap(missingReturn),
|
|
@@ -590,6 +887,8 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
|
|
|
590
887
|
}
|
|
591
888
|
if (ctx.crawl.hitPageLimit) {
|
|
592
889
|
out.push(finding(m("site.crawl-limit-reached"), `Crawl stopped after ${ctx.crawl.pagesCrawled} pages (limit ${ctx.crawl.maxPages}); ${ctx.crawl.pagesDiscovered} URLs were discovered.`, {
|
|
890
|
+
// About the audit, not the site: the change is a larger --max-pages, not an edit.
|
|
891
|
+
fix: [],
|
|
593
892
|
evidence: {
|
|
594
893
|
pagesCrawled: ctx.crawl.pagesCrawled,
|
|
595
894
|
pagesDiscovered: ctx.crawl.pagesDiscovered,
|
|
@@ -599,8 +898,39 @@ const hreflangFindings = ({ ctx, key, pagesByKey }) => {
|
|
|
599
898
|
}
|
|
600
899
|
return out;
|
|
601
900
|
};
|
|
901
|
+
/** The canonical a page should name instead of a broken, redirecting, noindex or chained one. */
|
|
902
|
+
const canonicalRepair = (scope, p, problem) => {
|
|
903
|
+
if (problem.kind === "redirects" || problem.kind === "canonical") {
|
|
904
|
+
return canonicalEdit(p, problem.to ?? p.url);
|
|
905
|
+
}
|
|
906
|
+
if (problem.kind === "noindex") {
|
|
907
|
+
return canonicalEdit(p, p.url, "The page's own URL. If the noindex page is the one meant to rank, remove its noindex and keep this canonical instead.");
|
|
908
|
+
}
|
|
909
|
+
const live = p.page.canonical ? liveSpelling(scope, p.page.canonical) : null;
|
|
910
|
+
return canonicalEdit(p, live ?? p.url, live
|
|
911
|
+
? undefined
|
|
912
|
+
: "The page's own URL. If another page is the preferred one, use its live URL.");
|
|
913
|
+
};
|
|
602
914
|
/** What is wrong with a URL that metadata (a canonical, an hreflang link) points to, or null. */
|
|
603
|
-
|
|
915
|
+
/**
|
|
916
|
+
* Where a chain of canonicals ends: the first page that names itself (or nothing), or the last URL
|
|
917
|
+
* before the chain loops or leaves the crawled pages.
|
|
918
|
+
*/
|
|
919
|
+
const finalCanonical = ({ key, pagesByKey }, url) => {
|
|
920
|
+
let current = url;
|
|
921
|
+
const seen = new Set([key(url)]);
|
|
922
|
+
for (let hops = 0; hops < 10; hops += 1) {
|
|
923
|
+
const next = pagesByKey.get(key(current))?.page.canonical;
|
|
924
|
+
if (!next || seen.has(key(next))) {
|
|
925
|
+
return current;
|
|
926
|
+
}
|
|
927
|
+
seen.add(key(next));
|
|
928
|
+
current = next;
|
|
929
|
+
}
|
|
930
|
+
return current;
|
|
931
|
+
};
|
|
932
|
+
const targetProblem = (scope, url) => {
|
|
933
|
+
const { key, pagesByKey, checksByKey, isIndexable } = scope;
|
|
604
934
|
const check = checksByKey.get(key(url));
|
|
605
935
|
if (check && (check.error || (check.status ?? 0) >= 400)) {
|
|
606
936
|
return {
|
|
@@ -609,7 +939,11 @@ const targetProblem = ({ key, pagesByKey, checksByKey, isIndexable }, url) => {
|
|
|
609
939
|
};
|
|
610
940
|
}
|
|
611
941
|
if (check && check.redirectChain.length > 0) {
|
|
612
|
-
return {
|
|
942
|
+
return {
|
|
943
|
+
kind: "redirects",
|
|
944
|
+
detail: `redirects to ${check.finalUrl}`,
|
|
945
|
+
to: check.finalUrl,
|
|
946
|
+
};
|
|
613
947
|
}
|
|
614
948
|
const target = pagesByKey.get(key(url));
|
|
615
949
|
if (!target) {
|
|
@@ -620,7 +954,11 @@ const targetProblem = ({ key, pagesByKey, checksByKey, isIndexable }, url) => {
|
|
|
620
954
|
}
|
|
621
955
|
const { canonical } = target.page;
|
|
622
956
|
return canonical && key(canonical) !== key(target.url)
|
|
623
|
-
? {
|
|
957
|
+
? {
|
|
958
|
+
kind: "canonical",
|
|
959
|
+
detail: `names ${canonical} as canonical`,
|
|
960
|
+
to: finalCanonical(scope, canonical),
|
|
961
|
+
}
|
|
624
962
|
: null;
|
|
625
963
|
};
|
|
626
964
|
/** Canonicals that point at broken, redirecting, noindex or re-canonicalized URLs. */
|
|
@@ -632,6 +970,12 @@ const canonicalGraphFindings = (scope) => {
|
|
|
632
970
|
noindex: [],
|
|
633
971
|
canonical: [],
|
|
634
972
|
};
|
|
973
|
+
const repairs = {
|
|
974
|
+
broken: [],
|
|
975
|
+
redirects: [],
|
|
976
|
+
noindex: [],
|
|
977
|
+
canonical: [],
|
|
978
|
+
};
|
|
635
979
|
for (const p of okPages) {
|
|
636
980
|
const { canonical } = p.page;
|
|
637
981
|
// A noindex page's canonical carries no weight; the noindex wins.
|
|
@@ -645,6 +989,7 @@ const canonicalGraphFindings = (scope) => {
|
|
|
645
989
|
canonical,
|
|
646
990
|
problem: problem.detail,
|
|
647
991
|
});
|
|
992
|
+
repairs[problem.kind].push(canonicalRepair(scope, p, problem));
|
|
648
993
|
}
|
|
649
994
|
}
|
|
650
995
|
const rules = [
|
|
@@ -674,6 +1019,7 @@ const canonicalGraphFindings = (scope) => {
|
|
|
674
1019
|
return list.length > 0
|
|
675
1020
|
? [
|
|
676
1021
|
finding(m(id), `${list.length} page(s) ${label}.`, {
|
|
1022
|
+
fix: repairs[kind].slice(0, MAX_EDITS),
|
|
677
1023
|
evidence: { count: list.length, examples: cap(list) },
|
|
678
1024
|
}),
|
|
679
1025
|
]
|
|
@@ -691,7 +1037,7 @@ const isUtilityPage = (url) => {
|
|
|
691
1037
|
}
|
|
692
1038
|
};
|
|
693
1039
|
/** Noindex pages that other pages of the site link to, most linked first. */
|
|
694
|
-
const noindexLinkFindings = ({ ctx, key, okPages, isIndexable, }) => {
|
|
1040
|
+
const noindexLinkFindings = ({ ctx, key, pagesByKey, okPages, isIndexable, }) => {
|
|
695
1041
|
const linkedFrom = new Map();
|
|
696
1042
|
for (const [target, sources] of ctx.linkSources) {
|
|
697
1043
|
const k = key(target);
|
|
@@ -713,15 +1059,42 @@ const noindexLinkFindings = ({ ctx, key, okPages, isIndexable, }) => {
|
|
|
713
1059
|
.toSorted((a, b) => b.linkedFrom - a.linkedFrom);
|
|
714
1060
|
return linked.length > 0
|
|
715
1061
|
? [
|
|
716
|
-
finding(m("site.links-to-noindex"), `${linked.length} noindex page(s) are linked from other pages; the most linked: ${linked[0]?.url} (${linked[0]?.linkedFrom} pages).`, {
|
|
1062
|
+
finding(m("site.links-to-noindex"), `${linked.length} noindex page(s) are linked from other pages; the most linked: ${linked[0]?.url} (${linked[0]?.linkedFrom} pages).`, {
|
|
1063
|
+
fix: linked
|
|
1064
|
+
.flatMap((l) => {
|
|
1065
|
+
const record = pagesByKey.get(key(l.url));
|
|
1066
|
+
return record
|
|
1067
|
+
? removeNoindexEdits(record, `Only if ${l.url} belongs in search results; leave the noindex on a page kept out on purpose.`)
|
|
1068
|
+
: [];
|
|
1069
|
+
})
|
|
1070
|
+
.slice(0, MAX_EDITS),
|
|
1071
|
+
evidence: { count: linked.length, pages: cap(linked) },
|
|
1072
|
+
}),
|
|
717
1073
|
]
|
|
718
1074
|
: [];
|
|
719
1075
|
};
|
|
1076
|
+
/**
|
|
1077
|
+
* Points an hreflang link at the canonical, live URL its alternate leads to, or removes it when the
|
|
1078
|
+
* alternate is broken or noindex: an annotation to such a page is ignored anyway.
|
|
1079
|
+
*/
|
|
1080
|
+
const hreflangRepair = (p, alt, problem) => {
|
|
1081
|
+
const at = `link[rel="alternate"][hreflang="${alt.lang}"]`;
|
|
1082
|
+
if ((problem.kind === "redirects" || problem.kind === "canonical") &&
|
|
1083
|
+
problem.to) {
|
|
1084
|
+
return onUrl(setAttributeEdit(alt.tag, at, "href", escapeAttribute(problem.to)), locationOf(p));
|
|
1085
|
+
}
|
|
1086
|
+
return removeEdit("html", alt.tag, {
|
|
1087
|
+
url: locationOf(p),
|
|
1088
|
+
at,
|
|
1089
|
+
write: `${alt.url} ${problem.detail}. If that language version should exist, list its live, indexable URL here instead.`,
|
|
1090
|
+
});
|
|
1091
|
+
};
|
|
720
1092
|
/** hreflang alternates that are broken, redirect, are noindex or canonicalize elsewhere. */
|
|
721
1093
|
const hreflangTargetFindings = (scope) => {
|
|
722
1094
|
const { key, okPages } = scope;
|
|
723
1095
|
const invalid = [];
|
|
724
1096
|
const seen = new Set();
|
|
1097
|
+
const fix = [];
|
|
725
1098
|
for (const p of okPages) {
|
|
726
1099
|
for (const alt of p.page.hreflang) {
|
|
727
1100
|
if (!alt.url || key(alt.url) === key(p.url)) {
|
|
@@ -739,26 +1112,19 @@ const hreflangTargetFindings = (scope) => {
|
|
|
739
1112
|
alternate: alt.url,
|
|
740
1113
|
problem: problem.detail,
|
|
741
1114
|
});
|
|
1115
|
+
fix.push(hreflangRepair(p, alt, problem));
|
|
742
1116
|
}
|
|
743
1117
|
}
|
|
744
1118
|
}
|
|
745
1119
|
return invalid.length > 0
|
|
746
1120
|
? [
|
|
747
|
-
finding(m("site.hreflang-target-invalid"), `${invalid.length} hreflang alternate(s) point to URLs that are broken, redirect, are noindex or are not canonical.`, {
|
|
1121
|
+
finding(m("site.hreflang-target-invalid"), `${invalid.length} hreflang alternate(s) point to URLs that are broken, redirect, are noindex or are not canonical.`, {
|
|
1122
|
+
fix: fix.slice(0, MAX_EDITS),
|
|
1123
|
+
evidence: { count: invalid.length, examples: cap(invalid) },
|
|
1124
|
+
}),
|
|
748
1125
|
]
|
|
749
1126
|
: [];
|
|
750
1127
|
};
|
|
751
|
-
/** The URL with its path lower-cased and trailing slashes dropped: one key for every spelling. */
|
|
752
|
-
const spellingKey = (url) => {
|
|
753
|
-
try {
|
|
754
|
-
const u = new URL(url);
|
|
755
|
-
const pathname = trimTrailingSlashes(u.pathname.toLowerCase()) || "/";
|
|
756
|
-
return `${u.origin}${pathname}${u.search}`;
|
|
757
|
-
}
|
|
758
|
-
catch {
|
|
759
|
-
return url;
|
|
760
|
-
}
|
|
761
|
-
};
|
|
762
1128
|
const pathnameOf = (url) => {
|
|
763
1129
|
try {
|
|
764
1130
|
return new URL(url).pathname;
|
|
@@ -811,8 +1177,19 @@ const urlVariantFindings = (scope) => {
|
|
|
811
1177
|
return [];
|
|
812
1178
|
}
|
|
813
1179
|
const kinds = [...new Set(duplicates.map((d) => d.kind))];
|
|
1180
|
+
const fix = duplicates.slice(0, MAX_EDITS).map((d) => ({
|
|
1181
|
+
in: "http",
|
|
1182
|
+
action: "replace",
|
|
1183
|
+
url: d.variant,
|
|
1184
|
+
current: statusLine(d.status ?? 200),
|
|
1185
|
+
code: redirectTo(d.url),
|
|
1186
|
+
write: d.kind === "trailing-slash"
|
|
1187
|
+
? "For every URL of the site, to the one spelling with or without the trailing slash: the trailingSlash setting in Next.js, Nuxt or Astro, or a redirect rule at the server or CDN."
|
|
1188
|
+
: "For every URL spelled in another case, to the spelling the site links to: a redirect rule at the server or CDN.",
|
|
1189
|
+
}));
|
|
814
1190
|
return [
|
|
815
1191
|
finding(m("site.url-variant-duplicate"), `${duplicates.length} page(s) also answer at a ${kinds.join(" and a ")} variant of their URL.`, {
|
|
1192
|
+
fix,
|
|
816
1193
|
evidence: {
|
|
817
1194
|
count: duplicates.length,
|
|
818
1195
|
examples: cap(duplicates.map(({ kind, url, variant, canonical }) => ({
|
|
@@ -835,7 +1212,16 @@ const withoutQuery = (url) => {
|
|
|
835
1212
|
}
|
|
836
1213
|
};
|
|
837
1214
|
/** Pages that differ only in their query string, show the same text and do not consolidate. */
|
|
838
|
-
|
|
1215
|
+
/** Canonicals from each parameter version to the URL without the parameter, or the first URL. */
|
|
1216
|
+
const parameterCanonicals = ({ key, pagesByKey }, group) => {
|
|
1217
|
+
const target = group.urls.find((u) => !u.includes("?")) ?? group.urls[0] ?? group.path;
|
|
1218
|
+
return group.urls.flatMap((u) => {
|
|
1219
|
+
const record = key(u) === key(target) ? undefined : pagesByKey.get(key(u));
|
|
1220
|
+
return record ? [canonicalEdit(record, target)] : [];
|
|
1221
|
+
});
|
|
1222
|
+
};
|
|
1223
|
+
const parameterDuplicateFindings = (scope) => {
|
|
1224
|
+
const { key, okPages, isIndexable } = scope;
|
|
839
1225
|
const byPath = new Map();
|
|
840
1226
|
for (const p of okPages) {
|
|
841
1227
|
const { canonical } = p.page;
|
|
@@ -873,12 +1259,41 @@ const parameterDuplicateFindings = ({ key, okPages, isIndexable, }) => {
|
|
|
873
1259
|
}
|
|
874
1260
|
return groups.length > 0
|
|
875
1261
|
? [
|
|
876
|
-
finding(m("site.parameter-duplicates"), `${groups.length} path(s) are crawled under several query strings with the same text.`, {
|
|
1262
|
+
finding(m("site.parameter-duplicates"), `${groups.length} path(s) are crawled under several query strings with the same text.`, {
|
|
1263
|
+
fix: groups
|
|
1264
|
+
.flatMap((g) => parameterCanonicals(scope, g))
|
|
1265
|
+
.slice(0, MAX_EDITS),
|
|
1266
|
+
evidence: { count: groups.length, groups: cap(groups) },
|
|
1267
|
+
}),
|
|
877
1268
|
]
|
|
878
1269
|
: [];
|
|
879
1270
|
};
|
|
1271
|
+
/** Points each <img> that shows a broken image at a slot for the image's working URL. */
|
|
1272
|
+
const brokenImageEdits = ({ key, pagesByKey }, image, pages) => {
|
|
1273
|
+
const write = `The URL the image file is deployed at; or upload the missing file to ${image.url}.`;
|
|
1274
|
+
const edits = [...pages].flatMap((u) => {
|
|
1275
|
+
const record = pagesByKey.get(key(u));
|
|
1276
|
+
return record
|
|
1277
|
+
? imagesOf(record, image.url).map((img) => onUrl(setAttributeEdit(img.tag, "img", "src", slot("working URL of the image"), write), locationOf(record)))
|
|
1278
|
+
: [];
|
|
1279
|
+
});
|
|
1280
|
+
// An image from srcset or a <picture> source has no <img src> to quote: upload the file instead.
|
|
1281
|
+
return edits.length > 0
|
|
1282
|
+
? edits
|
|
1283
|
+
: [
|
|
1284
|
+
{
|
|
1285
|
+
in: "asset",
|
|
1286
|
+
action: "add",
|
|
1287
|
+
url: image.url,
|
|
1288
|
+
at: "new file",
|
|
1289
|
+
code: slot("the image file"),
|
|
1290
|
+
write: `Upload the image the pages show here, or change the pages that point to ${image.url}.`,
|
|
1291
|
+
},
|
|
1292
|
+
];
|
|
1293
|
+
};
|
|
880
1294
|
/** Images that answer 4xx/5xx or do not load, with the pages that show them. */
|
|
881
|
-
const brokenImageFindings = (
|
|
1295
|
+
const brokenImageFindings = (scope) => {
|
|
1296
|
+
const { ctx } = scope;
|
|
882
1297
|
const broken = [];
|
|
883
1298
|
for (const [url, check] of ctx.imageChecks ?? []) {
|
|
884
1299
|
if (check.error || (check.status ?? 0) >= 400) {
|
|
@@ -895,7 +1310,12 @@ const brokenImageFindings = (ctx) => {
|
|
|
895
1310
|
}
|
|
896
1311
|
return broken.length > 0
|
|
897
1312
|
? [
|
|
898
|
-
finding(m("site.broken-images"), `${broken.length} image URL(s) are broken.`, {
|
|
1313
|
+
finding(m("site.broken-images"), `${broken.length} image URL(s) are broken.`, {
|
|
1314
|
+
fix: broken
|
|
1315
|
+
.flatMap((b) => brokenImageEdits(scope, b, ctx.imageSources?.get(b.url) ?? []))
|
|
1316
|
+
.slice(0, MAX_EDITS),
|
|
1317
|
+
evidence: { count: broken.length, examples: cap(broken, 20) },
|
|
1318
|
+
}),
|
|
899
1319
|
]
|
|
900
1320
|
: [];
|
|
901
1321
|
};
|
|
@@ -919,20 +1339,21 @@ export const runSiteRules = (ctx) => {
|
|
|
919
1339
|
okPages: ctx.pages.filter((p) => (p.response?.status ?? 200) < 400),
|
|
920
1340
|
isIndexable: (p) => !robotsDirectives(p.page, p.response).noindex &&
|
|
921
1341
|
(p.response?.status ?? 200) < 400,
|
|
1342
|
+
linksTo: (p, targetKey) => linksTo(p, targetKey, key),
|
|
922
1343
|
};
|
|
923
1344
|
return [
|
|
924
|
-
...robotsFindings(
|
|
1345
|
+
...robotsFindings(scope),
|
|
925
1346
|
...robotsContentFindings(ctx),
|
|
926
1347
|
...blockedResourceFindings(scope),
|
|
927
1348
|
...sitemapFindings(scope),
|
|
928
1349
|
...sitemapProtocolFindings(ctx),
|
|
929
|
-
...linkHealthFindings(
|
|
930
|
-
...unreachedFindings(
|
|
1350
|
+
...linkHealthFindings(scope),
|
|
1351
|
+
...unreachedFindings(scope),
|
|
931
1352
|
...duplicateFindings(scope),
|
|
932
1353
|
...architectureFindings(scope),
|
|
933
1354
|
...urlVariantFindings(scope),
|
|
934
1355
|
...parameterDuplicateFindings(scope),
|
|
935
|
-
...brokenImageFindings(
|
|
1356
|
+
...brokenImageFindings(scope),
|
|
936
1357
|
...canonicalGraphFindings(scope),
|
|
937
1358
|
...noindexLinkFindings(scope),
|
|
938
1359
|
...hreflangTargetFindings(scope),
|