@rivi-software/agentic-seo-check 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +210 -2
- package/dist/brand.d.ts +11 -0
- package/dist/brand.js +22 -0
- package/dist/brand.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +194 -0
- package/dist/cli.js.map +1 -0
- package/dist/core/audit.d.ts +72 -0
- package/dist/core/audit.js +452 -0
- package/dist/core/audit.js.map +1 -0
- package/dist/core/concurrency.d.ts +4 -0
- package/dist/core/concurrency.js +39 -0
- package/dist/core/concurrency.js.map +1 -0
- package/dist/core/crawl.d.ts +40 -0
- package/dist/core/crawl.js +324 -0
- package/dist/core/crawl.js.map +1 -0
- package/dist/core/fetch.d.ts +27 -0
- package/dist/core/fetch.js +186 -0
- package/dist/core/fetch.js.map +1 -0
- package/dist/core/html/dom.d.ts +13 -0
- package/dist/core/html/dom.js +17 -0
- package/dist/core/html/dom.js.map +1 -0
- package/dist/core/html/extract.d.ts +10 -0
- package/dist/core/html/extract.js +207 -0
- package/dist/core/html/extract.js.map +1 -0
- package/dist/core/html/structured-data.d.ts +21 -0
- package/dist/core/html/structured-data.js +220 -0
- package/dist/core/html/structured-data.js.map +1 -0
- package/dist/core/html/text.d.ts +23 -0
- package/dist/core/html/text.js +152 -0
- package/dist/core/html/text.js.map +1 -0
- package/dist/core/lighthouse.d.ts +50 -0
- package/dist/core/lighthouse.js +216 -0
- package/dist/core/lighthouse.js.map +1 -0
- package/dist/core/performance-rules.d.ts +3 -0
- package/dist/core/performance-rules.js +125 -0
- package/dist/core/performance-rules.js.map +1 -0
- package/dist/core/render-rules.d.ts +8 -0
- package/dist/core/render-rules.js +144 -0
- package/dist/core/render-rules.js.map +1 -0
- package/dist/core/render.d.ts +22 -0
- package/dist/core/render.js +79 -0
- package/dist/core/render.js.map +1 -0
- package/dist/core/report/markdown.d.ts +2 -0
- package/dist/core/report/markdown.js +131 -0
- package/dist/core/report/markdown.js.map +1 -0
- package/dist/core/robots.d.ts +11 -0
- package/dist/core/robots.js +48 -0
- package/dist/core/robots.js.map +1 -0
- package/dist/core/rules/canonical.d.ts +2 -0
- package/dist/core/rules/canonical.js +141 -0
- package/dist/core/rules/canonical.js.map +1 -0
- package/dist/core/rules/content.d.ts +1 -0
- package/dist/core/rules/content.js +55 -0
- package/dist/core/rules/content.js.map +1 -0
- package/dist/core/rules/document.d.ts +1 -0
- package/dist/core/rules/document.js +83 -0
- package/dist/core/rules/document.js.map +1 -0
- package/dist/core/rules/headings.d.ts +1 -0
- package/dist/core/rules/headings.js +81 -0
- package/dist/core/rules/headings.js.map +1 -0
- package/dist/core/rules/hreflang.d.ts +1 -0
- package/dist/core/rules/hreflang.js +100 -0
- package/dist/core/rules/hreflang.js.map +1 -0
- package/dist/core/rules/images.d.ts +1 -0
- package/dist/core/rules/images.js +77 -0
- package/dist/core/rules/images.js.map +1 -0
- package/dist/core/rules/index.d.ts +14 -0
- package/dist/core/rules/index.js +77 -0
- package/dist/core/rules/index.js.map +1 -0
- package/dist/core/rules/indexing.d.ts +10 -0
- package/dist/core/rules/indexing.js +75 -0
- package/dist/core/rules/indexing.js.map +1 -0
- package/dist/core/rules/keyword.d.ts +1 -0
- package/dist/core/rules/keyword.js +111 -0
- package/dist/core/rules/keyword.js.map +1 -0
- package/dist/core/rules/links.d.ts +1 -0
- package/dist/core/rules/links.js +107 -0
- package/dist/core/rules/links.js.map +1 -0
- package/dist/core/rules/meta.d.ts +1 -0
- package/dist/core/rules/meta.js +81 -0
- package/dist/core/rules/meta.js.map +1 -0
- package/dist/core/rules/response.d.ts +1 -0
- package/dist/core/rules/response.js +93 -0
- package/dist/core/rules/response.js.map +1 -0
- package/dist/core/rules/rule.d.ts +42 -0
- package/dist/core/rules/rule.js +51 -0
- package/dist/core/rules/rule.js.map +1 -0
- package/dist/core/rules/social.d.ts +1 -0
- package/dist/core/rules/social.js +115 -0
- package/dist/core/rules/social.js.map +1 -0
- package/dist/core/rules/structured-data.d.ts +25 -0
- package/dist/core/rules/structured-data.js +381 -0
- package/dist/core/rules/structured-data.js.map +1 -0
- package/dist/core/rules/title.d.ts +1 -0
- package/dist/core/rules/title.js +74 -0
- package/dist/core/rules/title.js.map +1 -0
- package/dist/core/score.d.ts +3 -0
- package/dist/core/score.js +17 -0
- package/dist/core/score.js.map +1 -0
- package/dist/core/site-rules.d.ts +64 -0
- package/dist/core/site-rules.js +462 -0
- package/dist/core/site-rules.js.map +1 -0
- package/dist/core/sitemap.d.ts +13 -0
- package/dist/core/sitemap.js +140 -0
- package/dist/core/sitemap.js.map +1 -0
- package/dist/core/static.d.ts +9 -0
- package/dist/core/static.js +281 -0
- package/dist/core/static.js.map +1 -0
- package/dist/core/summary.d.ts +2 -0
- package/dist/core/summary.js +33 -0
- package/dist/core/summary.js.map +1 -0
- package/dist/core/types.d.ts +296 -0
- package/dist/core/types.js +33 -0
- package/dist/core/types.js.map +1 -0
- package/dist/core/url.d.ts +20 -0
- package/dist/core/url.js +191 -0
- package/dist/core/url.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp-bin.d.ts +2 -0
- package/dist/mcp-bin.js +10 -0
- package/dist/mcp-bin.js.map +1 -0
- package/dist/mcp.d.ts +5 -0
- package/dist/mcp.js +380 -0
- package/dist/mcp.js.map +1 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +6 -0
- package/dist/version.js.map +1 -0
- package/package.json +107 -3
- package/skills/seo-audit/SKILL.md +53 -0
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
import { normalizeForComparison, shingles, jaccard } from "./html/text.js";
|
|
2
|
+
import { robotsDirectives } from "./rules/indexing.js";
|
|
3
|
+
import { finding, GOOGLE_DOCS, cap } from "./rules/rule.js";
|
|
4
|
+
import { normalizeUrl, normalizePath } from "./url.js";
|
|
5
|
+
const meta = (id, severity, description, recommendation, docs) => ({
|
|
6
|
+
id,
|
|
7
|
+
category: "site",
|
|
8
|
+
severity,
|
|
9
|
+
description,
|
|
10
|
+
recommendation,
|
|
11
|
+
docs,
|
|
12
|
+
requires: "crawl",
|
|
13
|
+
});
|
|
14
|
+
export const SITE_RULES = [
|
|
15
|
+
meta("site.robots-txt-missing", "warning", "No robots.txt was found at the site root.", 'Add /robots.txt. At minimum: "User-agent: *" with no disallow rules, plus a "Sitemap:" line.', GOOGLE_DOCS.robotsTxt),
|
|
16
|
+
meta("site.robots-txt-error", "warning", 'robots.txt returned a server error or could not be fetched. Google treats 5xx robots.txt as "do not crawl anything".', "Make /robots.txt return 200 (or 404 if you have none).", GOOGLE_DOCS.robotsTxt),
|
|
17
|
+
meta("site.robots-txt-blocks-all", "error", "robots.txt disallows crawling of the whole site for all user agents.", 'Remove "Disallow: /" for "User-agent: *" unless the site is intentionally private (staging).', GOOGLE_DOCS.robotsTxt),
|
|
18
|
+
meta("site.robots-blocked-pages", "warning", "Pages linked internally (or listed in the sitemap) are disallowed by robots.txt.", "Allow the paths in robots.txt, or stop linking to them and remove them from the sitemap if they should stay hidden.", GOOGLE_DOCS.robotsTxt),
|
|
19
|
+
meta("site.sitemap-missing", "warning", "No XML sitemap was found (robots.txt Sitemap: line or /sitemap.xml).", "Generate a sitemap.xml listing canonical, indexable URLs and reference it from robots.txt.", GOOGLE_DOCS.sitemaps),
|
|
20
|
+
meta("site.sitemap-error", "error", "A sitemap could not be fetched or parsed.", "Make the sitemap return 200 with valid XML (urlset or sitemapindex).", GOOGLE_DOCS.sitemaps),
|
|
21
|
+
meta("site.sitemap-noindex-page", "error", "The sitemap lists pages that are set to noindex.", "Either remove noindex (if the page should rank) or remove the URL from the sitemap. Conflicting signals waste crawl budget.", GOOGLE_DOCS.sitemaps),
|
|
22
|
+
meta("site.sitemap-non-canonical", "warning", "The sitemap lists URLs whose canonical points to a different URL.", "List only canonical URLs in the sitemap.", GOOGLE_DOCS.sitemaps),
|
|
23
|
+
meta("site.sitemap-broken-url", "error", "The sitemap lists URLs that return 4xx/5xx.", "Remove dead URLs from the sitemap or fix them.", GOOGLE_DOCS.sitemaps),
|
|
24
|
+
meta("site.sitemap-redirecting-url", "warning", "The sitemap lists URLs that redirect.", "Replace redirecting URLs in the sitemap with their final destination.", GOOGLE_DOCS.sitemaps),
|
|
25
|
+
meta("site.not-in-sitemap", "info", "Indexable pages discovered by crawling are not listed in the sitemap.", "Add the missing URLs to the sitemap (or generate it from the router so it stays complete).", GOOGLE_DOCS.sitemaps),
|
|
26
|
+
meta("site.orphan-pages", "info", "Sitemap URLs that no crawled page links to.", "Link orphan pages from relevant pages or navigation; pages without internal links get little authority and are crawled less.", GOOGLE_DOCS.links),
|
|
27
|
+
meta("site.broken-internal-links", "error", "Internal links point to URLs that return 4xx/5xx or fail to load.", "Fix or remove the links listed in evidence. Each entry shows the broken target and the pages linking to it.", GOOGLE_DOCS.links),
|
|
28
|
+
meta("site.redirected-internal-links", "warning", "Internal links point to URLs that redirect.", "Update the links to the final URL to save a round trip for users and crawlers.", GOOGLE_DOCS.redirects),
|
|
29
|
+
meta("site.redirect-chains", "warning", "Internal URLs go through two or more redirects (or loop).", "Point each redirect straight at its final destination.", GOOGLE_DOCS.redirects),
|
|
30
|
+
meta("site.duplicate-titles", "warning", "Several indexable pages share the same <title>.", "Give each page a unique title that describes that page (templates: include the item name, category and brand).", GOOGLE_DOCS.titles),
|
|
31
|
+
meta("site.duplicate-descriptions", "warning", "Several indexable pages share the same meta description.", "Write a unique description per page, or generate it from page-specific data.", GOOGLE_DOCS.snippets),
|
|
32
|
+
meta("site.duplicate-h1", "info", "Several pages share the same <h1>.", "Make H1s page-specific."),
|
|
33
|
+
meta("site.duplicate-content", "warning", "Pages with identical or near-identical body text.", "Consolidate duplicates with a canonical to one URL, 301 redirect, or make the content distinct.", GOOGLE_DOCS.canonical),
|
|
34
|
+
meta("site.deep-pages", "info", "Pages that are more than the configured number of clicks (default 3) from the start URL.", "Flatten the architecture: link important deep pages from category pages, navigation or the homepage.", GOOGLE_DOCS.links),
|
|
35
|
+
meta("site.soft-404", "warning", "A clearly non-existent URL returns HTTP 200 instead of 404.", "Return a real 404 (or 410) status for unknown routes; otherwise search engines index error pages and waste crawl budget.", GOOGLE_DOCS.softs404),
|
|
36
|
+
meta("site.http-not-redirected", "warning", "The http:// version of the site does not redirect to https://.", "Add a permanent (301/308) redirect from http to https at the server or CDN.", GOOGLE_DOCS.redirects),
|
|
37
|
+
meta("site.host-variant-not-redirected", "warning", "The www/non-www variant of the host does not redirect to the main host.", "Redirect the alternate host (www or bare domain) to the canonical host with a 301/308.", GOOGLE_DOCS.canonical),
|
|
38
|
+
meta("site.hreflang-no-return-link", "warning", "hreflang alternates do not link back to the page that references them.", "Every page in an hreflang cluster must list all alternates including itself; mismatches invalidate the annotations.", GOOGLE_DOCS.hreflang),
|
|
39
|
+
meta("site.broken-external-links", "warning", "External links point to URLs that return 4xx/5xx or fail to load.", "Fix or remove the dead outbound links; they hurt user trust and signal an unmaintained page.", GOOGLE_DOCS.links),
|
|
40
|
+
meta("site.unlinked-pages", "info", "HTML files in the build output that no other page links to.", "Link the pages from navigation or related content, or remove them from the build if they are obsolete.", GOOGLE_DOCS.links),
|
|
41
|
+
meta("site.crawl-limit-reached", "info", "The crawl stopped at the page limit, so site-wide checks (orphans, duplicates, sitemap coverage) are partial.", "Re-run with a higher maxPages value for a complete picture."),
|
|
42
|
+
];
|
|
43
|
+
const byId = new Map(SITE_RULES.map((r) => [r.id, r]));
|
|
44
|
+
const m = (id) => {
|
|
45
|
+
const rule = byId.get(id);
|
|
46
|
+
if (!rule) {
|
|
47
|
+
throw new Error(`Unknown site rule: ${id}`);
|
|
48
|
+
}
|
|
49
|
+
return rule;
|
|
50
|
+
};
|
|
51
|
+
/** Appends `value` to the list under `k`, starting the list when there is none. */
|
|
52
|
+
const pushTo = (groups, k, value) => {
|
|
53
|
+
const list = groups.get(k);
|
|
54
|
+
if (list) {
|
|
55
|
+
list.push(value);
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
groups.set(k, [value]);
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
const pathKey = (url) => {
|
|
62
|
+
try {
|
|
63
|
+
return normalizePath(new URL(url).pathname);
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return url;
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
const pushDuplicates = (out, rule, pages, pick, label) => {
|
|
70
|
+
const groups = new Map();
|
|
71
|
+
for (const p of pages) {
|
|
72
|
+
const value = pick(p);
|
|
73
|
+
if (!value) {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
const k = normalizeForComparison(value);
|
|
77
|
+
pushTo(groups, k, p.url);
|
|
78
|
+
}
|
|
79
|
+
const dupes = [...groups.entries()].filter(([, urls]) => urls.length > 1);
|
|
80
|
+
if (!dupes.length) {
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
const pagesAffected = dupes.reduce((n, [, urls]) => n + urls.length, 0);
|
|
84
|
+
out.push(finding(rule, `${dupes.length} ${label}(s) are shared by ${pagesAffected} pages.`, {
|
|
85
|
+
evidence: {
|
|
86
|
+
groups: cap(dupes.map(([value, urls]) => ({
|
|
87
|
+
value: value.slice(0, 120),
|
|
88
|
+
pages: cap(urls, 5),
|
|
89
|
+
count: urls.length,
|
|
90
|
+
}))),
|
|
91
|
+
},
|
|
92
|
+
}));
|
|
93
|
+
};
|
|
94
|
+
const pushDuplicateContent = (out, pages) => {
|
|
95
|
+
const candidates = pages.filter((p) => p.page.wordCount >= 50);
|
|
96
|
+
const groups = [];
|
|
97
|
+
const exact = new Map();
|
|
98
|
+
for (const p of candidates) {
|
|
99
|
+
const k = normalizeForComparison(p.page.text);
|
|
100
|
+
pushTo(exact, k, p.url);
|
|
101
|
+
}
|
|
102
|
+
const seen = new Set();
|
|
103
|
+
for (const urls of exact.values()) {
|
|
104
|
+
if (urls.length > 1) {
|
|
105
|
+
groups.push({ pages: urls, similarity: 1 });
|
|
106
|
+
for (const u of urls) {
|
|
107
|
+
seen.add(u);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
if (candidates.length <= 150) {
|
|
112
|
+
const sets = candidates.map((p) => ({
|
|
113
|
+
url: p.url,
|
|
114
|
+
set: shingles(p.page.text),
|
|
115
|
+
}));
|
|
116
|
+
for (const [i, a] of sets.entries()) {
|
|
117
|
+
for (const b of sets.slice(i + 1)) {
|
|
118
|
+
if (seen.has(a.url) && seen.has(b.url)) {
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const sim = jaccard(a.set, b.set);
|
|
122
|
+
if (sim >= 0.9) {
|
|
123
|
+
groups.push({
|
|
124
|
+
pages: [a.url, b.url],
|
|
125
|
+
similarity: Number(sim.toFixed(2)),
|
|
126
|
+
});
|
|
127
|
+
seen.add(a.url);
|
|
128
|
+
seen.add(b.url);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (groups.length) {
|
|
134
|
+
out.push(finding(m("site.duplicate-content"), `${groups.length} group(s) of pages have (near-)identical body text.`, { evidence: { groups: cap(groups) } }));
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
const robotsFindings = (ctx) => {
|
|
138
|
+
const out = [];
|
|
139
|
+
/* robots.txt */
|
|
140
|
+
if (ctx.robots) {
|
|
141
|
+
if (ctx.robots.error ||
|
|
142
|
+
(ctx.robots.status !== null && ctx.robots.status >= 500)) {
|
|
143
|
+
out.push(finding(m("site.robots-txt-error"), `robots.txt at ${ctx.robots.url} could not be read (${ctx.robots.error ?? `HTTP ${ctx.robots.status}`}).`, {
|
|
144
|
+
evidence: {
|
|
145
|
+
url: ctx.robots.url,
|
|
146
|
+
status: ctx.robots.status,
|
|
147
|
+
error: ctx.robots.error,
|
|
148
|
+
},
|
|
149
|
+
}));
|
|
150
|
+
}
|
|
151
|
+
else if (!ctx.robots.found) {
|
|
152
|
+
out.push(finding(m("site.robots-txt-missing"), `No robots.txt at ${ctx.robots.url}.`, {
|
|
153
|
+
severity: ctx.source === "directory" ? "info" : "warning",
|
|
154
|
+
evidence: { url: ctx.robots.url, status: ctx.robots.status },
|
|
155
|
+
}));
|
|
156
|
+
}
|
|
157
|
+
else if (ctx.robotsAllows && !ctx.robotsAllows(ctx.startUrl)) {
|
|
158
|
+
out.push(finding(m("site.robots-txt-blocks-all"), "robots.txt blocks the start URL for the crawler user agent.", {
|
|
159
|
+
evidence: {
|
|
160
|
+
url: ctx.robots.url,
|
|
161
|
+
robotsTxt: ctx.robots.text?.slice(0, 500),
|
|
162
|
+
},
|
|
163
|
+
}));
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
if (ctx.blockedByRobots.length) {
|
|
167
|
+
out.push(finding(m("site.robots-blocked-pages"), `${ctx.blockedByRobots.length} linked/sitemap URL(s) are disallowed by robots.txt.`, {
|
|
168
|
+
evidence: {
|
|
169
|
+
count: ctx.blockedByRobots.length,
|
|
170
|
+
examples: cap(ctx.blockedByRobots),
|
|
171
|
+
},
|
|
172
|
+
}));
|
|
173
|
+
}
|
|
174
|
+
return out;
|
|
175
|
+
};
|
|
176
|
+
/** Noindex, non-canonical, broken and redirecting URLs listed in the sitemap. */
|
|
177
|
+
const sitemapUrlFindings = ({ ctx, key, pagesByKey, checksByKey, }) => {
|
|
178
|
+
const out = [];
|
|
179
|
+
const noindex = [];
|
|
180
|
+
const nonCanonical = [];
|
|
181
|
+
const broken = [];
|
|
182
|
+
const redirecting = [];
|
|
183
|
+
for (const smUrl of ctx.sitemaps.urls) {
|
|
184
|
+
const k = key(smUrl);
|
|
185
|
+
const page = pagesByKey.get(k);
|
|
186
|
+
if (page) {
|
|
187
|
+
if (robotsDirectives(page.page, page.response).noindex) {
|
|
188
|
+
noindex.push(smUrl);
|
|
189
|
+
}
|
|
190
|
+
const canon = page.page.canonical;
|
|
191
|
+
if (canon && key(canon) !== k) {
|
|
192
|
+
nonCanonical.push({ url: smUrl, canonical: canon });
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
const check = checksByKey.get(k);
|
|
196
|
+
if (check) {
|
|
197
|
+
if ((check.status ?? 0) >= 400 || check.error) {
|
|
198
|
+
broken.push({ url: smUrl, status: check.status });
|
|
199
|
+
}
|
|
200
|
+
else if (check.redirectChain.length > 0) {
|
|
201
|
+
redirecting.push({ url: smUrl, finalUrl: check.finalUrl });
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
if (noindex.length) {
|
|
206
|
+
out.push(finding(m("site.sitemap-noindex-page"), `${noindex.length} sitemap URL(s) are noindex.`, { evidence: { count: noindex.length, examples: cap(noindex) } }));
|
|
207
|
+
}
|
|
208
|
+
if (nonCanonical.length) {
|
|
209
|
+
out.push(finding(m("site.sitemap-non-canonical"), `${nonCanonical.length} sitemap URL(s) canonicalise to another URL.`, {
|
|
210
|
+
evidence: {
|
|
211
|
+
count: nonCanonical.length,
|
|
212
|
+
examples: cap(nonCanonical),
|
|
213
|
+
},
|
|
214
|
+
}));
|
|
215
|
+
}
|
|
216
|
+
if (broken.length) {
|
|
217
|
+
out.push(finding(m("site.sitemap-broken-url"), `${broken.length} sitemap URL(s) return errors.`, { evidence: { count: broken.length, examples: cap(broken) } }));
|
|
218
|
+
}
|
|
219
|
+
if (redirecting.length) {
|
|
220
|
+
out.push(finding(m("site.sitemap-redirecting-url"), `${redirecting.length} sitemap URL(s) redirect.`, {
|
|
221
|
+
evidence: { count: redirecting.length, examples: cap(redirecting) },
|
|
222
|
+
}));
|
|
223
|
+
}
|
|
224
|
+
return out;
|
|
225
|
+
};
|
|
226
|
+
/** Indexable pages missing from the sitemap, and sitemap URLs nothing links to. */
|
|
227
|
+
const sitemapCoverageFindings = ({ ctx, key, okPages, isIndexable, }) => {
|
|
228
|
+
const out = [];
|
|
229
|
+
const sitemapKeys = new Set(ctx.sitemaps.urls.map(key));
|
|
230
|
+
const notInSitemap = okPages
|
|
231
|
+
.filter((p) => isIndexable(p) &&
|
|
232
|
+
!sitemapKeys.has(key(p.url)) &&
|
|
233
|
+
(!p.page.canonical || key(p.page.canonical) === key(p.url)))
|
|
234
|
+
.map((p) => p.url);
|
|
235
|
+
if (notInSitemap.length) {
|
|
236
|
+
out.push(finding(m("site.not-in-sitemap"), `${notInSitemap.length} crawled indexable page(s) are missing from the sitemap.`, {
|
|
237
|
+
evidence: {
|
|
238
|
+
count: notInSitemap.length,
|
|
239
|
+
examples: cap(notInSitemap),
|
|
240
|
+
},
|
|
241
|
+
}));
|
|
242
|
+
}
|
|
243
|
+
if (!ctx.crawl.hitPageLimit) {
|
|
244
|
+
const linked = new Set();
|
|
245
|
+
for (const target of ctx.linkSources.keys()) {
|
|
246
|
+
linked.add(key(target));
|
|
247
|
+
}
|
|
248
|
+
linked.add(key(ctx.startUrl));
|
|
249
|
+
const orphans = ctx.sitemaps.urls.filter((u) => !linked.has(key(u)));
|
|
250
|
+
if (orphans.length) {
|
|
251
|
+
out.push(finding(m("site.orphan-pages"), `${orphans.length} sitemap URL(s) are not linked from any crawled page.`, { evidence: { count: orphans.length, examples: cap(orphans) } }));
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return out;
|
|
255
|
+
};
|
|
256
|
+
const sitemapFindings = (scope) => {
|
|
257
|
+
const { ctx } = scope;
|
|
258
|
+
const out = [];
|
|
259
|
+
if (ctx.sitemaps.errors.length) {
|
|
260
|
+
out.push(finding(m("site.sitemap-error"), `${ctx.sitemaps.errors.length} sitemap problem(s).`, { evidence: { errors: cap(ctx.sitemaps.errors) } }));
|
|
261
|
+
}
|
|
262
|
+
if (ctx.sitemaps.sources.length === 0 && ctx.sitemaps.errors.length === 0) {
|
|
263
|
+
out.push(finding(m("site.sitemap-missing"), "No sitemap was found.", {
|
|
264
|
+
severity: ctx.source === "directory" ? "info" : "warning",
|
|
265
|
+
}));
|
|
266
|
+
}
|
|
267
|
+
if (ctx.sitemaps.urlCount > 0) {
|
|
268
|
+
out.push(...sitemapUrlFindings(scope), ...sitemapCoverageFindings(scope));
|
|
269
|
+
}
|
|
270
|
+
return out;
|
|
271
|
+
};
|
|
272
|
+
/** Internal link targets that are broken, redirect once, or redirect in a chain. */
|
|
273
|
+
const linkHealthFindings = (ctx) => {
|
|
274
|
+
const out = [];
|
|
275
|
+
/* link health */
|
|
276
|
+
const broken = [];
|
|
277
|
+
const redirected = [];
|
|
278
|
+
const chains = [];
|
|
279
|
+
for (const [k, check] of ctx.linkChecks) {
|
|
280
|
+
const sources = [...(ctx.linkSources.get(k) ?? [])];
|
|
281
|
+
if (sources.length === 0) {
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
// Chains of two or more hops are handled first, so a first hop here is the only one.
|
|
285
|
+
const [firstHop] = check.redirectChain;
|
|
286
|
+
if (check.error || (check.status ?? 0) >= 400) {
|
|
287
|
+
broken.push({
|
|
288
|
+
url: check.url,
|
|
289
|
+
status: check.status,
|
|
290
|
+
error: check.error,
|
|
291
|
+
linkedFrom: cap(sources, 3),
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
else if (check.redirectChain.length >= 2) {
|
|
295
|
+
chains.push({
|
|
296
|
+
url: check.url,
|
|
297
|
+
hops: check.redirectChain.length,
|
|
298
|
+
finalUrl: check.finalUrl,
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
else if (firstHop) {
|
|
302
|
+
redirected.push({
|
|
303
|
+
url: check.url,
|
|
304
|
+
finalUrl: check.finalUrl,
|
|
305
|
+
status: firstHop.status,
|
|
306
|
+
linkedFrom: cap(sources, 3),
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
if (broken.length) {
|
|
311
|
+
out.push(finding(m("site.broken-internal-links"), `${broken.length} internal link target(s) are broken.`, { evidence: { count: broken.length, examples: cap(broken, 20) } }));
|
|
312
|
+
}
|
|
313
|
+
if (redirected.length) {
|
|
314
|
+
out.push(finding(m("site.redirected-internal-links"), `${redirected.length} internal link target(s) redirect.`, {
|
|
315
|
+
evidence: { count: redirected.length, examples: cap(redirected, 20) },
|
|
316
|
+
}));
|
|
317
|
+
}
|
|
318
|
+
if (chains.length) {
|
|
319
|
+
out.push(finding(m("site.redirect-chains"), `${chains.length} internal URL(s) have redirect chains.`, { evidence: { count: chains.length, examples: cap(chains) } }));
|
|
320
|
+
}
|
|
321
|
+
return out;
|
|
322
|
+
};
|
|
323
|
+
/** Broken external links (when they were probed) and HTML files nothing links to. */
|
|
324
|
+
const unreachedFindings = (ctx) => {
|
|
325
|
+
const out = [];
|
|
326
|
+
if (ctx.externalChecks) {
|
|
327
|
+
const dead = [];
|
|
328
|
+
for (const [u, check] of ctx.externalChecks) {
|
|
329
|
+
if (check.error || (check.status ?? 0) >= 400) {
|
|
330
|
+
dead.push({
|
|
331
|
+
url: u,
|
|
332
|
+
status: check.status,
|
|
333
|
+
error: check.error,
|
|
334
|
+
linkedFrom: cap([...(ctx.externalSources?.get(u) ?? [])], 3),
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
if (dead.length) {
|
|
339
|
+
out.push(finding(m("site.broken-external-links"), `${dead.length} external link target(s) are broken.`, { evidence: { count: dead.length, examples: cap(dead, 20) } }));
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
if (ctx.unlinkedPages?.length) {
|
|
343
|
+
out.push(finding(m("site.unlinked-pages"), `${ctx.unlinkedPages.length} HTML file(s) are not linked from any other page.`, {
|
|
344
|
+
evidence: {
|
|
345
|
+
count: ctx.unlinkedPages.length,
|
|
346
|
+
examples: cap(ctx.unlinkedPages),
|
|
347
|
+
},
|
|
348
|
+
}));
|
|
349
|
+
}
|
|
350
|
+
return out;
|
|
351
|
+
};
|
|
352
|
+
const duplicateFindings = ({ okPages, isIndexable }) => {
|
|
353
|
+
const out = [];
|
|
354
|
+
/* duplicates */
|
|
355
|
+
const indexable = okPages.filter(isIndexable);
|
|
356
|
+
pushDuplicates(out, m("site.duplicate-titles"), indexable, (p) => p.page.title, "title");
|
|
357
|
+
pushDuplicates(out, m("site.duplicate-descriptions"), indexable, (p) => p.page.metaDescription, "meta description");
|
|
358
|
+
pushDuplicates(out, m("site.duplicate-h1"), indexable, (p) => p.page.h1s[0] ?? null, "H1");
|
|
359
|
+
pushDuplicateContent(out, indexable);
|
|
360
|
+
return out;
|
|
361
|
+
};
|
|
362
|
+
/** Pages deep in the site, and how the server answers a made-up URL and the other host names. */
|
|
363
|
+
const architectureFindings = ({ ctx, okPages }) => {
|
|
364
|
+
const out = [];
|
|
365
|
+
/* architecture */
|
|
366
|
+
const deep = okPages.filter((p) => p.depth > ctx.thresholds.deepPageDepth);
|
|
367
|
+
if (deep.length) {
|
|
368
|
+
out.push(finding(m("site.deep-pages"), `${deep.length} page(s) are more than ${ctx.thresholds.deepPageDepth} clicks from the start URL.`, {
|
|
369
|
+
evidence: {
|
|
370
|
+
count: deep.length,
|
|
371
|
+
examples: cap(deep.map((p) => ({ url: p.url, depth: p.depth }))),
|
|
372
|
+
},
|
|
373
|
+
}));
|
|
374
|
+
}
|
|
375
|
+
/* probes */
|
|
376
|
+
if (ctx.probes.soft404 &&
|
|
377
|
+
ctx.probes.soft404.status !== null &&
|
|
378
|
+
ctx.probes.soft404.status < 400) {
|
|
379
|
+
out.push(finding(m("site.soft-404"), `${ctx.probes.soft404.url} returned HTTP ${ctx.probes.soft404.status}.`, { evidence: ctx.probes.soft404 }));
|
|
380
|
+
}
|
|
381
|
+
if (ctx.probes.httpRedirect &&
|
|
382
|
+
ctx.probes.httpRedirect.status !== null &&
|
|
383
|
+
!ctx.probes.httpRedirect.redirectsToHttps) {
|
|
384
|
+
out.push(finding(m("site.http-not-redirected"), `${ctx.probes.httpRedirect.url} returned HTTP ${ctx.probes.httpRedirect.status} without redirecting to https.`, { evidence: ctx.probes.httpRedirect }));
|
|
385
|
+
}
|
|
386
|
+
if (ctx.probes.hostVariant &&
|
|
387
|
+
ctx.probes.hostVariant.status !== null &&
|
|
388
|
+
!ctx.probes.hostVariant.redirectsToMain) {
|
|
389
|
+
out.push(finding(m("site.host-variant-not-redirected"), `${ctx.probes.hostVariant.url} returned HTTP ${ctx.probes.hostVariant.status} without redirecting to the main host.`, { evidence: ctx.probes.hostVariant }));
|
|
390
|
+
}
|
|
391
|
+
return out;
|
|
392
|
+
};
|
|
393
|
+
const hreflangFindings = ({ ctx, key, pagesByKey }) => {
|
|
394
|
+
const out = [];
|
|
395
|
+
/* hreflang return links */
|
|
396
|
+
const missingReturn = [];
|
|
397
|
+
for (const p of ctx.pages) {
|
|
398
|
+
for (const alt of p.page.hreflang) {
|
|
399
|
+
if (!alt.url) {
|
|
400
|
+
continue;
|
|
401
|
+
}
|
|
402
|
+
const target = pagesByKey.get(key(alt.url));
|
|
403
|
+
if (!target || target === p) {
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
const linksBack = target.page.hreflang.some((h) => h.url && key(h.url) === key(p.url));
|
|
407
|
+
if (!linksBack) {
|
|
408
|
+
missingReturn.push({ page: p.url, alternate: alt.url });
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
if (missingReturn.length) {
|
|
413
|
+
out.push(finding(m("site.hreflang-no-return-link"), `${missingReturn.length} hreflang alternate(s) do not link back.`, {
|
|
414
|
+
evidence: {
|
|
415
|
+
count: missingReturn.length,
|
|
416
|
+
examples: cap(missingReturn),
|
|
417
|
+
},
|
|
418
|
+
}));
|
|
419
|
+
}
|
|
420
|
+
if (ctx.crawl.hitPageLimit) {
|
|
421
|
+
out.push(finding(m("site.crawl-limit-reached"), `Crawl stopped after ${ctx.crawl.pagesCrawled} pages (limit ${ctx.crawl.maxPages}); ${ctx.crawl.pagesDiscovered} URLs were discovered.`, {
|
|
422
|
+
evidence: {
|
|
423
|
+
pagesCrawled: ctx.crawl.pagesCrawled,
|
|
424
|
+
pagesDiscovered: ctx.crawl.pagesDiscovered,
|
|
425
|
+
maxPages: ctx.crawl.maxPages,
|
|
426
|
+
},
|
|
427
|
+
}));
|
|
428
|
+
}
|
|
429
|
+
return out;
|
|
430
|
+
};
|
|
431
|
+
export const runSiteRules = (ctx) => {
|
|
432
|
+
const key = (url) => ctx.pathOnly ? pathKey(url) : (normalizeUrl(url) ?? url);
|
|
433
|
+
const pagesByKey = new Map();
|
|
434
|
+
for (const p of ctx.pages) {
|
|
435
|
+
pagesByKey.set(key(p.url), p);
|
|
436
|
+
}
|
|
437
|
+
const checksByKey = new Map();
|
|
438
|
+
for (const [u, c] of ctx.linkChecks) {
|
|
439
|
+
if (!checksByKey.has(key(u))) {
|
|
440
|
+
checksByKey.set(key(u), c);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
const scope = {
|
|
444
|
+
ctx,
|
|
445
|
+
key,
|
|
446
|
+
pagesByKey,
|
|
447
|
+
checksByKey,
|
|
448
|
+
okPages: ctx.pages.filter((p) => (p.response?.status ?? 200) < 400),
|
|
449
|
+
isIndexable: (p) => !robotsDirectives(p.page, p.response).noindex &&
|
|
450
|
+
(p.response?.status ?? 200) < 400,
|
|
451
|
+
};
|
|
452
|
+
return [
|
|
453
|
+
...robotsFindings(ctx),
|
|
454
|
+
...sitemapFindings(scope),
|
|
455
|
+
...linkHealthFindings(ctx),
|
|
456
|
+
...unreachedFindings(ctx),
|
|
457
|
+
...duplicateFindings(scope),
|
|
458
|
+
...architectureFindings(scope),
|
|
459
|
+
...hreflangFindings(scope),
|
|
460
|
+
];
|
|
461
|
+
};
|
|
462
|
+
//# sourceMappingURL=site-rules.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"site-rules.js","sourceRoot":"","sources":["../../src/core/site-rules.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAW5D,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEvD,MAAM,IAAI,GAAG,CACX,EAAU,EACV,QAA8B,EAC9B,WAAmB,EACnB,cAAsB,EACtB,IAAa,EACH,EAAE,CAAC,CAAC;IACd,EAAE;IACF,QAAQ,EAAE,MAAM;IAChB,QAAQ;IACR,WAAW;IACX,cAAc;IACd,IAAI;IACJ,QAAQ,EAAE,OAAO;CAClB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAe;IACpC,IAAI,CACF,yBAAyB,EACzB,SAAS,EACT,2CAA2C,EAC3C,8FAA8F,EAC9F,WAAW,CAAC,SAAS,CACtB;IACD,IAAI,CACF,uBAAuB,EACvB,SAAS,EACT,sHAAsH,EACtH,wDAAwD,EACxD,WAAW,CAAC,SAAS,CACtB;IACD,IAAI,CACF,4BAA4B,EAC5B,OAAO,EACP,sEAAsE,EACtE,8FAA8F,EAC9F,WAAW,CAAC,SAAS,CACtB;IACD,IAAI,CACF,2BAA2B,EAC3B,SAAS,EACT,kFAAkF,EAClF,qHAAqH,EACrH,WAAW,CAAC,SAAS,CACtB;IACD,IAAI,CACF,sBAAsB,EACtB,SAAS,EACT,sEAAsE,EACtE,4FAA4F,EAC5F,WAAW,CAAC,QAAQ,CACrB;IACD,IAAI,CACF,oBAAoB,EACpB,OAAO,EACP,2CAA2C,EAC3C,sEAAsE,EACtE,WAAW,CAAC,QAAQ,CACrB;IACD,IAAI,CACF,2BAA2B,EAC3B,OAAO,EACP,kDAAkD,EAClD,6HAA6H,EAC7H,WAAW,CAAC,QAAQ,CACrB;IACD,IAAI,CACF,4BAA4B,EAC5B,SAAS,EACT,mEAAmE,EACnE,0CAA0C,EAC1C,WAAW,CAAC,QAAQ,CACrB;IACD,IAAI,CACF,yBAAyB,EACzB,OAAO,EACP,6CAA6C,EAC7C,gDAAgD,EAChD,WAAW,CAAC,QAAQ,CACrB;IACD,IAAI,CACF,8BAA8B,EAC9B,SAAS,EACT,uCAAuC,EACvC,uEAAuE,EACvE,WAAW,CAAC,QAAQ,CACrB;IACD,IAAI,CACF,qBAAqB,EACrB,MAAM,EACN,uEAAuE,EACvE,4FAA4F,EAC5F,WAAW,CAAC,QAAQ,CACrB;IACD,IAAI,CACF,mBAAmB,EACnB,MAAM,EACN,6CAA6C,EAC7C,8HAA8H,EAC9H,WAAW,CAAC,KAAK,CAClB;IACD,IAAI,CACF,4BAA4B,EAC5B,OAAO,EACP,mEAAmE,EACnE,6GAA6G,EAC7G,WAAW,CAAC,KAAK,CAClB;IACD,IAAI,CACF,gCAAgC,EAChC,SAAS,EACT,6CAA6C,EAC7C,gFAAgF,EAChF,WAAW,CAAC,SAAS,CACtB;IACD,IAAI,CACF,sBAAsB,EACtB,SAAS,EACT,2DAA2D,EAC3D,wDAAwD,EACxD,WAAW,CAAC,SAAS,CACtB;IACD,IAAI,CACF,uBAAuB,EACvB,SAAS,EACT,iDAAiD,EACjD,gHAAgH,EAChH,WAAW,CAAC,MAAM,CACnB;IACD,IAAI,CACF,6BAA6B,EAC7B,SAAS,EACT,0DAA0D,EAC1D,8EAA8E,EAC9E,WAAW,CAAC,QAAQ,CACrB;IACD,IAAI,CACF,mBAAmB,EACnB,MAAM,EACN,oCAAoC,EACpC,yBAAyB,CAC1B;IACD,IAAI,CACF,wBAAwB,EACxB,SAAS,EACT,mDAAmD,EACnD,iGAAiG,EACjG,WAAW,CAAC,SAAS,CACtB;IACD,IAAI,CACF,iBAAiB,EACjB,MAAM,EACN,0FAA0F,EAC1F,sGAAsG,EACtG,WAAW,CAAC,KAAK,CAClB;IACD,IAAI,CACF,eAAe,EACf,SAAS,EACT,6DAA6D,EAC7D,0HAA0H,EAC1H,WAAW,CAAC,QAAQ,CACrB;IACD,IAAI,CACF,0BAA0B,EAC1B,SAAS,EACT,gEAAgE,EAChE,6EAA6E,EAC7E,WAAW,CAAC,SAAS,CACtB;IACD,IAAI,CACF,kCAAkC,EAClC,SAAS,EACT,yEAAyE,EACzE,wFAAwF,EACxF,WAAW,CAAC,SAAS,CACtB;IACD,IAAI,CACF,8BAA8B,EAC9B,SAAS,EACT,wEAAwE,EACxE,qHAAqH,EACrH,WAAW,CAAC,QAAQ,CACrB;IACD,IAAI,CACF,4BAA4B,EAC5B,SAAS,EACT,mEAAmE,EACnE,8FAA8F,EAC9F,WAAW,CAAC,KAAK,CAClB;IACD,IAAI,CACF,qBAAqB,EACrB,MAAM,EACN,6DAA6D,EAC7D,wGAAwG,EACxG,WAAW,CAAC,KAAK,CAClB;IACD,IAAI,CACF,0BAA0B,EAC1B,MAAM,EACN,+GAA+G,EAC/G,6DAA6D,CAC9D;CACF,CAAC;AAEF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAEvD,MAAM,CAAC,GAAG,CAAC,EAAU,EAAY,EAAE;IACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAE1B,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AA6DF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CACb,MAA6B,EAC7B,CAAS,EACT,KAAa,EACP,EAAE;IACR,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAE3B,IAAI,IAAI,EAAE,CAAC;QACT,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;IACzB,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAC,GAAW,EAAU,EAAE;IACtC,IAAI,CAAC;QACH,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CACrB,GAAc,EACd,IAAc,EACd,KAAmB,EACnB,IAAsC,EACtC,KAAa,EACP,EAAE;IACR,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE3C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QAED,MAAM,CAAC,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAExC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE1E,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO;IACT,CAAC;IAED,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACxE,GAAG,CAAC,IAAI,CACN,OAAO,CACL,IAAI,EACJ,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,qBAAqB,aAAa,SAAS,EACnE;QACE,QAAQ,EAAE;YACR,MAAM,EAAE,GAAG,CACT,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC5B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBAC1B,KAAK,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBACnB,KAAK,EAAE,IAAI,CAAC,MAAM;aACnB,CAAC,CAAC,CACJ;SACF;KACF,CACF,CACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,GAAc,EAAE,KAAmB,EAAQ,EAAE;IACzE,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAC/D,MAAM,MAAM,GAA8C,EAAE,CAAC;IAC7D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE1C,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;YAE5C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAClC,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC,CAAC;QAEJ,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACpC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAClC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvC,SAAS;gBACX,CAAC;gBAED,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;gBAElC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,CAAC;wBACV,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC;wBACrB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;qBACnC,CAAC,CAAC;oBACH,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBAChB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,wBAAwB,CAAC,EAC3B,GAAG,MAAM,CAAC,MAAM,qDAAqD,EACrE,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CACtC,CACF,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAaF,MAAM,cAAc,GAAG,CAAC,GAAgB,EAAa,EAAE;IACrD,MAAM,GAAG,GAAc,EAAE,CAAC;IAE1B,gBAAgB;IAChB,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QACf,IACE,GAAG,CAAC,MAAM,CAAC,KAAK;YAChB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,EACxD,CAAC;YACD,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,uBAAuB,CAAC,EAC1B,iBAAiB,GAAG,CAAC,MAAM,CAAC,GAAG,uBAAuB,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,QAAQ,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EACzG;gBACE,QAAQ,EAAE;oBACR,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG;oBACnB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;oBACzB,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK;iBACxB;aACF,CACF,CACF,CAAC;QACJ,CAAC;aAAM,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC7B,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,yBAAyB,CAAC,EAC5B,oBAAoB,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,EACrC;gBACE,QAAQ,EAAE,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBACzD,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE;aAC7D,CACF,CACF,CAAC;QACJ,CAAC;aAAM,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,4BAA4B,CAAC,EAC/B,6DAA6D,EAC7D;gBACE,QAAQ,EAAE;oBACR,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG;oBACnB,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;iBAC1C;aACF,CACF,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,GAAG,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,2BAA2B,CAAC,EAC9B,GAAG,GAAG,CAAC,eAAe,CAAC,MAAM,sDAAsD,EACnF;YACE,QAAQ,EAAE;gBACR,KAAK,EAAE,GAAG,CAAC,eAAe,CAAC,MAAM;gBACjC,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC;aACnC;SACF,CACF,CACF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,iFAAiF;AACjF,MAAM,kBAAkB,GAAG,CAAC,EAC1B,GAAG,EACH,GAAG,EACH,UAAU,EACV,WAAW,GACD,EAAa,EAAE;IACzB,MAAM,GAAG,GAAc,EAAE,CAAC;IAE1B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,YAAY,GAAyC,EAAE,CAAC;IAC9D,MAAM,MAAM,GAA6C,EAAE,CAAC;IAC5D,MAAM,WAAW,GAAwC,EAAE,CAAC;IAE5D,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAE/B,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;gBACvD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;YAED,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YAElC,IAAI,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,GAAG,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACpD,CAAC;iBAAM,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,WAAW,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,2BAA2B,CAAC,EAC9B,GAAG,OAAO,CAAC,MAAM,8BAA8B,EAC/C,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,CAChE,CACF,CAAC;IACJ,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;QACxB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,4BAA4B,CAAC,EAC/B,GAAG,YAAY,CAAC,MAAM,8CAA8C,EACpE;YACE,QAAQ,EAAE;gBACR,KAAK,EAAE,YAAY,CAAC,MAAM;gBAC1B,QAAQ,EAAE,GAAG,CAAC,YAAY,CAAC;aAC5B;SACF,CACF,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,yBAAyB,CAAC,EAC5B,GAAG,MAAM,CAAC,MAAM,gCAAgC,EAChD,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAC9D,CACF,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACvB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,8BAA8B,CAAC,EACjC,GAAG,WAAW,CAAC,MAAM,2BAA2B,EAChD;YACE,QAAQ,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,WAAW,CAAC,EAAE;SACpE,CACF,CACF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,uBAAuB,GAAG,CAAC,EAC/B,GAAG,EACH,GAAG,EACH,OAAO,EACP,WAAW,GACD,EAAa,EAAE;IACzB,MAAM,GAAG,GAAc,EAAE,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAExD,MAAM,YAAY,GAAG,OAAO;SACzB,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,WAAW,CAAC,CAAC,CAAC;QACd,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAC9D;SACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAErB,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;QACxB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,qBAAqB,CAAC,EACxB,GAAG,YAAY,CAAC,MAAM,0DAA0D,EAChF;YACE,QAAQ,EAAE;gBACR,KAAK,EAAE,YAAY,CAAC,MAAM;gBAC1B,QAAQ,EAAE,GAAG,CAAC,YAAY,CAAC;aAC5B;SACF,CACF,CACF,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;QAEjC,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;YAC5C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAErE,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,mBAAmB,CAAC,EACtB,GAAG,OAAO,CAAC,MAAM,uDAAuD,EACxE,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,CAChE,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,KAAgB,EAAa,EAAE;IACtD,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;IACtB,MAAM,GAAG,GAAc,EAAE,CAAC;IAE1B,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,oBAAoB,CAAC,EACvB,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,sBAAsB,EACnD,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,CACnD,CACF,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,IAAI,CACN,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAAC,EAAE,uBAAuB,EAAE;YAC1D,QAAQ,EAAE,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;SAC1D,CAAC,CACH,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,KAAK,CAAC,EAAE,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,oFAAoF;AACpF,MAAM,kBAAkB,GAAG,CAAC,GAAgB,EAAa,EAAE;IACzD,MAAM,GAAG,GAAc,EAAE,CAAC;IAE1B,iBAAiB;IACjB,MAAM,MAAM,GAKN,EAAE,CAAC;IAET,MAAM,UAAU,GAKV,EAAE,CAAC;IAET,MAAM,MAAM,GAAsD,EAAE,CAAC;IAErE,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,SAAS;QACX,CAAC;QAED,qFAAqF;QACrF,MAAM,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC;QAEvC,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;YAC9C,MAAM,CAAC,IAAI,CAAC;gBACV,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,UAAU,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC;gBACV,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,aAAa,CAAC,MAAM;gBAChC,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,QAAQ,EAAE,CAAC;YACpB,UAAU,CAAC,IAAI,CAAC;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,UAAU,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,4BAA4B,CAAC,EAC/B,GAAG,MAAM,CAAC,MAAM,sCAAsC,EACtD,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAClE,CACF,CAAC;IACJ,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,gCAAgC,CAAC,EACnC,GAAG,UAAU,CAAC,MAAM,oCAAoC,EACxD;YACE,QAAQ,EAAE,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE;SACtE,CACF,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,sBAAsB,CAAC,EACzB,GAAG,MAAM,CAAC,MAAM,wCAAwC,EACxD,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAC9D,CACF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,qFAAqF;AACrF,MAAM,iBAAiB,GAAG,CAAC,GAAgB,EAAa,EAAE;IACxD,MAAM,GAAG,GAAc,EAAE,CAAC;IAE1B,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;QACvB,MAAM,IAAI,GAKJ,EAAE,CAAC;QAET,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC;oBACR,GAAG,EAAE,CAAC;oBACN,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,UAAU,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;iBAC7D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,4BAA4B,CAAC,EAC/B,GAAG,IAAI,CAAC,MAAM,sCAAsC,EACpD,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAC9D,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,qBAAqB,CAAC,EACxB,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,mDAAmD,EAC9E;YACE,QAAQ,EAAE;gBACR,KAAK,EAAE,GAAG,CAAC,aAAa,CAAC,MAAM;gBAC/B,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC;aACjC;SACF,CACF,CACF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,EAAE,OAAO,EAAE,WAAW,EAAa,EAAa,EAAE;IAC3E,MAAM,GAAG,GAAc,EAAE,CAAC;IAE1B,gBAAgB;IAChB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC9C,cAAc,CACZ,GAAG,EACH,CAAC,CAAC,uBAAuB,CAAC,EAC1B,SAAS,EACT,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,EACnB,OAAO,CACR,CAAC;IACF,cAAc,CACZ,GAAG,EACH,CAAC,CAAC,6BAA6B,CAAC,EAChC,SAAS,EACT,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,EAC7B,kBAAkB,CACnB,CAAC;IACF,cAAc,CACZ,GAAG,EACH,CAAC,CAAC,mBAAmB,CAAC,EACtB,SAAS,EACT,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,EAC5B,IAAI,CACL,CAAC;IACF,oBAAoB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAErC,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,iGAAiG;AACjG,MAAM,oBAAoB,GAAG,CAAC,EAAE,GAAG,EAAE,OAAO,EAAa,EAAa,EAAE;IACtE,MAAM,GAAG,GAAc,EAAE,CAAC;IAE1B,kBAAkB;IAClB,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IAE3E,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,iBAAiB,CAAC,EACpB,GAAG,IAAI,CAAC,MAAM,0BAA0B,GAAG,CAAC,UAAU,CAAC,aAAa,6BAA6B,EACjG;YACE,QAAQ,EAAE;gBACR,KAAK,EAAE,IAAI,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;aACjE;SACF,CACF,CACF,CAAC;IACJ,CAAC;IAED,YAAY;IACZ,IACE,GAAG,CAAC,MAAM,CAAC,OAAO;QAClB,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,IAAI;QAClC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,EAC/B,CAAC;QACD,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,eAAe,CAAC,EAClB,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,kBAAkB,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,EACvE,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,CACjC,CACF,CAAC;IACJ,CAAC;IAED,IACE,GAAG,CAAC,MAAM,CAAC,YAAY;QACvB,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,KAAK,IAAI;QACvC,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,gBAAgB,EACzC,CAAC;QACD,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,0BAA0B,CAAC,EAC7B,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,kBAAkB,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,gCAAgC,EAC9G,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,YAAY,EAAE,CACtC,CACF,CAAC;IACJ,CAAC;IAED,IACE,GAAG,CAAC,MAAM,CAAC,WAAW;QACtB,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,IAAI;QACtC,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,eAAe,EACvC,CAAC;QACD,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,kCAAkC,CAAC,EACrC,GAAG,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,kBAAkB,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,wCAAwC,EACpH,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CACrC,CACF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAa,EAAa,EAAE;IAC1E,MAAM,GAAG,GAAc,EAAE,CAAC;IAE1B,2BAA2B;IAC3B,MAAM,aAAa,GAA0C,EAAE,CAAC;IAEhE,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QAC1B,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;gBACb,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAE5C,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CACzC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAC1C,CAAC;YAEF,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;QACzB,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,8BAA8B,CAAC,EACjC,GAAG,aAAa,CAAC,MAAM,0CAA0C,EACjE;YACE,QAAQ,EAAE;gBACR,KAAK,EAAE,aAAa,CAAC,MAAM;gBAC3B,QAAQ,EAAE,GAAG,CAAC,aAAa,CAAC;aAC7B;SACF,CACF,CACF,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QAC3B,GAAG,CAAC,IAAI,CACN,OAAO,CACL,CAAC,CAAC,0BAA0B,CAAC,EAC7B,uBAAuB,GAAG,CAAC,KAAK,CAAC,YAAY,iBAAiB,GAAG,CAAC,KAAK,CAAC,QAAQ,MAAM,GAAG,CAAC,KAAK,CAAC,eAAe,wBAAwB,EACvI;YACE,QAAQ,EAAE;gBACR,YAAY,EAAE,GAAG,CAAC,KAAK,CAAC,YAAY;gBACpC,eAAe,EAAE,GAAG,CAAC,KAAK,CAAC,eAAe;gBAC1C,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,QAAQ;aAC7B;SACF,CACF,CACF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,GAAgB,EAAa,EAAE;IAC1D,MAAM,GAAG,GAAG,CAAC,GAAW,EAAU,EAAE,CAClC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC;IAE3D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEjD,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QAC1B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,GAAG,EAAqB,CAAC;IAEjD,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAc;QACvB,GAAG;QACH,GAAG;QACH,UAAU;QACV,WAAW;QACX,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC;QACnE,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CACjB,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO;YAC7C,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG;KACpC,CAAC;IAEF,OAAO;QACL,GAAG,cAAc,CAAC,GAAG,CAAC;QACtB,GAAG,eAAe,CAAC,KAAK,CAAC;QACzB,GAAG,kBAAkB,CAAC,GAAG,CAAC;QAC1B,GAAG,iBAAiB,CAAC,GAAG,CAAC;QACzB,GAAG,iBAAiB,CAAC,KAAK,CAAC;QAC3B,GAAG,oBAAoB,CAAC,KAAK,CAAC;QAC9B,GAAG,gBAAgB,CAAC,KAAK,CAAC;KAC3B,CAAC;AACJ,CAAC,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { FetchOptions } from "./fetch.js";
|
|
2
|
+
import type { SitemapInfo } from "./types.js";
|
|
3
|
+
export interface SitemapLimits {
|
|
4
|
+
maxSitemaps?: number;
|
|
5
|
+
maxUrls?: number;
|
|
6
|
+
}
|
|
7
|
+
export interface ParsedSitemap {
|
|
8
|
+
kind: "urlset" | "sitemapindex" | "text" | "unknown";
|
|
9
|
+
locs: string[];
|
|
10
|
+
}
|
|
11
|
+
export declare const parseSitemap: (body: string, contentType?: string | null) => ParsedSitemap;
|
|
12
|
+
/** Fetch sitemap candidates (following sitemap indexes) and collect page URLs. */
|
|
13
|
+
export declare const loadSitemaps: (candidates: string[], options?: FetchOptions, limits?: SitemapLimits) => Promise<SitemapInfo>;
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { XMLParser } from "fast-xml-parser";
|
|
2
|
+
import { fetchPage } from "./fetch.js";
|
|
3
|
+
const parser = new XMLParser({
|
|
4
|
+
ignoreAttributes: true,
|
|
5
|
+
isArray: (name) => name === "url" || name === "sitemap",
|
|
6
|
+
});
|
|
7
|
+
const isString = (value) => typeof value === "string";
|
|
8
|
+
const isNumber = (value) => typeof value === "number";
|
|
9
|
+
const isEntry = (value) => Boolean(value) && typeof value === "object";
|
|
10
|
+
const locsOf = (entries) => {
|
|
11
|
+
const out = [];
|
|
12
|
+
for (const e of entries) {
|
|
13
|
+
const loc = isEntry(e) ? e.loc : undefined;
|
|
14
|
+
if (isString(loc) && loc.trim()) {
|
|
15
|
+
out.push(loc.trim());
|
|
16
|
+
}
|
|
17
|
+
else if (isNumber(loc)) {
|
|
18
|
+
out.push(String(loc));
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return out;
|
|
22
|
+
};
|
|
23
|
+
export const parseSitemap = (body, contentType = null) => {
|
|
24
|
+
const trimmed = body.trim();
|
|
25
|
+
if (!trimmed) {
|
|
26
|
+
return { kind: "unknown", locs: [] };
|
|
27
|
+
}
|
|
28
|
+
if (!trimmed.startsWith("<")) {
|
|
29
|
+
if ((contentType ?? "").includes("text/plain") ||
|
|
30
|
+
/^https?:\/\//mu.test(trimmed)) {
|
|
31
|
+
return {
|
|
32
|
+
kind: "text",
|
|
33
|
+
locs: trimmed
|
|
34
|
+
.split(/\r?\n/u)
|
|
35
|
+
.map((l) => l.trim())
|
|
36
|
+
.filter((l) => /^https?:\/\//iu.test(l)),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
return { kind: "unknown", locs: [] };
|
|
40
|
+
}
|
|
41
|
+
let doc;
|
|
42
|
+
try {
|
|
43
|
+
doc = parser.parse(trimmed);
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return { kind: "unknown", locs: [] };
|
|
47
|
+
}
|
|
48
|
+
if (doc.sitemapindex) {
|
|
49
|
+
const entries = doc.sitemapindex.sitemap;
|
|
50
|
+
return {
|
|
51
|
+
kind: "sitemapindex",
|
|
52
|
+
locs: Array.isArray(entries) ? locsOf(entries) : [],
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
if (doc.urlset) {
|
|
56
|
+
const entries = doc.urlset.url;
|
|
57
|
+
return {
|
|
58
|
+
kind: "urlset",
|
|
59
|
+
locs: Array.isArray(entries) ? locsOf(entries) : [],
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
return { kind: "unknown", locs: [] };
|
|
63
|
+
};
|
|
64
|
+
const fetchSitemap = async (url, candidates, options) => {
|
|
65
|
+
if (url.endsWith(".gz")) {
|
|
66
|
+
return {
|
|
67
|
+
kind: "error",
|
|
68
|
+
message: `${url}: gzip sitemaps are not supported yet`,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
const res = await fetchPage(url, {
|
|
72
|
+
...options,
|
|
73
|
+
timeoutMs: Math.min(options.timeoutMs ?? 15_000, 20_000),
|
|
74
|
+
maxBodyBytes: 15_000_000,
|
|
75
|
+
});
|
|
76
|
+
if (res.error) {
|
|
77
|
+
return { kind: "error", message: `${url}: ${res.error}` };
|
|
78
|
+
}
|
|
79
|
+
if (res.status !== 200 || res.body === null) {
|
|
80
|
+
// A sitemap that answers 404 is passed over without an error.
|
|
81
|
+
if (res.status !== 404) {
|
|
82
|
+
return { kind: "error", message: `${url}: HTTP ${res.status}` };
|
|
83
|
+
}
|
|
84
|
+
return { kind: "skip" };
|
|
85
|
+
}
|
|
86
|
+
const parsed = parseSitemap(res.body, res.contentType);
|
|
87
|
+
if (parsed.kind !== "unknown") {
|
|
88
|
+
return { kind: "parsed", parsed };
|
|
89
|
+
}
|
|
90
|
+
if (!/<html/iu.test(res.body.slice(0, 1000))) {
|
|
91
|
+
return { kind: "error", message: `${url}: could not parse as a sitemap` };
|
|
92
|
+
}
|
|
93
|
+
// The one guessed /sitemap.xml answered with an SPA's fallback page: there is no sitemap.
|
|
94
|
+
if (candidates.includes(url) && candidates.length === 1) {
|
|
95
|
+
return { kind: "skip" };
|
|
96
|
+
}
|
|
97
|
+
return { kind: "error", message: `${url}: returned an HTML page, not XML` };
|
|
98
|
+
};
|
|
99
|
+
/** Fetch sitemap candidates (following sitemap indexes) and collect page URLs. */
|
|
100
|
+
export const loadSitemaps = async (candidates, options = {}, limits = {}) => {
|
|
101
|
+
const maxSitemaps = limits.maxSitemaps ?? 25;
|
|
102
|
+
const maxUrls = limits.maxUrls ?? 5000;
|
|
103
|
+
const info = { sources: [], urls: [], urlCount: 0, errors: [] };
|
|
104
|
+
const queue = [...new Set(candidates)];
|
|
105
|
+
const seen = new Set();
|
|
106
|
+
const urls = new Set();
|
|
107
|
+
while (queue.length > 0 &&
|
|
108
|
+
info.sources.length < maxSitemaps &&
|
|
109
|
+
urls.size < maxUrls) {
|
|
110
|
+
const url = queue.shift();
|
|
111
|
+
if (url === undefined || seen.has(url)) {
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
seen.add(url);
|
|
115
|
+
// oxlint-disable-next-line eslint/no-await-in-loop -- a sitemap index adds the sitemaps to fetch next, and the limits count what the earlier ones gave
|
|
116
|
+
const fetched = await fetchSitemap(url, candidates, options);
|
|
117
|
+
if (fetched.kind === "error") {
|
|
118
|
+
info.errors.push(fetched.message);
|
|
119
|
+
}
|
|
120
|
+
if (fetched.kind !== "parsed") {
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const { parsed } = fetched;
|
|
124
|
+
info.sources.push(url);
|
|
125
|
+
if (parsed.kind === "sitemapindex") {
|
|
126
|
+
queue.push(...parsed.locs);
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
for (const loc of parsed.locs) {
|
|
130
|
+
if (urls.size < maxUrls) {
|
|
131
|
+
urls.add(loc);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
info.urls = [...urls];
|
|
137
|
+
info.urlCount = urls.size;
|
|
138
|
+
return info;
|
|
139
|
+
};
|
|
140
|
+
//# sourceMappingURL=sitemap.js.map
|