@rivi-software/agentic-seo-check 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +65 -15
- package/dist/brand.d.ts +11 -0
- package/dist/brand.js +22 -0
- package/dist/brand.js.map +1 -0
- package/dist/cli.js +125 -80
- package/dist/cli.js.map +1 -1
- package/dist/core/audit.d.ts +16 -16
- package/dist/core/audit.js +363 -248
- package/dist/core/audit.js.map +1 -1
- package/dist/core/concurrency.d.ts +4 -4
- package/dist/core/concurrency.js +33 -23
- package/dist/core/concurrency.js.map +1 -1
- package/dist/core/crawl.d.ts +5 -5
- package/dist/core/crawl.js +250 -121
- package/dist/core/crawl.js.map +1 -1
- package/dist/core/fetch.d.ts +5 -5
- package/dist/core/fetch.js +151 -100
- package/dist/core/fetch.js.map +1 -1
- package/dist/core/html/dom.d.ts +13 -0
- package/dist/core/html/dom.js +17 -0
- package/dist/core/html/dom.js.map +1 -0
- package/dist/core/html/extract.d.ts +2 -2
- package/dist/core/html/extract.js +168 -111
- package/dist/core/html/extract.js.map +1 -1
- package/dist/core/html/structured-data.d.ts +16 -6
- package/dist/core/html/structured-data.js +180 -145
- package/dist/core/html/structured-data.js.map +1 -1
- package/dist/core/html/text.d.ts +14 -14
- package/dist/core/html/text.js +110 -107
- package/dist/core/html/text.js.map +1 -1
- package/dist/core/lighthouse.d.ts +26 -25
- package/dist/core/lighthouse.js +181 -105
- package/dist/core/lighthouse.js.map +1 -1
- package/dist/core/performance-rules.d.ts +2 -2
- package/dist/core/performance-rules.js +105 -33
- package/dist/core/performance-rules.js.map +1 -1
- package/dist/core/render-rules.d.ts +5 -4
- package/dist/core/render-rules.js +129 -62
- package/dist/core/render-rules.js.map +1 -1
- package/dist/core/render.d.ts +2 -2
- package/dist/core/render.js +29 -23
- package/dist/core/render.js.map +1 -1
- package/dist/core/report/markdown.d.ts +2 -2
- package/dist/core/report/markdown.js +110 -131
- package/dist/core/report/markdown.js.map +1 -1
- package/dist/core/robots.d.ts +5 -5
- package/dist/core/robots.js +31 -16
- package/dist/core/robots.js.map +1 -1
- package/dist/core/rules/canonical.d.ts +1 -1
- package/dist/core/rules/canonical.js +93 -57
- package/dist/core/rules/canonical.js.map +1 -1
- package/dist/core/rules/content.js +40 -26
- package/dist/core/rules/content.js.map +1 -1
- package/dist/core/rules/document.js +64 -41
- package/dist/core/rules/document.js.map +1 -1
- package/dist/core/rules/headings.js +58 -38
- package/dist/core/rules/headings.js.map +1 -1
- package/dist/core/rules/hreflang.js +70 -38
- package/dist/core/rules/hreflang.js.map +1 -1
- package/dist/core/rules/images.js +44 -26
- package/dist/core/rules/images.js.map +1 -1
- package/dist/core/rules/index.d.ts +6 -6
- package/dist/core/rules/index.js +36 -35
- package/dist/core/rules/index.js.map +1 -1
- package/dist/core/rules/indexing.d.ts +2 -2
- package/dist/core/rules/indexing.js +48 -26
- package/dist/core/rules/indexing.js.map +1 -1
- package/dist/core/rules/keyword.js +84 -47
- package/dist/core/rules/keyword.js.map +1 -1
- package/dist/core/rules/links.js +72 -37
- package/dist/core/rules/links.js.map +1 -1
- package/dist/core/rules/meta.js +55 -34
- package/dist/core/rules/meta.js.map +1 -1
- package/dist/core/rules/response.js +75 -41
- package/dist/core/rules/response.js.map +1 -1
- package/dist/core/rules/rule.d.ts +6 -6
- package/dist/core/rules/rule.js +36 -36
- package/dist/core/rules/rule.js.map +1 -1
- package/dist/core/rules/social.js +86 -45
- package/dist/core/rules/social.js.map +1 -1
- package/dist/core/rules/structured-data.d.ts +6 -4
- package/dist/core/rules/structured-data.js +314 -102
- package/dist/core/rules/structured-data.js.map +1 -1
- package/dist/core/rules/title.js +48 -33
- package/dist/core/rules/title.js.map +1 -1
- package/dist/core/score.d.ts +3 -3
- package/dist/core/score.js +11 -9
- package/dist/core/score.js.map +1 -1
- package/dist/core/site-rules.d.ts +3 -3
- package/dist/core/site-rules.js +409 -198
- package/dist/core/site-rules.js.map +1 -1
- package/dist/core/sitemap.d.ts +5 -5
- package/dist/core/sitemap.js +109 -64
- package/dist/core/sitemap.js.map +1 -1
- package/dist/core/static.d.ts +2 -2
- package/dist/core/static.js +209 -112
- package/dist/core/static.js.map +1 -1
- package/dist/core/summary.d.ts +2 -2
- package/dist/core/summary.js +11 -6
- package/dist/core/summary.js.map +1 -1
- package/dist/core/types.d.ts +14 -14
- package/dist/core/types.js +15 -8
- package/dist/core/types.js.map +1 -1
- package/dist/core/url.d.ts +11 -11
- package/dist/core/url.js +122 -70
- package/dist/core/url.js.map +1 -1
- package/dist/index.d.ts +11 -11
- package/dist/index.js +9 -9
- package/dist/index.js.map +1 -1
- package/dist/mcp-bin.js +7 -4
- package/dist/mcp-bin.js.map +1 -1
- package/dist/mcp.d.ts +3 -3
- package/dist/mcp.js +312 -122
- package/dist/mcp.js.map +1 -1
- package/dist/version.d.ts +0 -1
- package/dist/version.js +4 -2
- package/dist/version.js.map +1 -1
- package/package.json +27 -9
package/dist/core/audit.js
CHANGED
|
@@ -1,32 +1,281 @@
|
|
|
1
|
-
import { promises as fs } from
|
|
2
|
-
import path from
|
|
3
|
-
import { crawlSite } from
|
|
4
|
-
import { DEFAULT_USER_AGENT, fetchPage, toResponseInfo } from
|
|
5
|
-
import { extractPage } from
|
|
6
|
-
import { runLighthouse, LighthouseUnavailableError } from
|
|
7
|
-
import { performanceFindings } from
|
|
8
|
-
import { compareRender } from
|
|
9
|
-
import { renderPage, RenderUnavailableError } from
|
|
10
|
-
import { runPageRules, sortFindings } from
|
|
11
|
-
import { pageScore, siteScore } from
|
|
12
|
-
import { runSiteRules } from
|
|
13
|
-
import { scanDirectory } from
|
|
14
|
-
import { buildSummary } from
|
|
15
|
-
import { countFindings, DEFAULT_THRESHOLDS
|
|
16
|
-
import { isHttpUrl } from
|
|
17
|
-
export const DEFAULT_CRAWL = {
|
|
1
|
+
import { promises as fs } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { crawlSite } from "./crawl.js";
|
|
4
|
+
import { DEFAULT_USER_AGENT, fetchPage, toResponseInfo } from "./fetch.js";
|
|
5
|
+
import { extractPage } from "./html/extract.js";
|
|
6
|
+
import { runLighthouse, LighthouseUnavailableError } from "./lighthouse.js";
|
|
7
|
+
import { performanceFindings } from "./performance-rules.js";
|
|
8
|
+
import { compareRender } from "./render-rules.js";
|
|
9
|
+
import { renderPage, RenderUnavailableError } from "./render.js";
|
|
10
|
+
import { runPageRules, sortFindings } from "./rules/index.js";
|
|
11
|
+
import { pageScore, siteScore } from "./score.js";
|
|
12
|
+
import { runSiteRules } from "./site-rules.js";
|
|
13
|
+
import { scanDirectory } from "./static.js";
|
|
14
|
+
import { buildSummary } from "./summary.js";
|
|
15
|
+
import { countFindings, DEFAULT_THRESHOLDS } from "./types.js";
|
|
16
|
+
import { isHttpUrl } from "./url.js";
|
|
17
|
+
export const DEFAULT_CRAWL = {
|
|
18
|
+
maxPages: 50,
|
|
19
|
+
maxDepth: 3,
|
|
20
|
+
concurrency: 5,
|
|
21
|
+
delayMs: 0,
|
|
22
|
+
maxLinkChecks: 200,
|
|
23
|
+
};
|
|
24
|
+
const toolFinding = (rule, message, url) => ({
|
|
25
|
+
rule,
|
|
26
|
+
severity: "info",
|
|
27
|
+
category: "tool",
|
|
28
|
+
message,
|
|
29
|
+
recommendation: "Install the optional dependency (see message) or run without this option. The rest of the audit is complete.",
|
|
30
|
+
url,
|
|
31
|
+
});
|
|
32
|
+
// The raw HTML is not kept on the audit (too large); re-fetch it for the render comparison.
|
|
33
|
+
const rawHtmlFor = async (audit, options) => {
|
|
34
|
+
const res = await fetchPage(audit.response?.finalUrl ?? audit.url, options.fetch ?? {});
|
|
35
|
+
return res.body ?? "";
|
|
36
|
+
};
|
|
37
|
+
/** Renders the page in Chromium and compares it with the server HTML, on `audit.rendering`. */
|
|
38
|
+
const renderFindings = async (audit, options, url) => {
|
|
39
|
+
options.onProgress?.({
|
|
40
|
+
phase: "render",
|
|
41
|
+
message: `Rendering ${url} with Playwright`,
|
|
42
|
+
});
|
|
43
|
+
try {
|
|
44
|
+
const rendered = await renderPage(url, {
|
|
45
|
+
timeoutMs: Math.max(options.fetch?.timeoutMs ?? 0, 30_000),
|
|
46
|
+
userAgent: options.fetch?.userAgent,
|
|
47
|
+
headers: options.fetch?.headers,
|
|
48
|
+
});
|
|
49
|
+
const rawPage = extractPage(await rawHtmlFor(audit, options), {
|
|
50
|
+
url,
|
|
51
|
+
includeSubdomains: options.includeSubdomains,
|
|
52
|
+
});
|
|
53
|
+
const renderedPage = extractPage(rendered.html, {
|
|
54
|
+
url: rendered.finalUrl,
|
|
55
|
+
siteHost: new URL(url).hostname,
|
|
56
|
+
includeSubdomains: options.includeSubdomains,
|
|
57
|
+
});
|
|
58
|
+
const { comparison, findings } = compareRender(rawPage, renderedPage, rendered.finalUrl, rendered.consoleErrors, audit.response);
|
|
59
|
+
audit.rendering = comparison;
|
|
60
|
+
return findings;
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
return [
|
|
64
|
+
toolFinding("tool.render-unavailable", error instanceof RenderUnavailableError
|
|
65
|
+
? error.message
|
|
66
|
+
: `Rendering failed: ${error instanceof Error ? error.message : String(error)}`, url),
|
|
67
|
+
];
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
/** Runs Lighthouse on the page, on `audit.performance`. */
|
|
71
|
+
const lighthouseFindings = async (audit, options, url) => {
|
|
72
|
+
options.onProgress?.({
|
|
73
|
+
phase: "lighthouse",
|
|
74
|
+
message: `Running Lighthouse (${options.formFactor ?? "mobile"}) on ${url}`,
|
|
75
|
+
});
|
|
76
|
+
try {
|
|
77
|
+
const perf = await runLighthouse(url, {
|
|
78
|
+
formFactor: options.formFactor ?? "mobile",
|
|
79
|
+
userAgent: options.fetch?.userAgent,
|
|
80
|
+
headers: options.fetch?.headers,
|
|
81
|
+
});
|
|
82
|
+
audit.performance = perf;
|
|
83
|
+
return performanceFindings(perf);
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
return [
|
|
87
|
+
toolFinding("tool.lighthouse-unavailable", error instanceof LighthouseUnavailableError
|
|
88
|
+
? error.message
|
|
89
|
+
: `Lighthouse failed: ${error instanceof Error ? error.message : String(error)}`, url),
|
|
90
|
+
];
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
const enrichPage = async (audit, options) => {
|
|
94
|
+
const url = audit.response?.finalUrl ?? audit.url;
|
|
95
|
+
const extra = [];
|
|
96
|
+
if (options.render) {
|
|
97
|
+
extra.push(...(await renderFindings(audit, options, url)));
|
|
98
|
+
}
|
|
99
|
+
if (options.lighthouse) {
|
|
100
|
+
extra.push(...(await lighthouseFindings(audit, options, url)));
|
|
101
|
+
}
|
|
102
|
+
if (extra.length) {
|
|
103
|
+
const filtered = options.includeInfo === false
|
|
104
|
+
? extra.filter((f) => f.severity !== "info")
|
|
105
|
+
: extra;
|
|
106
|
+
audit.findings = sortFindings([...audit.findings, ...filtered]);
|
|
107
|
+
audit.counts = countFindings(audit.findings);
|
|
108
|
+
audit.score = pageScore(audit.counts);
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
const auditRecord = (record, source, options) => {
|
|
112
|
+
const thresholds = { ...DEFAULT_THRESHOLDS, ...options.thresholds };
|
|
113
|
+
let findings = runPageRules({
|
|
114
|
+
page: record.page,
|
|
115
|
+
source,
|
|
116
|
+
response: record.response,
|
|
117
|
+
keyword: options.keyword,
|
|
118
|
+
thresholds,
|
|
119
|
+
depth: record.depth,
|
|
120
|
+
}, { disable: new Set(options.disableRules) });
|
|
121
|
+
if (options.includeInfo === false) {
|
|
122
|
+
findings = findings.filter((f) => f.severity !== "info");
|
|
123
|
+
}
|
|
124
|
+
const audit = {
|
|
125
|
+
schemaVersion: 1,
|
|
126
|
+
kind: "page",
|
|
127
|
+
target: record.file ?? record.requestedUrl,
|
|
128
|
+
url: record.file ?? record.url,
|
|
129
|
+
source,
|
|
130
|
+
auditedAt: new Date().toISOString(),
|
|
131
|
+
summary: buildSummary(record.page, record.response),
|
|
132
|
+
findings,
|
|
133
|
+
counts: countFindings(findings),
|
|
134
|
+
score: 0,
|
|
135
|
+
depth: record.depth,
|
|
136
|
+
};
|
|
137
|
+
if (record.response) {
|
|
138
|
+
audit.response = record.response;
|
|
139
|
+
}
|
|
140
|
+
audit.score = pageScore(audit.counts);
|
|
141
|
+
return audit;
|
|
142
|
+
};
|
|
143
|
+
export const rollupIssues = (pages) => {
|
|
144
|
+
const map = new Map();
|
|
145
|
+
for (const page of pages) {
|
|
146
|
+
const seenHere = new Set();
|
|
147
|
+
for (const f of page.findings) {
|
|
148
|
+
if (seenHere.has(f.rule)) {
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
seenHere.add(f.rule);
|
|
152
|
+
let entry = map.get(f.rule);
|
|
153
|
+
if (!entry) {
|
|
154
|
+
entry = {
|
|
155
|
+
rule: f.rule,
|
|
156
|
+
severity: f.severity,
|
|
157
|
+
category: f.category,
|
|
158
|
+
pages: 0,
|
|
159
|
+
message: f.message,
|
|
160
|
+
recommendation: f.recommendation,
|
|
161
|
+
examples: [],
|
|
162
|
+
urls: new Set(),
|
|
163
|
+
};
|
|
164
|
+
map.set(f.rule, entry);
|
|
165
|
+
}
|
|
166
|
+
entry.pages += 1;
|
|
167
|
+
entry.urls.add(page.url);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
const rank = { error: 0, warning: 1, info: 2 };
|
|
171
|
+
return [...map.values()]
|
|
172
|
+
.toSorted((a, b) => rank[a.severity] - rank[b.severity] || b.pages - a.pages)
|
|
173
|
+
.map(({ urls, ...rest }) => ({
|
|
174
|
+
...rest,
|
|
175
|
+
examples: [...urls].slice(0, 10),
|
|
176
|
+
}));
|
|
177
|
+
};
|
|
178
|
+
export const recomputeSite = (site) => {
|
|
179
|
+
const counts = countFindings(site.siteFindings);
|
|
180
|
+
for (const p of site.pages) {
|
|
181
|
+
counts.error += p.counts.error;
|
|
182
|
+
counts.warning += p.counts.warning;
|
|
183
|
+
counts.info += p.counts.info;
|
|
184
|
+
}
|
|
185
|
+
site.counts = counts;
|
|
186
|
+
site.issues = rollupIssues(site.pages);
|
|
187
|
+
site.score = siteScore(site.pages, site.siteFindings);
|
|
188
|
+
};
|
|
189
|
+
const buildSiteAudit = (result, source, target, options) => {
|
|
190
|
+
options.onProgress?.({
|
|
191
|
+
phase: "rules",
|
|
192
|
+
message: `Evaluating rules for ${result.records.length} pages`,
|
|
193
|
+
});
|
|
194
|
+
const thresholds = { ...DEFAULT_THRESHOLDS, ...options.thresholds };
|
|
195
|
+
const pages = result.records.map((record) => auditRecord(record, source === "directory" ? "file" : "url", options));
|
|
196
|
+
let siteFindings = runSiteRules({
|
|
197
|
+
startUrl: result.startUrl,
|
|
198
|
+
source,
|
|
199
|
+
pages: result.records,
|
|
200
|
+
linkChecks: result.linkChecks,
|
|
201
|
+
linkSources: result.linkSources,
|
|
202
|
+
externalChecks: result.externalChecks,
|
|
203
|
+
externalSources: result.externalSources,
|
|
204
|
+
robots: result.robots,
|
|
205
|
+
robotsAllows: result.robotsAllows,
|
|
206
|
+
blockedByRobots: result.blockedByRobots,
|
|
207
|
+
sitemaps: result.sitemaps,
|
|
208
|
+
probes: result.probes,
|
|
209
|
+
crawl: result.stats,
|
|
210
|
+
thresholds,
|
|
211
|
+
pathOnly: source === "directory",
|
|
212
|
+
unlinkedPages: result.unlinkedPages,
|
|
213
|
+
});
|
|
214
|
+
const disabled = new Set(options.disableRules);
|
|
215
|
+
siteFindings = sortFindings(siteFindings.filter((f) => !disabled.has(f.rule)));
|
|
216
|
+
if (options.includeInfo === false) {
|
|
217
|
+
siteFindings = siteFindings.filter((f) => f.severity !== "info");
|
|
218
|
+
}
|
|
219
|
+
const site = {
|
|
220
|
+
schemaVersion: 1,
|
|
221
|
+
kind: "site",
|
|
222
|
+
target,
|
|
223
|
+
source,
|
|
224
|
+
auditedAt: new Date().toISOString(),
|
|
225
|
+
crawl: result.stats,
|
|
226
|
+
robotsTxt: result.robots,
|
|
227
|
+
sitemaps: { ...result.sitemaps, urls: result.sitemaps.urls.slice(0, 200) },
|
|
228
|
+
pages: pages.toSorted((a, b) => (a.depth ?? 0) - (b.depth ?? 0) || a.score - b.score),
|
|
229
|
+
siteFindings,
|
|
230
|
+
issues: [],
|
|
231
|
+
counts: { error: 0, warning: 0, info: 0 },
|
|
232
|
+
score: 0,
|
|
233
|
+
};
|
|
234
|
+
recomputeSite(site);
|
|
235
|
+
return site;
|
|
236
|
+
};
|
|
237
|
+
/** Keeps only each page's summary when the caller asked for `pageDetail: "summary"`. */
|
|
238
|
+
const applyPageDetail = (site, options) => {
|
|
239
|
+
if (options.pageDetail === "summary") {
|
|
240
|
+
site.pages = site.pages.map((p) => ({
|
|
241
|
+
...p,
|
|
242
|
+
findings: [],
|
|
243
|
+
summary: { ...p.summary, headings: [], openGraph: {} },
|
|
244
|
+
}));
|
|
245
|
+
}
|
|
246
|
+
return site;
|
|
247
|
+
};
|
|
248
|
+
const readStdin = async () => {
|
|
249
|
+
const chunks = [];
|
|
250
|
+
for await (const chunk of process.stdin) {
|
|
251
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
252
|
+
}
|
|
253
|
+
return Buffer.concat(chunks).toString("utf-8");
|
|
254
|
+
};
|
|
18
255
|
/** Audit an HTML string. `url` is used to resolve links and for display. */
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
const
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
256
|
+
// oxlint-disable-next-line eslint/require-await -- public API: it returns a promise like the other audit functions, so a throw reaches callers as a rejection
|
|
257
|
+
export const auditHtml = async (html, input = {}, options = {}) => {
|
|
258
|
+
const url = input.url ?? options.baseUrl ?? "http://localhost/";
|
|
259
|
+
const page = extractPage(html, {
|
|
260
|
+
url,
|
|
261
|
+
includeSubdomains: options.includeSubdomains,
|
|
262
|
+
});
|
|
263
|
+
const source = input.source ?? "html";
|
|
264
|
+
const thresholds = { ...DEFAULT_THRESHOLDS, ...options.thresholds };
|
|
265
|
+
let findings = runPageRules({
|
|
266
|
+
page,
|
|
267
|
+
source,
|
|
268
|
+
response: input.response,
|
|
269
|
+
keyword: options.keyword,
|
|
270
|
+
thresholds,
|
|
271
|
+
depth: input.depth,
|
|
272
|
+
}, { disable: new Set(options.disableRules) });
|
|
273
|
+
if (options.includeInfo === false) {
|
|
274
|
+
findings = findings.filter((f) => f.severity !== "info");
|
|
275
|
+
}
|
|
27
276
|
const audit = {
|
|
28
277
|
schemaVersion: 1,
|
|
29
|
-
kind:
|
|
278
|
+
kind: "page",
|
|
30
279
|
target: input.target ?? url,
|
|
31
280
|
url: input.response?.finalUrl ?? url,
|
|
32
281
|
source,
|
|
@@ -36,25 +285,27 @@ export async function auditHtml(html, input = {}, options = {}) {
|
|
|
36
285
|
counts: countFindings(findings),
|
|
37
286
|
score: 0,
|
|
38
287
|
};
|
|
39
|
-
if (input.response)
|
|
288
|
+
if (input.response) {
|
|
40
289
|
audit.response = input.response;
|
|
41
|
-
|
|
290
|
+
}
|
|
291
|
+
if (input.depth !== undefined) {
|
|
42
292
|
audit.depth = input.depth;
|
|
293
|
+
}
|
|
43
294
|
audit.score = pageScore(audit.counts);
|
|
44
295
|
return audit;
|
|
45
|
-
}
|
|
296
|
+
};
|
|
46
297
|
/** Fetch and audit one URL. Optionally render and/or run Lighthouse. */
|
|
47
|
-
export async
|
|
298
|
+
export const auditUrl = async (url, options = {}) => {
|
|
48
299
|
const res = await fetchPage(url, options.fetch ?? {});
|
|
49
300
|
const response = toResponseInfo(res);
|
|
50
|
-
const audit = await auditHtml(res.body ??
|
|
301
|
+
const audit = await auditHtml(res.body ?? "", { url: res.finalUrl, response, source: "url", target: url }, options);
|
|
51
302
|
if (res.error) {
|
|
52
303
|
audit.findings.unshift({
|
|
53
|
-
rule:
|
|
54
|
-
severity:
|
|
55
|
-
category:
|
|
304
|
+
rule: "response.fetch-failed",
|
|
305
|
+
severity: "error",
|
|
306
|
+
category: "response",
|
|
56
307
|
message: `Could not fetch ${url}: ${res.error}`,
|
|
57
|
-
recommendation:
|
|
308
|
+
recommendation: "Check that the URL is reachable from this machine (DNS, TLS, firewall, dev server running) and that it responds within the timeout.",
|
|
58
309
|
url,
|
|
59
310
|
evidence: { error: res.error, redirectChain: res.redirectChain },
|
|
60
311
|
});
|
|
@@ -64,17 +315,17 @@ export async function auditUrl(url, options = {}) {
|
|
|
64
315
|
}
|
|
65
316
|
await enrichPage(audit, options);
|
|
66
317
|
return audit;
|
|
67
|
-
}
|
|
318
|
+
};
|
|
68
319
|
/** Audit a single .html file from disk. */
|
|
69
|
-
export async
|
|
70
|
-
const html = await fs.readFile(filePath,
|
|
71
|
-
const base = options.baseUrl ??
|
|
72
|
-
const url = new URL(`/${path.basename(filePath)}`, base.endsWith(
|
|
73
|
-
return auditHtml(html, { url, source:
|
|
74
|
-
}
|
|
320
|
+
export const auditFile = async (filePath, options = {}) => {
|
|
321
|
+
const html = await fs.readFile(filePath, "utf-8");
|
|
322
|
+
const base = options.baseUrl ?? "http://localhost/";
|
|
323
|
+
const url = new URL(`/${path.basename(filePath)}`, base.endsWith("/") ? base : `${base}/`).href;
|
|
324
|
+
return auditHtml(html, { url, source: "file", target: filePath }, options);
|
|
325
|
+
};
|
|
75
326
|
/** Crawl a site and audit every page plus site-level checks. */
|
|
76
|
-
export async
|
|
77
|
-
const fetchOpts = { ...
|
|
327
|
+
export const auditSite = async (startUrl, options = {}) => {
|
|
328
|
+
const fetchOpts = { ...options.fetch };
|
|
78
329
|
fetchOpts.userAgent ??= DEFAULT_USER_AGENT;
|
|
79
330
|
const internal = {
|
|
80
331
|
maxPages: options.maxPages ?? DEFAULT_CRAWL.maxPages,
|
|
@@ -85,247 +336,111 @@ export async function auditSite(startUrl, options = {}) {
|
|
|
85
336
|
includeSubdomains: options.includeSubdomains ?? false,
|
|
86
337
|
checkExternalLinks: options.checkExternalLinks ?? false,
|
|
87
338
|
maxLinkChecks: options.maxLinkChecks ?? DEFAULT_CRAWL.maxLinkChecks,
|
|
339
|
+
// oxlint-disable-next-line eslint/require-unicode-regexp -- the user's own patterns, in JavaScript's default syntax; the u flag would reject some of them
|
|
88
340
|
includePatterns: (options.includePatterns ?? []).map((p) => new RegExp(p)),
|
|
341
|
+
// oxlint-disable-next-line eslint/require-unicode-regexp -- the user's own patterns, in JavaScript's default syntax; the u flag would reject some of them
|
|
89
342
|
excludePatterns: (options.excludePatterns ?? []).map((p) => new RegExp(p)),
|
|
90
343
|
fetch: fetchOpts,
|
|
91
344
|
onProgress: options.onProgress,
|
|
92
345
|
};
|
|
93
346
|
const result = await crawlSite(startUrl, internal);
|
|
94
|
-
const site =
|
|
347
|
+
const site = buildSiteAudit(result, "crawl", startUrl, options);
|
|
95
348
|
// Render/Lighthouse are expensive: apply them to the start page only.
|
|
96
349
|
const startPage = site.pages.find((p) => p.depth === 0) ?? site.pages[0];
|
|
97
350
|
if (startPage && (options.render || options.lighthouse)) {
|
|
98
351
|
await enrichPage(startPage, options);
|
|
99
352
|
recomputeSite(site);
|
|
100
353
|
}
|
|
101
|
-
return
|
|
102
|
-
}
|
|
354
|
+
return applyPageDetail(site, options);
|
|
355
|
+
};
|
|
103
356
|
/** Audit a directory of built HTML files (dist/, out/, build/). */
|
|
104
|
-
export async
|
|
357
|
+
export const auditDirectory = async (dir, options = {}) => {
|
|
105
358
|
const result = await scanDirectory(dir, {
|
|
106
|
-
baseUrl: options.baseUrl ??
|
|
359
|
+
baseUrl: options.baseUrl ?? "http://localhost/",
|
|
107
360
|
includeSubdomains: options.includeSubdomains ?? false,
|
|
108
361
|
maxPages: options.maxPages ?? 500,
|
|
109
362
|
userAgent: options.fetch?.userAgent,
|
|
110
363
|
});
|
|
111
|
-
const site =
|
|
112
|
-
return
|
|
113
|
-
}
|
|
364
|
+
const site = buildSiteAudit(result, "directory", dir, options);
|
|
365
|
+
return applyPageDetail(site, options);
|
|
366
|
+
};
|
|
114
367
|
/**
|
|
115
368
|
* Auto-detect the target type: http(s) URL, directory, .html file, or "-" for stdin.
|
|
116
369
|
* `crawl` forces a site crawl for URLs.
|
|
117
370
|
*/
|
|
118
|
-
export async
|
|
119
|
-
if (target ===
|
|
371
|
+
export const auditTarget = async (target, options = {}) => {
|
|
372
|
+
if (target === "-" || options.stdinHtml !== undefined) {
|
|
120
373
|
const html = options.stdinHtml ?? (await readStdin());
|
|
121
|
-
return auditHtml(html, { url: options.baseUrl, source:
|
|
374
|
+
return auditHtml(html, { url: options.baseUrl, source: "html", target: "stdin" }, options);
|
|
375
|
+
}
|
|
376
|
+
if (isHttpUrl(target)) {
|
|
377
|
+
return options.crawl
|
|
378
|
+
? auditSite(target, options)
|
|
379
|
+
: auditUrl(target, options);
|
|
122
380
|
}
|
|
123
|
-
if (
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
const url = `http${/^localhost/i.test(target) ? '' : 's'}://${target}`;
|
|
381
|
+
if (/^localhost(?::\d+)?(?:\/|$)/iu.test(target) ||
|
|
382
|
+
/^[\w.-]+\.[a-z]{2,}(?:\/|$)/iu.test(target)) {
|
|
383
|
+
const url = `http${/^localhost/iu.test(target) ? "" : "s"}://${target}`;
|
|
127
384
|
return options.crawl ? auditSite(url, options) : auditUrl(url, options);
|
|
128
385
|
}
|
|
129
386
|
const stat = await fs.stat(target).catch(() => null);
|
|
130
|
-
if (!stat)
|
|
387
|
+
if (!stat) {
|
|
131
388
|
throw new Error(`Target "${target}" is not a URL, file or directory.`);
|
|
132
|
-
if (stat.isDirectory())
|
|
133
|
-
return auditDirectory(target, options);
|
|
134
|
-
return auditFile(target, options);
|
|
135
|
-
}
|
|
136
|
-
async function enrichPage(audit, options) {
|
|
137
|
-
const url = audit.response?.finalUrl ?? audit.url;
|
|
138
|
-
const extra = [];
|
|
139
|
-
if (options.render) {
|
|
140
|
-
options.onProgress?.({ phase: 'render', message: `Rendering ${url} with Playwright` });
|
|
141
|
-
try {
|
|
142
|
-
const rendered = await renderPage(url, { timeoutMs: Math.max(options.fetch?.timeoutMs ?? 0, 30_000), userAgent: options.fetch?.userAgent, headers: options.fetch?.headers });
|
|
143
|
-
const rawPage = extractPage(await rawHtmlFor(audit, options), { url, includeSubdomains: options.includeSubdomains });
|
|
144
|
-
const renderedPage = extractPage(rendered.html, { url: rendered.finalUrl, siteHost: new URL(url).hostname, includeSubdomains: options.includeSubdomains });
|
|
145
|
-
const { comparison, findings } = compareRender(rawPage, renderedPage, rendered.finalUrl, rendered.consoleErrors, audit.response);
|
|
146
|
-
audit.rendering = comparison;
|
|
147
|
-
extra.push(...findings);
|
|
148
|
-
}
|
|
149
|
-
catch (err) {
|
|
150
|
-
extra.push(toolFinding('tool.render-unavailable', err instanceof RenderUnavailableError ? err.message : `Rendering failed: ${err.message}`, url));
|
|
151
|
-
}
|
|
152
389
|
}
|
|
153
|
-
if (
|
|
154
|
-
|
|
155
|
-
try {
|
|
156
|
-
const perf = await runLighthouse(url, { formFactor: options.formFactor ?? 'mobile', userAgent: options.fetch?.userAgent, headers: options.fetch?.headers });
|
|
157
|
-
audit.performance = perf;
|
|
158
|
-
extra.push(...performanceFindings(perf));
|
|
159
|
-
}
|
|
160
|
-
catch (err) {
|
|
161
|
-
extra.push(toolFinding('tool.lighthouse-unavailable', err instanceof LighthouseUnavailableError ? err.message : `Lighthouse failed: ${err.message}`, url));
|
|
162
|
-
}
|
|
390
|
+
if (stat.isDirectory()) {
|
|
391
|
+
return auditDirectory(target, options);
|
|
163
392
|
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
393
|
+
return auditFile(target, options);
|
|
394
|
+
};
|
|
395
|
+
/** Keeps the first three items of every evidence list. */
|
|
396
|
+
const trimFinding = (f) => {
|
|
397
|
+
if (!f.evidence) {
|
|
398
|
+
return f;
|
|
169
399
|
}
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
const res = await fetchPage(audit.response?.finalUrl ?? audit.url, options.fetch ?? {});
|
|
174
|
-
return res.body ?? '';
|
|
175
|
-
}
|
|
176
|
-
function toolFinding(rule, message, url) {
|
|
177
|
-
return {
|
|
178
|
-
rule,
|
|
179
|
-
severity: 'info',
|
|
180
|
-
category: 'tool',
|
|
181
|
-
message,
|
|
182
|
-
recommendation: 'Install the optional dependency (see message) or run without this option. The rest of the audit is complete.',
|
|
183
|
-
url,
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
async function buildSiteAudit(result, source, target, options) {
|
|
187
|
-
options.onProgress?.({ phase: 'rules', message: `Evaluating rules for ${result.records.length} pages` });
|
|
188
|
-
const thresholds = { ...DEFAULT_THRESHOLDS, ...(options.thresholds ?? {}) };
|
|
189
|
-
const pages = [];
|
|
190
|
-
for (const record of result.records) {
|
|
191
|
-
const audit = await auditRecord(record, source === 'directory' ? 'file' : 'url', options);
|
|
192
|
-
pages.push(audit);
|
|
400
|
+
const evidence = {};
|
|
401
|
+
for (const [k, v] of Object.entries(f.evidence)) {
|
|
402
|
+
evidence[k] = Array.isArray(v) ? v.slice(0, 3) : v;
|
|
193
403
|
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
probes: result.probes,
|
|
207
|
-
crawl: result.stats,
|
|
208
|
-
thresholds,
|
|
209
|
-
pathOnly: source === 'directory',
|
|
210
|
-
unlinkedPages: result.unlinkedPages,
|
|
211
|
-
});
|
|
212
|
-
const disabled = new Set(options.disableRules ?? []);
|
|
213
|
-
siteFindings = sortFindings(siteFindings.filter((f) => !disabled.has(f.rule)));
|
|
214
|
-
if (options.includeInfo === false)
|
|
215
|
-
siteFindings = siteFindings.filter((f) => f.severity !== 'info');
|
|
216
|
-
const site = {
|
|
217
|
-
schemaVersion: 1,
|
|
218
|
-
kind: 'site',
|
|
219
|
-
target,
|
|
220
|
-
source,
|
|
221
|
-
auditedAt: new Date().toISOString(),
|
|
222
|
-
crawl: result.stats,
|
|
223
|
-
robotsTxt: result.robots,
|
|
224
|
-
sitemaps: { ...result.sitemaps, urls: result.sitemaps.urls.slice(0, 200) },
|
|
225
|
-
pages: pages.sort((a, b) => (a.depth ?? 0) - (b.depth ?? 0) || a.score - b.score),
|
|
226
|
-
siteFindings,
|
|
227
|
-
issues: [],
|
|
228
|
-
counts: { error: 0, warning: 0, info: 0 },
|
|
229
|
-
score: 0,
|
|
404
|
+
return { ...f, evidence };
|
|
405
|
+
};
|
|
406
|
+
const trimPage = (p) => {
|
|
407
|
+
const { performance, rendering, ...rest } = p;
|
|
408
|
+
const out = {
|
|
409
|
+
...rest,
|
|
410
|
+
summary: {
|
|
411
|
+
...p.summary,
|
|
412
|
+
headings: p.summary.headings.slice(0, 10),
|
|
413
|
+
openGraph: {},
|
|
414
|
+
},
|
|
415
|
+
findings: p.findings.map(trimFinding),
|
|
230
416
|
};
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
}
|
|
234
|
-
async function auditRecord(record, source, options) {
|
|
235
|
-
const thresholds = { ...DEFAULT_THRESHOLDS, ...(options.thresholds ?? {}) };
|
|
236
|
-
let findings = runPageRules({ page: record.page, source, response: record.response, keyword: options.keyword, thresholds, depth: record.depth }, { disable: new Set(options.disableRules ?? []) });
|
|
237
|
-
if (options.includeInfo === false)
|
|
238
|
-
findings = findings.filter((f) => f.severity !== 'info');
|
|
239
|
-
const audit = {
|
|
240
|
-
schemaVersion: 1,
|
|
241
|
-
kind: 'page',
|
|
242
|
-
target: record.file ?? record.requestedUrl,
|
|
243
|
-
url: record.file ?? record.url,
|
|
244
|
-
source,
|
|
245
|
-
auditedAt: new Date().toISOString(),
|
|
246
|
-
summary: buildSummary(record.page, record.response),
|
|
247
|
-
findings,
|
|
248
|
-
counts: countFindings(findings),
|
|
249
|
-
score: 0,
|
|
250
|
-
depth: record.depth,
|
|
251
|
-
};
|
|
252
|
-
if (record.response)
|
|
253
|
-
audit.response = record.response;
|
|
254
|
-
audit.score = pageScore(audit.counts);
|
|
255
|
-
return audit;
|
|
256
|
-
}
|
|
257
|
-
export function recomputeSite(site) {
|
|
258
|
-
const counts = countFindings(site.siteFindings);
|
|
259
|
-
for (const p of site.pages) {
|
|
260
|
-
counts.error += p.counts.error;
|
|
261
|
-
counts.warning += p.counts.warning;
|
|
262
|
-
counts.info += p.counts.info;
|
|
417
|
+
if (p.response) {
|
|
418
|
+
out.response = { ...p.response, headers: {} };
|
|
263
419
|
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
const seenHere = new Set();
|
|
272
|
-
for (const f of page.findings) {
|
|
273
|
-
if (seenHere.has(f.rule))
|
|
274
|
-
continue;
|
|
275
|
-
seenHere.add(f.rule);
|
|
276
|
-
let entry = map.get(f.rule);
|
|
277
|
-
if (!entry) {
|
|
278
|
-
entry = { rule: f.rule, severity: f.severity, category: f.category, pages: 0, message: f.message, recommendation: f.recommendation, examples: [], urls: new Set() };
|
|
279
|
-
map.set(f.rule, entry);
|
|
280
|
-
}
|
|
281
|
-
entry.pages++;
|
|
282
|
-
entry.urls.add(page.url);
|
|
283
|
-
}
|
|
420
|
+
if (performance) {
|
|
421
|
+
out.performance = {
|
|
422
|
+
...performance,
|
|
423
|
+
opportunities: performance.opportunities.slice(0, 5),
|
|
424
|
+
agenticBrowsingAudits: [],
|
|
425
|
+
failedSeoAudits: performance.failedSeoAudits.slice(0, 5),
|
|
426
|
+
};
|
|
284
427
|
}
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
}
|
|
290
|
-
function shapeSite(site, options) {
|
|
291
|
-
if (options.pageDetail === 'summary') {
|
|
292
|
-
site.pages = site.pages.map((p) => ({ ...p, findings: [], summary: { ...p.summary, headings: [], openGraph: {} } }));
|
|
428
|
+
if (rendering) {
|
|
429
|
+
out.rendering = {
|
|
430
|
+
...rendering,
|
|
431
|
+
consoleErrors: rendering.consoleErrors.slice(0, 3),
|
|
432
|
+
};
|
|
293
433
|
}
|
|
294
|
-
return
|
|
295
|
-
}
|
|
296
|
-
async function readStdin() {
|
|
297
|
-
const chunks = [];
|
|
298
|
-
for await (const chunk of process.stdin)
|
|
299
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
300
|
-
return Buffer.concat(chunks).toString('utf8');
|
|
301
|
-
}
|
|
434
|
+
return out;
|
|
435
|
+
};
|
|
302
436
|
/** Drop bulky, low-value fields to save tokens when the result is consumed by an LLM. */
|
|
303
|
-
export
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
const evidence = {};
|
|
308
|
-
for (const [k, v] of Object.entries(f.evidence))
|
|
309
|
-
evidence[k] = Array.isArray(v) ? v.slice(0, 3) : v;
|
|
310
|
-
return { ...f, evidence };
|
|
311
|
-
};
|
|
312
|
-
const trimPage = (p) => {
|
|
313
|
-
const { performance, rendering, ...rest } = p;
|
|
314
|
-
const out = {
|
|
315
|
-
...rest,
|
|
316
|
-
summary: { ...p.summary, headings: p.summary.headings.slice(0, 10), openGraph: {} },
|
|
317
|
-
findings: p.findings.map(trimFinding),
|
|
318
|
-
};
|
|
319
|
-
if (p.response)
|
|
320
|
-
out.response = { ...p.response, headers: {} };
|
|
321
|
-
if (performance)
|
|
322
|
-
out.performance = { ...performance, opportunities: performance.opportunities.slice(0, 5), agenticBrowsingAudits: [], failedSeoAudits: performance.failedSeoAudits.slice(0, 5) };
|
|
323
|
-
if (rendering)
|
|
324
|
-
out.rendering = { ...rendering, consoleErrors: rendering.consoleErrors.slice(0, 3) };
|
|
325
|
-
return out;
|
|
326
|
-
};
|
|
327
|
-
if (audit.kind === 'page')
|
|
437
|
+
export const compactAudit = (audit) => {
|
|
438
|
+
if (audit.kind === "page") {
|
|
439
|
+
// SAFETY: T is the page audit here (kind is "page"), and trimPage returns a page audit with
|
|
440
|
+
// every field it was given.
|
|
328
441
|
return trimPage(audit);
|
|
442
|
+
}
|
|
443
|
+
// SAFETY: T is the site audit here; the spread keeps every field and only shortens lists.
|
|
329
444
|
return {
|
|
330
445
|
...audit,
|
|
331
446
|
robotsTxt: audit.robotsTxt ? { ...audit.robotsTxt, text: null } : null,
|
|
@@ -333,5 +448,5 @@ export function compactAudit(audit) {
|
|
|
333
448
|
pages: audit.pages.map(trimPage),
|
|
334
449
|
siteFindings: audit.siteFindings.map(trimFinding),
|
|
335
450
|
};
|
|
336
|
-
}
|
|
451
|
+
};
|
|
337
452
|
//# sourceMappingURL=audit.js.map
|