ax-audit 3.0.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -0
- package/README.md +61 -221
- package/dist/checks/agent-access.d.ts +16 -0
- package/dist/checks/agent-access.d.ts.map +1 -0
- package/dist/checks/agent-access.js +110 -0
- package/dist/checks/agent-access.js.map +1 -0
- package/dist/checks/content-negotiation.d.ts +4 -0
- package/dist/checks/content-negotiation.d.ts.map +1 -0
- package/dist/checks/content-negotiation.js +138 -0
- package/dist/checks/content-negotiation.js.map +1 -0
- package/dist/checks/crawl-efficiency.d.ts +4 -0
- package/dist/checks/crawl-efficiency.d.ts.map +1 -0
- package/dist/checks/crawl-efficiency.js +122 -0
- package/dist/checks/crawl-efficiency.js.map +1 -0
- package/dist/checks/index.d.ts.map +1 -1
- package/dist/checks/index.js +8 -0
- package/dist/checks/index.js.map +1 -1
- package/dist/checks/robots-txt.d.ts +20 -0
- package/dist/checks/robots-txt.d.ts.map +1 -1
- package/dist/checks/robots-txt.js +111 -3
- package/dist/checks/robots-txt.js.map +1 -1
- package/dist/checks/rsl.d.ts +6 -0
- package/dist/checks/rsl.d.ts.map +1 -0
- package/dist/checks/rsl.js +252 -0
- package/dist/checks/rsl.js.map +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +20 -2
- package/dist/cli.js.map +1 -1
- package/dist/constants.d.ts +17 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +42 -1
- package/dist/constants.js.map +1 -1
- package/dist/fetcher.d.ts +7 -3
- package/dist/fetcher.d.ts.map +1 -1
- package/dist/fetcher.js +68 -30
- package/dist/fetcher.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/orchestrator.d.ts +2 -2
- package/dist/orchestrator.d.ts.map +1 -1
- package/dist/orchestrator.js +13 -6
- package/dist/orchestrator.js.map +1 -1
- package/dist/reporter/index.d.ts.map +1 -1
- package/dist/reporter/index.js +7 -0
- package/dist/reporter/index.js.map +1 -1
- package/dist/reporter/markdown.d.ts +8 -0
- package/dist/reporter/markdown.d.ts.map +1 -0
- package/dist/reporter/markdown.js +76 -0
- package/dist/reporter/markdown.js.map +1 -0
- package/dist/scorer.d.ts.map +1 -1
- package/dist/scorer.js +8 -0
- package/dist/scorer.js.map +1 -1
- package/dist/types.d.ts +17 -2
- package/dist/types.d.ts.map +1 -1
- package/docs/api.md +200 -0
- package/docs/architecture.md +88 -0
- package/docs/checks.md +322 -0
- package/docs/ci.md +89 -0
- package/docs/cli.md +67 -0
- package/docs/concepts.md +87 -0
- package/docs/faq.md +77 -0
- package/docs/getting-started.md +101 -0
- package/package.json +2 -1
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { guideUrl } from '../guide-urls.js';
|
|
2
|
+
import { buildResult } from './utils.js';
|
|
3
|
+
import { findLinkTags, getAttribute } from './html-utils.js';
|
|
4
|
+
export const meta = {
|
|
5
|
+
id: 'content-negotiation',
|
|
6
|
+
name: 'Content Negotiation',
|
|
7
|
+
description: 'Checks whether the homepage serves Markdown to AI agents via Accept: text/markdown',
|
|
8
|
+
weight: 0, // Informational in 3.x — will gain weight in v4.0 (score-affecting changes are treated as breaking).
|
|
9
|
+
};
|
|
10
|
+
/** Markdown sent by agents that support content negotiation (Claude Code, Cursor, OpenCode). */
|
|
11
|
+
const MARKDOWN_ACCEPT = 'text/markdown';
|
|
12
|
+
/** Score when Markdown is not negotiated but a `<link rel="alternate" type="text/markdown">` exists. */
|
|
13
|
+
const ALTERNATE_ONLY_SCORE = 40;
|
|
14
|
+
/**
|
|
15
|
+
* Detect an HTML document masquerading as Markdown. Markdown may legally
|
|
16
|
+
* contain inline HTML, so this only flags full documents (doctype / <html> /
|
|
17
|
+
* <head> at the start), not embedded tags.
|
|
18
|
+
*/
|
|
19
|
+
function looksLikeHtmlDocument(body) {
|
|
20
|
+
return /^\s*(?:<!doctype\s+html|<html[\s>]|<head[\s>])/i.test(body);
|
|
21
|
+
}
|
|
22
|
+
/** Find `<link rel="alternate" type="text/markdown">` tags in the homepage HTML. */
|
|
23
|
+
function findMarkdownAlternates(html) {
|
|
24
|
+
return findLinkTags(html, 'alternate').filter((tag) => {
|
|
25
|
+
const type = getAttribute(tag, 'type');
|
|
26
|
+
return type !== null && type.toLowerCase().includes('text/markdown');
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
export default async function check(ctx) {
|
|
30
|
+
const start = performance.now();
|
|
31
|
+
const findings = [];
|
|
32
|
+
const res = await ctx.fetch(ctx.url, { headers: { Accept: MARKDOWN_ACCEPT } });
|
|
33
|
+
if (res.status === 0) {
|
|
34
|
+
findings.push({
|
|
35
|
+
status: 'fail',
|
|
36
|
+
message: 'Could not fetch homepage with "Accept: text/markdown"',
|
|
37
|
+
detail: res.error ?? 'Network error',
|
|
38
|
+
learnMoreUrl: guideUrl(meta.id, 'fetch-error'),
|
|
39
|
+
});
|
|
40
|
+
return buildResult(meta, 0, findings, start);
|
|
41
|
+
}
|
|
42
|
+
const contentType = (res.headers['content-type'] ?? '').toLowerCase();
|
|
43
|
+
const servesMarkdown = res.ok && contentType.includes(MARKDOWN_ACCEPT);
|
|
44
|
+
if (!servesMarkdown) {
|
|
45
|
+
findings.push({
|
|
46
|
+
status: 'fail',
|
|
47
|
+
message: 'Homepage does not serve Markdown via content negotiation',
|
|
48
|
+
detail: res.status === 406
|
|
49
|
+
? 'Server responded 406 Not Acceptable to "Accept: text/markdown"'
|
|
50
|
+
: `Got ${contentType.split(';')[0] || 'no Content-Type'} (HTTP ${res.status}) for "Accept: text/markdown"`,
|
|
51
|
+
hint: 'Serve a Markdown representation of your pages when agents request "Accept: text/markdown". ' +
|
|
52
|
+
'Agents like Claude Code and Cursor ask for it, and Markdown cuts token usage by ~80% vs HTML. ' +
|
|
53
|
+
'Cloudflare ("Markdown for Agents") and Vercel can enable this without code changes.',
|
|
54
|
+
learnMoreUrl: guideUrl(meta.id, res.status === 406 ? 'http-406' : 'not-supported'),
|
|
55
|
+
});
|
|
56
|
+
const alternates = findMarkdownAlternates(ctx.html);
|
|
57
|
+
if (alternates.length > 0) {
|
|
58
|
+
findings.push({
|
|
59
|
+
status: 'pass',
|
|
60
|
+
message: `Markdown alternate advertised via <link rel="alternate" type="text/markdown"> (${alternates.length} link tag(s))`,
|
|
61
|
+
detail: 'Discoverable, but agents must perform an extra fetch instead of negotiating the same URL.',
|
|
62
|
+
});
|
|
63
|
+
return buildResult(meta, ALTERNATE_ONLY_SCORE, findings, start);
|
|
64
|
+
}
|
|
65
|
+
findings.push({
|
|
66
|
+
status: 'warn',
|
|
67
|
+
message: 'No <link rel="alternate" type="text/markdown"> fallback found on the homepage',
|
|
68
|
+
hint: 'If you cannot enable content negotiation, advertise a Markdown version with ' +
|
|
69
|
+
'<link rel="alternate" type="text/markdown" href="/index.md"> so agents can discover it.',
|
|
70
|
+
learnMoreUrl: guideUrl(meta.id, 'no-alternate'),
|
|
71
|
+
});
|
|
72
|
+
return buildResult(meta, 0, findings, start);
|
|
73
|
+
}
|
|
74
|
+
let score = 100;
|
|
75
|
+
findings.push({
|
|
76
|
+
status: 'pass',
|
|
77
|
+
message: 'Homepage serves Markdown via content negotiation (Accept: text/markdown)',
|
|
78
|
+
});
|
|
79
|
+
if (res.body.trim().length === 0) {
|
|
80
|
+
findings.push({
|
|
81
|
+
status: 'warn',
|
|
82
|
+
message: 'Markdown response body is empty',
|
|
83
|
+
hint: 'Return the page content as Markdown — an empty body gives agents nothing to work with.',
|
|
84
|
+
learnMoreUrl: guideUrl(meta.id, 'empty-body'),
|
|
85
|
+
});
|
|
86
|
+
score -= 30;
|
|
87
|
+
}
|
|
88
|
+
else if (looksLikeHtmlDocument(res.body)) {
|
|
89
|
+
findings.push({
|
|
90
|
+
status: 'warn',
|
|
91
|
+
message: 'Response is labeled text/markdown but the body is an HTML document',
|
|
92
|
+
hint: 'Convert the page to actual Markdown instead of relabeling the HTML response.',
|
|
93
|
+
learnMoreUrl: guideUrl(meta.id, 'mislabeled-content-type'),
|
|
94
|
+
});
|
|
95
|
+
score -= 25;
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
findings.push({ status: 'pass', message: 'Response body is Markdown, not an HTML document' });
|
|
99
|
+
}
|
|
100
|
+
const vary = (res.headers['vary'] ?? '').toLowerCase();
|
|
101
|
+
const variesOnAccept = vary
|
|
102
|
+
.split(',')
|
|
103
|
+
.map((v) => v.trim())
|
|
104
|
+
.some((v) => v === 'accept' || v === '*');
|
|
105
|
+
if (variesOnAccept) {
|
|
106
|
+
findings.push({ status: 'pass', message: 'Vary: Accept present (caches keep HTML and Markdown apart)' });
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
findings.push({
|
|
110
|
+
status: 'warn',
|
|
111
|
+
message: 'Vary header does not include "Accept"',
|
|
112
|
+
detail: vary ? `Vary: ${res.headers['vary']}` : 'No Vary header',
|
|
113
|
+
hint: 'Send "Vary: Accept" when the same URL serves both HTML and Markdown, ' +
|
|
114
|
+
'otherwise shared caches and CDNs may serve Markdown to browsers (or HTML to agents).',
|
|
115
|
+
learnMoreUrl: guideUrl(meta.id, 'missing-vary'),
|
|
116
|
+
});
|
|
117
|
+
score -= 15;
|
|
118
|
+
}
|
|
119
|
+
if (ctx.html.length > 0 && res.body.length > 0) {
|
|
120
|
+
const reduction = Math.round((1 - res.body.length / ctx.html.length) * 100);
|
|
121
|
+
if (reduction > 0) {
|
|
122
|
+
findings.push({
|
|
123
|
+
status: 'pass',
|
|
124
|
+
message: `Markdown is ~${reduction}% lighter than the HTML representation (${res.body.length} vs ${ctx.html.length} bytes)`,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
findings.push({
|
|
129
|
+
status: 'warn',
|
|
130
|
+
message: `Markdown response is not smaller than the HTML representation (${res.body.length} vs ${ctx.html.length} bytes)`,
|
|
131
|
+
hint: 'Strip navigation, boilerplate, and markup remnants from the Markdown output — its purpose is token efficiency.',
|
|
132
|
+
learnMoreUrl: guideUrl(meta.id, 'not-smaller'),
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return buildResult(meta, score, findings, start);
|
|
137
|
+
}
|
|
138
|
+
//# sourceMappingURL=content-negotiation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content-negotiation.js","sourceRoot":"","sources":["../../src/checks/content-negotiation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE7D,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,qBAAqB;IACzB,IAAI,EAAE,qBAAqB;IAC3B,WAAW,EAAE,oFAAoF;IACjG,MAAM,EAAE,CAAC,EAAE,qGAAqG;CACjH,CAAC;AAEF,gGAAgG;AAChG,MAAM,eAAe,GAAG,eAAe,CAAC;AAExC,wGAAwG;AACxG,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,IAAY;IACzC,OAAO,iDAAiD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACtE,CAAC;AAED,oFAAoF;AACpF,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QACpD,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACvC,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;IAE/E,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uDAAuD;YAChE,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,eAAe;YACpC,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC;SAC/C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACtE,MAAM,cAAc,GAAG,GAAG,CAAC,EAAE,IAAI,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IAEvE,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,0DAA0D;YACnE,MAAM,EACJ,GAAG,CAAC,MAAM,KAAK,GAAG;gBAChB,CAAC,CAAC,gEAAgE;gBAClE,CAAC,CAAC,OAAO,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,iBAAiB,UAAU,GAAG,CAAC,MAAM,+BAA+B;YAC9G,IAAI,EACF,6FAA6F;gBAC7F,gGAAgG;gBAChG,qFAAqF;YACvF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC;SACnF,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,kFAAkF,UAAU,CAAC,MAAM,eAAe;gBAC3H,MAAM,EAAE,2FAA2F;aACpG,CAAC,CAAC;YACH,OAAO,WAAW,CAAC,IAAI,EAAE,oBAAoB,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAClE,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,+EAA+E;YACxF,IAAI,EACF,8EAA8E;gBAC9E,yFAAyF;YAC3F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;SAChD,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,KAAK,GAAG,GAAG,CAAC;IAChB,QAAQ,CAAC,IAAI,CAAC;QACZ,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,0EAA0E;KACpF,CAAC,CAAC;IAEH,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,iCAAiC;YAC1C,IAAI,EAAE,wFAAwF;YAC9F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC;SAC9C,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;SAAM,IAAI,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,oEAAoE;YAC7E,IAAI,EAAE,8EAA8E;YACpF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,yBAAyB,CAAC;SAC3D,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iDAAiD,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACvD,MAAM,cAAc,GAAG,IAAI;SACxB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IAC5C,IAAI,cAAc,EAAE,CAAC;QACnB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,4DAA4D,EAAE,CAAC,CAAC;IAC3G,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uCAAuC;YAChD,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,gBAAgB;YAChE,IAAI,EACF,uEAAuE;gBACvE,sFAAsF;YACxF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;SAChD,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;IAED,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;QAC5E,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,gBAAgB,SAAS,2CAA2C,GAAG,CAAC,IAAI,CAAC,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,SAAS;aAC5H,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,kEAAkE,GAAG,CAAC,IAAI,CAAC,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,SAAS;gBACzH,IAAI,EAAE,gHAAgH;gBACtH,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC;aAC/C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawl-efficiency.d.ts","sourceRoot":"","sources":["../../src/checks/crawl-efficiency.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAGjF,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;AAMF,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA8G3E"}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { guideUrl } from '../guide-urls.js';
|
|
2
|
+
import { buildResult } from './utils.js';
|
|
3
|
+
export const meta = {
|
|
4
|
+
id: 'crawl-efficiency',
|
|
5
|
+
name: 'Crawl Efficiency',
|
|
6
|
+
description: 'Checks compression, conditional GET (ETag / Last-Modified), and response size',
|
|
7
|
+
weight: 0, // Informational in 3.x — will gain weight in v4.0 (score-affecting changes are treated as breaking).
|
|
8
|
+
};
|
|
9
|
+
/** Page sizes (decompressed HTML bytes) above this earn a warning. */
|
|
10
|
+
const LARGE_PAGE_BYTES = 2_000_000;
|
|
11
|
+
const ACCEPTABLE_PAGE_BYTES = 500_000;
|
|
12
|
+
export default async function check(ctx) {
|
|
13
|
+
const start = performance.now();
|
|
14
|
+
const findings = [];
|
|
15
|
+
let score = 100;
|
|
16
|
+
// Request explicitly advertising modern compression. The runtime's fetch
|
|
17
|
+
// transparently decompresses the body, but the Content-Encoding header still
|
|
18
|
+
// reflects what the server sent on the wire.
|
|
19
|
+
const res = await ctx.fetch(ctx.url, { headers: { 'Accept-Encoding': 'br, gzip, deflate' } });
|
|
20
|
+
if (!res.ok) {
|
|
21
|
+
findings.push({
|
|
22
|
+
status: 'fail',
|
|
23
|
+
message: 'Homepage request failed — cannot assess crawl efficiency',
|
|
24
|
+
detail: `HTTP ${res.status || 'network error'}`,
|
|
25
|
+
learnMoreUrl: guideUrl(meta.id, 'fetch-failed'),
|
|
26
|
+
});
|
|
27
|
+
return buildResult(meta, 0, findings, start);
|
|
28
|
+
}
|
|
29
|
+
/* ── Compression ──────────────────────────────────────────────────── */
|
|
30
|
+
const encoding = (res.headers['content-encoding'] ?? '').toLowerCase().trim();
|
|
31
|
+
if (encoding === 'br') {
|
|
32
|
+
findings.push({ status: 'pass', message: 'Response compressed with Brotli (br)' });
|
|
33
|
+
}
|
|
34
|
+
else if (encoding === 'gzip' || encoding === 'deflate' || encoding === 'zstd') {
|
|
35
|
+
findings.push({
|
|
36
|
+
status: 'pass',
|
|
37
|
+
message: `Response compressed with ${encoding}`,
|
|
38
|
+
detail: 'Brotli (br) typically compresses text 10–20% smaller — consider enabling it.',
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
findings.push({
|
|
43
|
+
status: 'warn',
|
|
44
|
+
message: 'Response is not compressed',
|
|
45
|
+
detail: encoding ? `Content-Encoding: ${encoding}` : 'No Content-Encoding header',
|
|
46
|
+
hint: 'Enable Brotli or gzip for text responses. Compression cuts bytes transferred by ~70–80% and ' +
|
|
47
|
+
'reduces crawl cost for every agent and bot that fetches your pages.',
|
|
48
|
+
learnMoreUrl: guideUrl(meta.id, 'no-compression'),
|
|
49
|
+
});
|
|
50
|
+
score -= 30;
|
|
51
|
+
}
|
|
52
|
+
/* ── Conditional GET (ETag / Last-Modified → 304) ─────────────────── */
|
|
53
|
+
const etag = res.headers['etag'];
|
|
54
|
+
const lastModified = res.headers['last-modified'];
|
|
55
|
+
if (!etag && !lastModified) {
|
|
56
|
+
findings.push({
|
|
57
|
+
status: 'warn',
|
|
58
|
+
message: 'No ETag or Last-Modified header — conditional requests unsupported',
|
|
59
|
+
hint: 'Send an ETag or Last-Modified header so crawlers can revalidate with If-None-Match / ' +
|
|
60
|
+
'If-Modified-Since and receive a cheap 304 Not Modified instead of the full body.',
|
|
61
|
+
learnMoreUrl: guideUrl(meta.id, 'no-validators'),
|
|
62
|
+
});
|
|
63
|
+
score -= 30;
|
|
64
|
+
}
|
|
65
|
+
else {
|
|
66
|
+
const validator = etag ? 'ETag' : 'Last-Modified';
|
|
67
|
+
findings.push({ status: 'pass', message: `Cache validator present (${validator})` });
|
|
68
|
+
const conditionalHeaders = {};
|
|
69
|
+
if (etag)
|
|
70
|
+
conditionalHeaders['If-None-Match'] = etag;
|
|
71
|
+
if (lastModified)
|
|
72
|
+
conditionalHeaders['If-Modified-Since'] = lastModified;
|
|
73
|
+
const conditional = await ctx.fetch(ctx.url, { headers: conditionalHeaders });
|
|
74
|
+
if (conditional.status === 304) {
|
|
75
|
+
findings.push({ status: 'pass', message: 'Conditional request returns 304 Not Modified' });
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
findings.push({
|
|
79
|
+
status: 'warn',
|
|
80
|
+
message: `Conditional request returned ${conditional.status} instead of 304 Not Modified`,
|
|
81
|
+
detail: `Re-requested with ${Object.keys(conditionalHeaders).join(' + ')}`,
|
|
82
|
+
hint: 'The server advertises a cache validator but does not honor If-None-Match / If-Modified-Since. ' +
|
|
83
|
+
'Configure it to return 304 when the validator matches, so crawlers avoid re-downloading unchanged pages.',
|
|
84
|
+
learnMoreUrl: guideUrl(meta.id, 'no-304'),
|
|
85
|
+
});
|
|
86
|
+
score -= 15;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/* ── Response size ────────────────────────────────────────────────── */
|
|
90
|
+
const bytes = Buffer.byteLength(res.body, 'utf8');
|
|
91
|
+
if (bytes > LARGE_PAGE_BYTES) {
|
|
92
|
+
findings.push({
|
|
93
|
+
status: 'warn',
|
|
94
|
+
message: `Homepage is very large (${formatBytes(bytes)} decompressed)`,
|
|
95
|
+
hint: 'Large documents inflate crawl cost and token usage. Trim inlined data, split content, ' +
|
|
96
|
+
'or serve a Markdown representation to agents (see the content-negotiation check).',
|
|
97
|
+
learnMoreUrl: guideUrl(meta.id, 'large-page'),
|
|
98
|
+
});
|
|
99
|
+
score -= 10;
|
|
100
|
+
}
|
|
101
|
+
else if (bytes > ACCEPTABLE_PAGE_BYTES) {
|
|
102
|
+
findings.push({
|
|
103
|
+
status: 'warn',
|
|
104
|
+
message: `Homepage is on the large side (${formatBytes(bytes)} decompressed)`,
|
|
105
|
+
hint: 'Consider trimming inlined payloads to reduce crawl cost.',
|
|
106
|
+
learnMoreUrl: guideUrl(meta.id, 'large-page'),
|
|
107
|
+
});
|
|
108
|
+
score -= 5;
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
findings.push({ status: 'pass', message: `Homepage size is reasonable (${formatBytes(bytes)} decompressed)` });
|
|
112
|
+
}
|
|
113
|
+
return buildResult(meta, score, findings, start);
|
|
114
|
+
}
|
|
115
|
+
function formatBytes(bytes) {
|
|
116
|
+
if (bytes < 1024)
|
|
117
|
+
return `${bytes} B`;
|
|
118
|
+
if (bytes < 1024 * 1024)
|
|
119
|
+
return `${(bytes / 1024).toFixed(1)} KB`;
|
|
120
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
121
|
+
}
|
|
122
|
+
//# sourceMappingURL=crawl-efficiency.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawl-efficiency.js","sourceRoot":"","sources":["../../src/checks/crawl-efficiency.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,kBAAkB;IACtB,IAAI,EAAE,kBAAkB;IACxB,WAAW,EAAE,+EAA+E;IAC5F,MAAM,EAAE,CAAC,EAAE,qGAAqG;CACjH,CAAC;AAEF,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,MAAM,qBAAqB,GAAG,OAAO,CAAC;AAEtC,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAG,GAAG,CAAC;IAEhB,yEAAyE;IACzE,6EAA6E;IAC7E,6CAA6C;IAC7C,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,EAAE,CAAC,CAAC;IAE9F,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,0DAA0D;YACnE,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,eAAe,EAAE;YAC/C,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;SAChD,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,yEAAyE;IAEzE,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IAC9E,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,sCAAsC,EAAE,CAAC,CAAC;IACrF,CAAC;SAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QAChF,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4BAA4B,QAAQ,EAAE;YAC/C,MAAM,EAAE,8EAA8E;SACvF,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4BAA4B;YACrC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC,CAAC,4BAA4B;YACjF,IAAI,EACF,8FAA8F;gBAC9F,qEAAqE;YACvE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC;SAClD,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;IAED,yEAAyE;IAEzE,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAElD,IAAI,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,oEAAoE;YAC7E,IAAI,EACF,uFAAuF;gBACvF,kFAAkF;YACpF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,eAAe,CAAC;SACjD,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;SAAM,CAAC;QACN,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC;QAClD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,4BAA4B,SAAS,GAAG,EAAE,CAAC,CAAC;QAErF,MAAM,kBAAkB,GAA2B,EAAE,CAAC;QACtD,IAAI,IAAI;YAAE,kBAAkB,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC;QACrD,IAAI,YAAY;YAAE,kBAAkB,CAAC,mBAAmB,CAAC,GAAG,YAAY,CAAC;QAEzE,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAC9E,IAAI,WAAW,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC/B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,8CAA8C,EAAE,CAAC,CAAC;QAC7F,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,gCAAgC,WAAW,CAAC,MAAM,8BAA8B;gBACzF,MAAM,EAAE,qBAAqB,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAC1E,IAAI,EACF,gGAAgG;oBAChG,0GAA0G;gBAC5G,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC;aAC1C,CAAC,CAAC;YACH,KAAK,IAAI,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IAED,yEAAyE;IAEzE,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,2BAA2B,WAAW,CAAC,KAAK,CAAC,gBAAgB;YACtE,IAAI,EACF,wFAAwF;gBACxF,mFAAmF;YACrF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC;SAC9C,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;SAAM,IAAI,KAAK,GAAG,qBAAqB,EAAE,CAAC;QACzC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,kCAAkC,WAAW,CAAC,KAAK,CAAC,gBAAgB;YAC7E,IAAI,EAAE,0DAA0D;YAChE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC;SAC9C,CAAC,CAAC;QACH,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,gCAAgC,WAAW,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACjH,CAAC;IAED,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,KAAK,GAAG,IAAI;QAAE,OAAO,GAAG,KAAK,IAAI,CAAC;IACtC,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAClE,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;AACpD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAqB/C,eAAO,MAAM,MAAM,EAAE,WAAW,EAmB/B,CAAC"}
|
package/dist/checks/index.js
CHANGED
|
@@ -12,6 +12,10 @@ import sitemap, { meta as sitemapMeta } from './sitemap.js';
|
|
|
12
12
|
import seoBasics, { meta as seoBasicsMeta } from './seo-basics.js';
|
|
13
13
|
import tlsHttps, { meta as tlsHttpsMeta } from './tls-https.js';
|
|
14
14
|
import wellKnownAi, { meta as wellKnownAiMeta } from './well-known-ai.js';
|
|
15
|
+
import contentNegotiation, { meta as contentNegotiationMeta } from './content-negotiation.js';
|
|
16
|
+
import rsl, { meta as rslMeta } from './rsl.js';
|
|
17
|
+
import agentAccess, { meta as agentAccessMeta } from './agent-access.js';
|
|
18
|
+
import crawlEfficiency, { meta as crawlEfficiencyMeta } from './crawl-efficiency.js';
|
|
15
19
|
export const checks = [
|
|
16
20
|
{ run: llmsTxt, meta: llmsTxtMeta },
|
|
17
21
|
{ run: robotsTxt, meta: robotsTxtMeta },
|
|
@@ -27,5 +31,9 @@ export const checks = [
|
|
|
27
31
|
{ run: seoBasics, meta: seoBasicsMeta },
|
|
28
32
|
{ run: tlsHttps, meta: tlsHttpsMeta },
|
|
29
33
|
{ run: wellKnownAi, meta: wellKnownAiMeta },
|
|
34
|
+
{ run: contentNegotiation, meta: contentNegotiationMeta },
|
|
35
|
+
{ run: rsl, meta: rslMeta },
|
|
36
|
+
{ run: agentAccess, meta: agentAccessMeta },
|
|
37
|
+
{ run: crawlEfficiency, meta: crawlEfficiencyMeta },
|
|
30
38
|
];
|
|
31
39
|
//# sourceMappingURL=index.js.map
|
package/dist/checks/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAEA,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,cAAc,EAAE,EAAE,IAAI,IAAI,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,QAAQ,EAAE,EAAE,IAAI,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,GAAG,EAAE,EAAE,IAAI,IAAI,OAAO,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,aAAa,EAAE,EAAE,IAAI,IAAI,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,QAAQ,EAAE,EAAE,IAAI,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAEA,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,cAAc,EAAE,EAAE,IAAI,IAAI,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,QAAQ,EAAE,EAAE,IAAI,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,GAAG,EAAE,EAAE,IAAI,IAAI,OAAO,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,aAAa,EAAE,EAAE,IAAI,IAAI,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,QAAQ,EAAE,EAAE,IAAI,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAO,kBAAkB,EAAE,EAAE,IAAI,IAAI,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAC9F,OAAO,GAAG,EAAE,EAAE,IAAI,IAAI,OAAO,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,eAAe,EAAE,EAAE,IAAI,IAAI,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAErF,MAAM,CAAC,MAAM,MAAM,GAAkB;IACnC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE;IACnC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE;IACvC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE;IACvC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACjD,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE;IACrC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE;IACnC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3B,EAAE,GAAG,EAAE,aAAa,EAAE,IAAI,EAAE,iBAAiB,EAAE;IAC/C,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE;IACnC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE;IACvC,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE;IACrC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,sBAAsB,EAAE;IACzD,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3B,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,mBAAmB,EAAE;CACpD,CAAC"}
|
|
@@ -1,4 +1,24 @@
|
|
|
1
1
|
import type { CheckContext, CheckResult, CheckMeta } from '../types.js';
|
|
2
2
|
export declare const meta: CheckMeta;
|
|
3
|
+
export interface BotEntry {
|
|
4
|
+
name: string;
|
|
5
|
+
disallowed: boolean;
|
|
6
|
+
hasRestrictions: boolean;
|
|
7
|
+
hasAllow: boolean;
|
|
8
|
+
}
|
|
3
9
|
export default function check(ctx: CheckContext): Promise<CheckResult>;
|
|
10
|
+
export declare function parseUserAgents(text: string): BotEntry[];
|
|
11
|
+
interface ContentSignalDecl {
|
|
12
|
+
/** User-agent names of the group this declaration belongs to (empty = outside any group). */
|
|
13
|
+
userAgents: string[];
|
|
14
|
+
/** Raw directive value, e.g. "search=yes, ai-train=no". */
|
|
15
|
+
raw: string;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Extract `Content-Signal:` declarations and the User-agent group each belongs to.
|
|
19
|
+
* Mirrors the grouping rules of `parseUserAgents`: consecutive User-agent lines form
|
|
20
|
+
* one group; any directive line closes the group.
|
|
21
|
+
*/
|
|
22
|
+
export declare function parseContentSignalDecls(text: string): ContentSignalDecl[];
|
|
23
|
+
export {};
|
|
4
24
|
//# sourceMappingURL=robots-txt.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robots-txt.d.ts","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAGjF,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;
|
|
1
|
+
{"version":3,"file":"robots-txt.d.ts","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAGjF,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;AAEF,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,OAAO,CAAC;IACpB,eAAe,EAAE,OAAO,CAAC;IACzB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAkI3E;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE,CAwCxD;AAID,UAAU,iBAAiB;IACzB,6FAA6F;IAC7F,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,2DAA2D;IAC3D,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAgCzE"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ALL_AI_CRAWLERS, CORE_AI_CRAWLERS } from '../constants.js';
|
|
1
|
+
import { ALL_AI_CRAWLERS, CONTENT_SIGNALS, CORE_AI_CRAWLERS } from '../constants.js';
|
|
2
2
|
import { guideUrl } from '../guide-urls.js';
|
|
3
3
|
import { buildResult } from './utils.js';
|
|
4
4
|
export const meta = {
|
|
@@ -101,6 +101,8 @@ export default async function check(ctx) {
|
|
|
101
101
|
});
|
|
102
102
|
score -= 5;
|
|
103
103
|
}
|
|
104
|
+
// Content Signals Policy (contentsignals.org) — informational in 3.x: findings only, no score impact.
|
|
105
|
+
addContentSignalFindings(text, findings);
|
|
104
106
|
const totalConfigured = ALL_AI_CRAWLERS.filter((bot) => configuredBots.some((b) => b.name.toLowerCase() === bot.toLowerCase()));
|
|
105
107
|
findings.push({
|
|
106
108
|
status: totalConfigured.length >= 10 ? 'pass' : 'warn',
|
|
@@ -114,7 +116,7 @@ export default async function check(ctx) {
|
|
|
114
116
|
});
|
|
115
117
|
return buildResult(meta, score, findings, start);
|
|
116
118
|
}
|
|
117
|
-
function parseUserAgents(text) {
|
|
119
|
+
export function parseUserAgents(text) {
|
|
118
120
|
const entries = [];
|
|
119
121
|
let currentGroup = [];
|
|
120
122
|
let inDirectives = false;
|
|
@@ -151,10 +153,116 @@ function parseUserAgents(text) {
|
|
|
151
153
|
entry.hasAllow = true;
|
|
152
154
|
}
|
|
153
155
|
}
|
|
154
|
-
else if (/^(Sitemap|Crawl-delay|Host):/i.test(trimmed)) {
|
|
156
|
+
else if (/^(Sitemap|Crawl-delay|Host|Content-Signal):/i.test(trimmed)) {
|
|
155
157
|
inDirectives = true;
|
|
156
158
|
}
|
|
157
159
|
}
|
|
158
160
|
return entries;
|
|
159
161
|
}
|
|
162
|
+
/**
|
|
163
|
+
* Extract `Content-Signal:` declarations and the User-agent group each belongs to.
|
|
164
|
+
* Mirrors the grouping rules of `parseUserAgents`: consecutive User-agent lines form
|
|
165
|
+
* one group; any directive line closes the group.
|
|
166
|
+
*/
|
|
167
|
+
export function parseContentSignalDecls(text) {
|
|
168
|
+
const decls = [];
|
|
169
|
+
let currentUAs = [];
|
|
170
|
+
let inDirectives = false;
|
|
171
|
+
for (const line of text.split('\n')) {
|
|
172
|
+
const trimmed = line.trim();
|
|
173
|
+
if (!trimmed || trimmed.startsWith('#'))
|
|
174
|
+
continue;
|
|
175
|
+
const uaMatch = trimmed.match(/^User-agent:\s*(.+)/i);
|
|
176
|
+
if (uaMatch) {
|
|
177
|
+
if (inDirectives) {
|
|
178
|
+
currentUAs = [];
|
|
179
|
+
inDirectives = false;
|
|
180
|
+
}
|
|
181
|
+
currentUAs.push(uaMatch[1].trim());
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
const csMatch = trimmed.match(/^Content-Signal:\s*(.+)/i);
|
|
185
|
+
if (csMatch) {
|
|
186
|
+
inDirectives = true;
|
|
187
|
+
decls.push({ userAgents: [...currentUAs], raw: csMatch[1].trim() });
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
if (/^(Disallow|Allow|Sitemap|Crawl-delay|Host):/i.test(trimmed)) {
|
|
191
|
+
inDirectives = true;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return decls;
|
|
195
|
+
}
|
|
196
|
+
function parseSignalPairs(raw) {
|
|
197
|
+
const result = { valid: [], malformed: [], unknown: [] };
|
|
198
|
+
for (const segment of raw.split(',')) {
|
|
199
|
+
const s = segment.trim();
|
|
200
|
+
if (!s)
|
|
201
|
+
continue;
|
|
202
|
+
const m = s.match(/^([a-z][a-z-]*)\s*=\s*(yes|no)$/i);
|
|
203
|
+
if (!m) {
|
|
204
|
+
result.malformed.push(s);
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
const name = m[1].toLowerCase();
|
|
208
|
+
if (!CONTENT_SIGNALS.includes(name)) {
|
|
209
|
+
result.unknown.push(s);
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
result.valid.push(`${name}=${m[2].toLowerCase()}`);
|
|
213
|
+
}
|
|
214
|
+
return result;
|
|
215
|
+
}
|
|
216
|
+
function addContentSignalFindings(text, findings) {
|
|
217
|
+
const decls = parseContentSignalDecls(text);
|
|
218
|
+
if (decls.length === 0) {
|
|
219
|
+
findings.push({
|
|
220
|
+
status: 'warn',
|
|
221
|
+
message: 'No Content-Signal directive found (optional)',
|
|
222
|
+
hint: 'Declare how crawlers may use your content after access with the Content Signals Policy, ' +
|
|
223
|
+
'e.g.: Content-Signal: search=yes, ai-train=no. Known signals: ' +
|
|
224
|
+
`${CONTENT_SIGNALS.join(', ')}. Generate yours at contentsignals.org.`,
|
|
225
|
+
learnMoreUrl: guideUrl(meta.id, 'missing-content-signals'),
|
|
226
|
+
});
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
for (const decl of decls) {
|
|
230
|
+
const group = decl.userAgents.length > 0 ? decl.userAgents.join(', ') : null;
|
|
231
|
+
const { valid, malformed, unknown } = parseSignalPairs(decl.raw);
|
|
232
|
+
if (group === null) {
|
|
233
|
+
findings.push({
|
|
234
|
+
status: 'warn',
|
|
235
|
+
message: 'Content-Signal directive outside a User-agent group',
|
|
236
|
+
detail: `Content-Signal: ${decl.raw}`,
|
|
237
|
+
hint: 'Place Content-Signal inside a User-agent group, after the User-agent line it applies to.',
|
|
238
|
+
learnMoreUrl: guideUrl(meta.id, 'invalid-content-signal'),
|
|
239
|
+
});
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
if (valid.length > 0) {
|
|
243
|
+
findings.push({
|
|
244
|
+
status: 'pass',
|
|
245
|
+
message: `Content signals declared for User-agent: ${group} — ${valid.join(', ')}`,
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
if (malformed.length > 0) {
|
|
249
|
+
findings.push({
|
|
250
|
+
status: 'warn',
|
|
251
|
+
message: `Malformed content signal segment(s) for User-agent: ${group}`,
|
|
252
|
+
detail: malformed.join(', '),
|
|
253
|
+
hint: 'Use comma-delimited signal=yes|no pairs, e.g.: Content-Signal: search=yes, ai-train=no.',
|
|
254
|
+
learnMoreUrl: guideUrl(meta.id, 'invalid-content-signal'),
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
if (unknown.length > 0) {
|
|
258
|
+
findings.push({
|
|
259
|
+
status: 'warn',
|
|
260
|
+
message: `Unknown content signal name(s) for User-agent: ${group}`,
|
|
261
|
+
detail: unknown.join(', '),
|
|
262
|
+
hint: `The Content Signals Policy defines: ${CONTENT_SIGNALS.join(', ')}. Other names are ignored by crawlers.`,
|
|
263
|
+
learnMoreUrl: guideUrl(meta.id, 'unknown-content-signal'),
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
}
|
|
160
268
|
//# sourceMappingURL=robots-txt.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robots-txt.js","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,YAAY;IAChB,IAAI,EAAE,YAAY;IAClB,WAAW,EAAE,+CAA+C;IAC5D,MAAM,EAAE,EAAE;CACX,CAAC;AASF,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAG,GAAG,CAAC;IAEhB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;IAErD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uBAAuB;YAChC,IAAI,EAAE,8IAA8I;YACpJ,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC;SAC7C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEjE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACrD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACvE,CAAC;IACF,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CACzC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACjF,CAAC;IAEF,IAAI,cAAc,CAAC,MAAM,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,gBAAgB,CAAC,MAAM,yCAAyC,EAAE,CAAC,CAAC;IACtH,CAAC;SAAM,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,IAAI,gBAAgB,CAAC,MAAM,8BAA8B;YAC1F,MAAM,EAAE,YAAY,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAC5C,IAAI,EAAE,sFAAsF;YAC5F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,kBAAkB,CAAC;SACpD,CAAC,CAAC;QACH,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3E,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,2CAA2C;YACpD,MAAM,EAAE,aAAa,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAClD,IAAI,EAAE,kJAAkJ;YACxJ,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,kBAAkB,CAAC;SACpD,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;IAED,mDAAmD;IACnD,IAAI,aAAa,EAAE,UAAU,EAAE,CAAC;QAC9B,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,MAAM,CAC/C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACjF,CAAC;QACF,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,iBAAiB,CAAC,MAAM,wDAAwD;gBAC5F,MAAM,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,IAAI,EAAE,sJAAsJ;gBAC5J,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAqB,CAAC;aACvD,CAAC,CAAC;YACH,KAAK,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CACjH,CAAC;IACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,WAAW,CAAC,MAAM,mCAAmC;YACjE,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACjD,IAAI,EAAE,oIAAoI;YAC1I,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,CAAC;SACtD,CAAC,CAAC;QACH,KAAK,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,0EAA0E;IAC1E,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAC1C,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,GAAG;QACd,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QACvE,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,CAAC,UAAU,CAChB,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,+CAA+C;YAChF,MAAM,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACpD,IAAI,EAAE,4JAA4J;YAClK,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,sBAAsB,CAAC;SACxD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC,CAAC;IAC1E,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4BAA4B;YACrC,IAAI,EAAE,wFAAwF;YAC9F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,iBAAiB,CAAC;SACnD,CAAC,CAAC;QACH,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACrD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACvE,CAAC;IACF,QAAQ,CAAC,IAAI,CAAC;QACZ,MAAM,EAAE,eAAe,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QACtD,OAAO,EAAE,GAAG,eAAe,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,wCAAwC;QACpG,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,EAAE;YAC7B,CAAC,CAAC;gBACE,IAAI,EAAE,mFAAmF;gBACzF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;aAChD;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAe,EAAE,CAAC;IAClC,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAElD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,GAAG,EAAE,CAAC;gBAClB,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;YACD,MAAM,KAAK,GAAa,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAChH,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;aAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC;gBACxB,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC;aAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9C,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC;aAAM,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;YACxB,CAAC;QACH,CAAC;aAAM,IAAI,+BAA+B,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzD,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
1
|
+
{"version":3,"file":"robots-txt.js","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACrF,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,YAAY;IAChB,IAAI,EAAE,YAAY;IAClB,WAAW,EAAE,+CAA+C;IAC5D,MAAM,EAAE,EAAE;CACX,CAAC;AASF,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAG,GAAG,CAAC;IAEhB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;IAErD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uBAAuB;YAChC,IAAI,EAAE,8IAA8I;YACpJ,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC;SAC7C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEjE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACrD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACvE,CAAC;IACF,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CACzC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACjF,CAAC;IAEF,IAAI,cAAc,CAAC,MAAM,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,gBAAgB,CAAC,MAAM,yCAAyC,EAAE,CAAC,CAAC;IACtH,CAAC;SAAM,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,IAAI,gBAAgB,CAAC,MAAM,8BAA8B;YAC1F,MAAM,EAAE,YAAY,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAC5C,IAAI,EAAE,sFAAsF;YAC5F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,kBAAkB,CAAC;SACpD,CAAC,CAAC;QACH,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3E,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,2CAA2C;YACpD,MAAM,EAAE,aAAa,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAClD,IAAI,EAAE,kJAAkJ;YACxJ,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,kBAAkB,CAAC;SACpD,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;IAED,mDAAmD;IACnD,IAAI,aAAa,EAAE,UAAU,EAAE,CAAC;QAC9B,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,MAAM,CAC/C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACjF,CAAC;QACF,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,iBAAiB,CAAC,MAAM,wDAAwD;gBAC5F,MAAM,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,IAAI,EAAE,sJAAsJ;gBAC5J,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAqB,CAAC;aACvD,CAAC,CAAC;YACH,KAAK,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CACjH,CAAC;IACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,WAAW,CAAC,MAAM,mCAAmC;YACjE,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACjD,IAAI,EAAE,oIAAoI;YAC1I,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,CAAC;SACtD,CAAC,CAAC;QACH,KAAK,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,0EAA0E;IAC1E,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAC1C,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,GAAG;QACd,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QACvE,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,CAAC,UAAU,CAChB,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,+CAA+C;YAChF,MAAM,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACpD,IAAI,EAAE,4JAA4J;YAClK,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,sBAAsB,CAAC;SACxD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC,CAAC;IAC1E,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4BAA4B;YACrC,IAAI,EAAE,wFAAwF;YAC9F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,iBAAiB,CAAC;SACnD,CAAC,CAAC;QACH,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,sGAAsG;IACtG,wBAAwB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAEzC,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACrD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACvE,CAAC;IACF,QAAQ,CAAC,IAAI,CAAC;QACZ,MAAM,EAAE,eAAe,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QACtD,OAAO,EAAE,GAAG,eAAe,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,wCAAwC;QACpG,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,EAAE;YAC7B,CAAC,CAAC;gBACE,IAAI,EAAE,mFAAmF;gBACzF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;aAChD;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAe,EAAE,CAAC;IAClC,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAElD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,GAAG,EAAE,CAAC;gBAClB,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;YACD,MAAM,KAAK,GAAa,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAChH,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;aAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC;gBACxB,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC;aAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9C,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC;aAAM,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;YACxB,CAAC;QACH,CAAC;aAAM,IAAI,8CAA8C,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACxE,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAWD;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,MAAM,KAAK,GAAwB,EAAE,CAAC;IACtC,IAAI,UAAU,GAAa,EAAE,CAAC;IAC9B,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAElD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,YAAY,EAAE,CAAC;gBACjB,UAAU,GAAG,EAAE,CAAC;gBAChB,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;YACD,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACnC,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC1D,IAAI,OAAO,EAAE,CAAC;YACZ,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC,GAAG,UAAU,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACpE,SAAS;QACX,CAAC;QAED,IAAI,8CAA8C,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACjE,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAQD,SAAS,gBAAgB,CAAC,GAAW;IACnC,MAAM,MAAM,GAAkB,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACxE,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzB,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACvB,SAAS;QACX,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAY,EAAE,QAAmB;IACjE,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAE5C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,8CAA8C;YACvD,IAAI,EACF,0FAA0F;gBAC1F,gEAAgE;gBAChE,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,yCAAyC;YACxE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,yBAAyB,CAAC;SAC3D,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7E,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,qDAAqD;gBAC9D,MAAM,EAAE,mBAAmB,IAAI,CAAC,GAAG,EAAE;gBACrC,IAAI,EAAE,0FAA0F;gBAChG,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,wBAAwB,CAAC;aAC1D,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,4CAA4C,KAAK,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aACnF,CAAC,CAAC;QACL,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,uDAAuD,KAAK,EAAE;gBACvE,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC5B,IAAI,EAAE,yFAAyF;gBAC/F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,wBAAwB,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,kDAAkD,KAAK,EAAE;gBAClE,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC1B,IAAI,EAAE,uCAAuC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,wCAAwC;gBAC/G,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,wBAAwB,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { CheckContext, CheckResult, CheckMeta } from '../types.js';
|
|
2
|
+
export declare const meta: CheckMeta;
|
|
3
|
+
/** Extract `License:` directive values from robots.txt (RSL 1.0 §4.4.1). */
|
|
4
|
+
export declare function parseRobotsLicenseDirectives(text: string): string[];
|
|
5
|
+
export default function check(ctx: CheckContext): Promise<CheckResult>;
|
|
6
|
+
//# sourceMappingURL=rsl.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rsl.d.ts","sourceRoot":"","sources":["../../src/checks/rsl.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAKjF,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;AAUF,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAOnE;AAWD,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA+F3E"}
|