ax-audit 4.2.1 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +39 -4
- package/dist/baseline.js +1 -1
- package/dist/baseline.js.map +1 -1
- package/dist/check-ids.js +1 -1
- package/dist/check-ids.js.map +1 -1
- package/dist/cli.js +1 -1
- package/dist/cli.js.map +1 -1
- package/dist/constants.d.ts +1 -85
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +0 -135
- package/dist/constants.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/license.d.ts +8 -0
- package/dist/license.d.ts.map +1 -0
- package/dist/license.js +18 -0
- package/dist/license.js.map +1 -0
- package/dist/metadata.d.ts +6 -0
- package/dist/metadata.d.ts.map +1 -0
- package/dist/metadata.js +205 -0
- package/dist/metadata.js.map +1 -0
- package/dist/orchestrator.d.ts +1 -0
- package/dist/orchestrator.d.ts.map +1 -1
- package/dist/orchestrator.js +134 -56
- package/dist/orchestrator.js.map +1 -1
- package/dist/scorer.js +2 -2
- package/dist/scorer.js.map +1 -1
- package/dist/types.d.ts +4 -0
- package/dist/types.d.ts.map +1 -1
- package/docs/api.md +14 -3
- package/docs/architecture.md +9 -98
- package/docs/ci.md +19 -0
- package/docs/cli.md +11 -0
- package/docs/faq.md +18 -0
- package/docs/getting-started.md +33 -1
- package/package.json +5 -5
- package/dist/checks/agent-access.d.ts +0 -33
- package/dist/checks/agent-access.d.ts.map +0 -1
- package/dist/checks/agent-access.js +0 -256
- package/dist/checks/agent-access.js.map +0 -1
- package/dist/checks/agent-card.d.ts +0 -37
- package/dist/checks/agent-card.d.ts.map +0 -1
- package/dist/checks/agent-card.js +0 -352
- package/dist/checks/agent-card.js.map +0 -1
- package/dist/checks/agent-operability.d.ts +0 -66
- package/dist/checks/agent-operability.d.ts.map +0 -1
- package/dist/checks/agent-operability.js +0 -383
- package/dist/checks/agent-operability.js.map +0 -1
- package/dist/checks/agent-skills.d.ts +0 -24
- package/dist/checks/agent-skills.d.ts.map +0 -1
- package/dist/checks/agent-skills.js +0 -316
- package/dist/checks/agent-skills.js.map +0 -1
- package/dist/checks/ai-catalog.d.ts +0 -28
- package/dist/checks/ai-catalog.d.ts.map +0 -1
- package/dist/checks/ai-catalog.js +0 -254
- package/dist/checks/ai-catalog.js.map +0 -1
- package/dist/checks/ai-directives.d.ts +0 -57
- package/dist/checks/ai-directives.d.ts.map +0 -1
- package/dist/checks/ai-directives.js +0 -263
- package/dist/checks/ai-directives.js.map +0 -1
- package/dist/checks/api-discovery.d.ts +0 -26
- package/dist/checks/api-discovery.d.ts.map +0 -1
- package/dist/checks/api-discovery.js +0 -432
- package/dist/checks/api-discovery.js.map +0 -1
- package/dist/checks/auth-discovery.d.ts +0 -28
- package/dist/checks/auth-discovery.d.ts.map +0 -1
- package/dist/checks/auth-discovery.js +0 -302
- package/dist/checks/auth-discovery.js.map +0 -1
- package/dist/checks/commerce-discovery.d.ts +0 -40
- package/dist/checks/commerce-discovery.d.ts.map +0 -1
- package/dist/checks/commerce-discovery.js +0 -295
- package/dist/checks/commerce-discovery.js.map +0 -1
- package/dist/checks/content-negotiation.d.ts +0 -4
- package/dist/checks/content-negotiation.d.ts.map +0 -1
- package/dist/checks/content-negotiation.js +0 -253
- package/dist/checks/content-negotiation.js.map +0 -1
- package/dist/checks/crawl-efficiency.d.ts +0 -16
- package/dist/checks/crawl-efficiency.d.ts.map +0 -1
- package/dist/checks/crawl-efficiency.js +0 -186
- package/dist/checks/crawl-efficiency.js.map +0 -1
- package/dist/checks/frontmatter.d.ts +0 -34
- package/dist/checks/frontmatter.d.ts.map +0 -1
- package/dist/checks/frontmatter.js +0 -100
- package/dist/checks/frontmatter.js.map +0 -1
- package/dist/checks/html-rendering.d.ts +0 -21
- package/dist/checks/html-rendering.d.ts.map +0 -1
- package/dist/checks/html-rendering.js +0 -221
- package/dist/checks/html-rendering.js.map +0 -1
- package/dist/checks/html-utils.d.ts +0 -66
- package/dist/checks/html-utils.d.ts.map +0 -1
- package/dist/checks/html-utils.js +0 -139
- package/dist/checks/html-utils.js.map +0 -1
- package/dist/checks/http-headers.d.ts +0 -11
- package/dist/checks/http-headers.d.ts.map +0 -1
- package/dist/checks/http-headers.js +0 -249
- package/dist/checks/http-headers.js.map +0 -1
- package/dist/checks/http-hygiene.d.ts +0 -26
- package/dist/checks/http-hygiene.d.ts.map +0 -1
- package/dist/checks/http-hygiene.js +0 -257
- package/dist/checks/http-hygiene.js.map +0 -1
- package/dist/checks/index.d.ts +0 -3
- package/dist/checks/index.d.ts.map +0 -1
- package/dist/checks/index.js +0 -55
- package/dist/checks/index.js.map +0 -1
- package/dist/checks/llms-txt.d.ts +0 -19
- package/dist/checks/llms-txt.d.ts.map +0 -1
- package/dist/checks/llms-txt.js +0 -281
- package/dist/checks/llms-txt.js.map +0 -1
- package/dist/checks/mcp-discovery.d.ts +0 -30
- package/dist/checks/mcp-discovery.d.ts.map +0 -1
- package/dist/checks/mcp-discovery.js +0 -523
- package/dist/checks/mcp-discovery.js.map +0 -1
- package/dist/checks/meta-tags.d.ts +0 -17
- package/dist/checks/meta-tags.d.ts.map +0 -1
- package/dist/checks/meta-tags.js +0 -188
- package/dist/checks/meta-tags.js.map +0 -1
- package/dist/checks/robots-parser.d.ts +0 -110
- package/dist/checks/robots-parser.d.ts.map +0 -1
- package/dist/checks/robots-parser.js +0 -277
- package/dist/checks/robots-parser.js.map +0 -1
- package/dist/checks/robots-txt.d.ts +0 -6
- package/dist/checks/robots-txt.d.ts.map +0 -1
- package/dist/checks/robots-txt.js +0 -367
- package/dist/checks/robots-txt.js.map +0 -1
- package/dist/checks/rsl.d.ts +0 -4
- package/dist/checks/rsl.d.ts.map +0 -1
- package/dist/checks/rsl.js +0 -242
- package/dist/checks/rsl.js.map +0 -1
- package/dist/checks/security-txt.d.ts +0 -4
- package/dist/checks/security-txt.d.ts.map +0 -1
- package/dist/checks/security-txt.js +0 -81
- package/dist/checks/security-txt.js.map +0 -1
- package/dist/checks/seo-basics.d.ts +0 -13
- package/dist/checks/seo-basics.d.ts.map +0 -1
- package/dist/checks/seo-basics.js +0 -221
- package/dist/checks/seo-basics.js.map +0 -1
- package/dist/checks/sitemap.d.ts +0 -12
- package/dist/checks/sitemap.d.ts.map +0 -1
- package/dist/checks/sitemap.js +0 -240
- package/dist/checks/sitemap.js.map +0 -1
- package/dist/checks/structured-data.d.ts +0 -4
- package/dist/checks/structured-data.d.ts.map +0 -1
- package/dist/checks/structured-data.js +0 -408
- package/dist/checks/structured-data.js.map +0 -1
- package/dist/checks/structured-fields.d.ts +0 -46
- package/dist/checks/structured-fields.d.ts.map +0 -1
- package/dist/checks/structured-fields.js +0 -112
- package/dist/checks/structured-fields.js.map +0 -1
- package/dist/checks/surface.d.ts +0 -59
- package/dist/checks/surface.d.ts.map +0 -1
- package/dist/checks/surface.js +0 -106
- package/dist/checks/surface.js.map +0 -1
- package/dist/checks/tls-https.d.ts +0 -13
- package/dist/checks/tls-https.d.ts.map +0 -1
- package/dist/checks/tls-https.js +0 -163
- package/dist/checks/tls-https.js.map +0 -1
- package/dist/checks/usage-policy.d.ts +0 -53
- package/dist/checks/usage-policy.d.ts.map +0 -1
- package/dist/checks/usage-policy.js +0 -339
- package/dist/checks/usage-policy.js.map +0 -1
- package/dist/checks/utils.d.ts +0 -40
- package/dist/checks/utils.d.ts.map +0 -1
- package/dist/checks/utils.js +0 -74
- package/dist/checks/utils.js.map +0 -1
- package/dist/checks/waf.d.ts +0 -75
- package/dist/checks/waf.d.ts.map +0 -1
- package/dist/checks/waf.js +0 -203
- package/dist/checks/waf.js.map +0 -1
- package/dist/checks/webmcp.d.ts +0 -55
- package/dist/checks/webmcp.d.ts.map +0 -1
- package/dist/checks/webmcp.js +0 -209
- package/dist/checks/webmcp.js.map +0 -1
- package/dist/checks/well-known.d.ts +0 -38
- package/dist/checks/well-known.d.ts.map +0 -1
- package/dist/checks/well-known.js +0 -202
- package/dist/checks/well-known.js.map +0 -1
- package/dist/fetcher.d.ts +0 -15
- package/dist/fetcher.d.ts.map +0 -1
- package/dist/fetcher.js +0 -127
- package/dist/fetcher.js.map +0 -1
- package/dist/guide-urls.d.ts +0 -2
- package/dist/guide-urls.d.ts.map +0 -1
- package/dist/guide-urls.js +0 -5
- package/dist/guide-urls.js.map +0 -1
- package/docs/roadmap.md +0 -367
|
@@ -1,339 +0,0 @@
|
|
|
1
|
-
import { RSL_MIME } from '../constants.js';
|
|
2
|
-
import { guideUrl } from '../guide-urls.js';
|
|
3
|
-
import { buildResult } from './utils.js';
|
|
4
|
-
import { findLinkTags, getAttribute, getMetaContent } from './html-utils.js';
|
|
5
|
-
import { parseLinkHeader } from './http-headers.js';
|
|
6
|
-
import { parseDictionary } from './structured-fields.js';
|
|
7
|
-
import { AIPREF_TOKENS, AIPREF_VALUES } from '../constants.js';
|
|
8
|
-
import { collectLicenses, parseContentSignalValue, parseContentUsageValue, parseRobotsTxt } from './robots-parser.js';
|
|
9
|
-
/**
|
|
10
|
-
* "usage-policy" — do your machine-readable usage signals agree with each other?
|
|
11
|
-
*
|
|
12
|
-
* A site can now state its AI usage terms in at least five places, written in
|
|
13
|
-
* vocabularies that mirror each other without matching:
|
|
14
|
-
*
|
|
15
|
-
* | Mechanism | Training | Grounding / RAG | Search |
|
|
16
|
-
* | --- | --- | --- | --- |
|
|
17
|
-
* | Content Signals (robots.txt) | `ai-train=yes\|no` | `ai-input=yes\|no` | `search=yes\|no` |
|
|
18
|
-
* | IETF AIPREF (robots.txt or header) | `train-ai=y\|n` | *(no category yet)* | `search=y\|n` |
|
|
19
|
-
* | RSL licence | `ai-train` | `ai-input` | `ai-index`, `search` |
|
|
20
|
-
* | TDMRep | `tdm-reservation: 0\|1` | — | — |
|
|
21
|
-
* | robots meta | `noai` | — | — |
|
|
22
|
-
*
|
|
23
|
-
* Nobody maintains five documents by hand without them drifting. This check
|
|
24
|
-
* normalises every signal onto three questions — may you train on it, may you
|
|
25
|
-
* ground an answer in it, may you index it — and reports where the answers
|
|
26
|
-
* disagree. A site whose robots.txt says `ai-train=no` while its RSL licence
|
|
27
|
-
* permits `ai-train` has published two contradictory legal positions, and which
|
|
28
|
-
* one a crawler honors is then a matter of which file it happened to read.
|
|
29
|
-
*
|
|
30
|
-
* On enforcement, the check is blunt: only robots.txt access rules are
|
|
31
|
-
* documented as honored by Google, OpenAI, Anthropic and Microsoft. Content
|
|
32
|
-
* Signals, AIPREF, RSL and TDMRep are declarations. Their weight is legal — the
|
|
33
|
-
* EU AI Act's opt-out provisions and the GPAI Code of Practice reference several
|
|
34
|
-
* of them — rather than technical, and reports say so rather than implying a
|
|
35
|
-
* crawler will obey.
|
|
36
|
-
*/
|
|
37
|
-
export const meta = {
|
|
38
|
-
id: 'usage-policy',
|
|
39
|
-
name: 'Usage Policy',
|
|
40
|
-
description: 'Checks machine-readable AI usage signals for coverage and internal consistency',
|
|
41
|
-
category: 'policy',
|
|
42
|
-
};
|
|
43
|
-
const DIMENSION_LABEL = {
|
|
44
|
-
train: 'training on your content',
|
|
45
|
-
ground: 'grounding an answer in your content',
|
|
46
|
-
index: 'indexing your content for search',
|
|
47
|
-
};
|
|
48
|
-
function yesNo(value) {
|
|
49
|
-
if (value === 'yes' || value === 'y')
|
|
50
|
-
return 'allow';
|
|
51
|
-
if (value === 'no' || value === 'n')
|
|
52
|
-
return 'deny';
|
|
53
|
-
return 'unstated';
|
|
54
|
-
}
|
|
55
|
-
/** Read a Content-Signal value into a declaration. */
|
|
56
|
-
export function fromContentSignal(raw, source) {
|
|
57
|
-
const pairs = new Map(parseContentSignalValue(raw).valid.map((p) => {
|
|
58
|
-
const [k, v] = p.split('=');
|
|
59
|
-
return [k, v];
|
|
60
|
-
}));
|
|
61
|
-
return {
|
|
62
|
-
source,
|
|
63
|
-
enforceable: false,
|
|
64
|
-
train: yesNo(pairs.get('ai-train')),
|
|
65
|
-
ground: yesNo(pairs.get('ai-input')),
|
|
66
|
-
index: yesNo(pairs.get('search')),
|
|
67
|
-
};
|
|
68
|
-
}
|
|
69
|
-
/** Read an AIPREF Content-Usage value into a declaration. AIPREF has no grounding category yet. */
|
|
70
|
-
export function fromContentUsage(raw, source) {
|
|
71
|
-
const pairs = new Map(parseContentUsageValue(raw).valid.map((p) => {
|
|
72
|
-
const [k, v] = p.split('=');
|
|
73
|
-
return [k, v];
|
|
74
|
-
}));
|
|
75
|
-
return {
|
|
76
|
-
source,
|
|
77
|
-
enforceable: false,
|
|
78
|
-
train: yesNo(pairs.get('train-ai')),
|
|
79
|
-
ground: 'unstated',
|
|
80
|
-
index: yesNo(pairs.get('search')),
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* Read RSL `permits` / `prohibits` usage tokens into a declaration.
|
|
85
|
-
* `all` and `ai-all` set several dimensions at once.
|
|
86
|
-
*/
|
|
87
|
-
export function fromRsl(document, source) {
|
|
88
|
-
const decl = { source, enforceable: false, train: 'unstated', ground: 'unstated', index: 'unstated' };
|
|
89
|
-
const body = document.replace(/<!--[\s\S]*?-->/g, '');
|
|
90
|
-
for (const m of body.matchAll(/<(permits|prohibits)\b([^>]*)>([^<]*)</gi)) {
|
|
91
|
-
const stance = m[1].toLowerCase() === 'permits' ? 'allow' : 'deny';
|
|
92
|
-
const type = (getAttribute(m[2], 'type') ?? '').toLowerCase();
|
|
93
|
-
if (type !== 'usage')
|
|
94
|
-
continue;
|
|
95
|
-
for (const token of m[3].trim().toLowerCase().split(/\s+/).filter(Boolean)) {
|
|
96
|
-
switch (token) {
|
|
97
|
-
case 'ai-train':
|
|
98
|
-
decl.train = stance;
|
|
99
|
-
break;
|
|
100
|
-
case 'ai-input':
|
|
101
|
-
decl.ground = stance;
|
|
102
|
-
break;
|
|
103
|
-
case 'ai-index':
|
|
104
|
-
case 'search':
|
|
105
|
-
decl.index = stance;
|
|
106
|
-
break;
|
|
107
|
-
case 'ai-all':
|
|
108
|
-
decl.train = stance;
|
|
109
|
-
decl.ground = stance;
|
|
110
|
-
break;
|
|
111
|
-
case 'all':
|
|
112
|
-
decl.train = stance;
|
|
113
|
-
decl.ground = stance;
|
|
114
|
-
decl.index = stance;
|
|
115
|
-
break;
|
|
116
|
-
default:
|
|
117
|
-
break;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
return decl;
|
|
122
|
-
}
|
|
123
|
-
/** Collect every usage declaration a site publishes. */
|
|
124
|
-
async function collect(ctx) {
|
|
125
|
-
const declarations = [];
|
|
126
|
-
const notes = [];
|
|
127
|
-
/* robots.txt: Content-Signal and Content-Usage, per group. */
|
|
128
|
-
const robotsRes = await ctx.fetch(`${ctx.url}/robots.txt`);
|
|
129
|
-
const robots = robotsRes.ok ? parseRobotsTxt(robotsRes.body) : null;
|
|
130
|
-
if (robots !== null) {
|
|
131
|
-
for (const group of robots.groups) {
|
|
132
|
-
const scope = group.userAgents.join(', ');
|
|
133
|
-
for (const raw of group.contentSignals) {
|
|
134
|
-
declarations.push(fromContentSignal(raw, `robots.txt Content-Signal (User-agent: ${scope})`));
|
|
135
|
-
}
|
|
136
|
-
for (const raw of group.contentUsage) {
|
|
137
|
-
declarations.push(fromContentUsage(raw, `robots.txt Content-Usage (User-agent: ${scope})`));
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
/* HTTP headers: the Content-Usage response header, and Cloudflare's content-signal. */
|
|
142
|
-
const headers = ctx.headers ?? {};
|
|
143
|
-
if (headers['content-usage'] !== undefined) {
|
|
144
|
-
declarations.push(fromContentUsage(headers['content-usage'], 'Content-Usage response header'));
|
|
145
|
-
const dict = parseDictionary(headers['content-usage']);
|
|
146
|
-
const bad = dict.members.filter((m) => !AIPREF_TOKENS.includes(m.key) || !AIPREF_VALUES.includes(m.value.toLowerCase()));
|
|
147
|
-
if (bad.length > 0) {
|
|
148
|
-
notes.push({
|
|
149
|
-
status: 'warn',
|
|
150
|
-
message: 'Content-Usage response header uses tokens outside the AIPREF vocabulary',
|
|
151
|
-
detail: bad.map((m) => m.raw).join(', '),
|
|
152
|
-
hint: 'The header takes a Structured Fields dictionary over train-ai and search with y/n values.',
|
|
153
|
-
learnMoreUrl: guideUrl(meta.id, 'header-vocabulary'),
|
|
154
|
-
});
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
if (headers['content-signal'] !== undefined) {
|
|
158
|
-
declarations.push(fromContentSignal(headers['content-signal'], 'content-signal response header'));
|
|
159
|
-
}
|
|
160
|
-
/* RSL licence, discovered the same three ways the rsl check uses. */
|
|
161
|
-
const licenceUrl = findLicenceUrl(ctx, robots === null ? [] : collectLicenses(robots));
|
|
162
|
-
if (licenceUrl !== null) {
|
|
163
|
-
const doc = await ctx.fetch(licenceUrl);
|
|
164
|
-
if (doc.ok && /<rsl[\s>]/i.test(doc.body)) {
|
|
165
|
-
declarations.push(fromRsl(doc.body, `RSL licence (${licenceUrl})`));
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
/* TDMRep: reservation is specifically about text-and-data-mining, i.e. training. */
|
|
169
|
-
const tdm = await readTdmRep(ctx);
|
|
170
|
-
if (tdm !== null)
|
|
171
|
-
declarations.push(tdm);
|
|
172
|
-
/* robots meta noai: a declared preference with no committed honorer. */
|
|
173
|
-
const robotsMeta = (getMetaContent(ctx.html ?? '', 'robots') ?? '').toLowerCase();
|
|
174
|
-
if (/\bnoai\b/.test(robotsMeta)) {
|
|
175
|
-
declarations.push({
|
|
176
|
-
source: '<meta name="robots" content="noai">',
|
|
177
|
-
enforceable: false,
|
|
178
|
-
train: 'deny',
|
|
179
|
-
ground: 'unstated',
|
|
180
|
-
index: 'unstated',
|
|
181
|
-
});
|
|
182
|
-
}
|
|
183
|
-
return { declarations, notes };
|
|
184
|
-
}
|
|
185
|
-
/** Resolve the first RSL licence URL from robots.txt, the Link header, or an HTML link. */
|
|
186
|
-
function findLicenceUrl(ctx, robotsLicences) {
|
|
187
|
-
const candidates = [...robotsLicences];
|
|
188
|
-
for (const entry of parseLinkHeader(ctx.headers?.['link'] ?? '')) {
|
|
189
|
-
if (entry.params['rel'] === 'license' && (entry.params['type'] ?? '').includes(RSL_MIME)) {
|
|
190
|
-
candidates.push(entry.url);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
for (const tag of findLinkTags(ctx.html ?? '', 'license')) {
|
|
194
|
-
const type = (getAttribute(tag, 'type') ?? '').toLowerCase();
|
|
195
|
-
const href = getAttribute(tag, 'href');
|
|
196
|
-
if (type.includes(RSL_MIME) && href !== null)
|
|
197
|
-
candidates.push(href);
|
|
198
|
-
}
|
|
199
|
-
if (candidates.length === 0)
|
|
200
|
-
return null;
|
|
201
|
-
try {
|
|
202
|
-
return new URL(candidates[0], `${ctx.url}/`).toString();
|
|
203
|
-
}
|
|
204
|
-
catch {
|
|
205
|
-
return null;
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
/** Read a TDM reservation from headers, meta tags, or the well-known file (precedence: meta > header > file). */
|
|
209
|
-
async function readTdmRep(ctx) {
|
|
210
|
-
const metaValue = getMetaContent(ctx.html ?? '', 'tdm-reservation');
|
|
211
|
-
if (metaValue !== null) {
|
|
212
|
-
return {
|
|
213
|
-
source: '<meta name="tdm-reservation">',
|
|
214
|
-
enforceable: false,
|
|
215
|
-
train: metaValue.trim() === '1' ? 'deny' : 'allow',
|
|
216
|
-
ground: 'unstated',
|
|
217
|
-
index: 'unstated',
|
|
218
|
-
};
|
|
219
|
-
}
|
|
220
|
-
const header = ctx.headers?.['tdm-reservation'];
|
|
221
|
-
if (header !== undefined) {
|
|
222
|
-
return {
|
|
223
|
-
source: 'tdm-reservation response header',
|
|
224
|
-
enforceable: false,
|
|
225
|
-
train: header.trim() === '1' ? 'deny' : 'allow',
|
|
226
|
-
ground: 'unstated',
|
|
227
|
-
index: 'unstated',
|
|
228
|
-
};
|
|
229
|
-
}
|
|
230
|
-
const res = await ctx.fetch(`${ctx.url}/.well-known/tdmrep.json`);
|
|
231
|
-
if (!res.ok)
|
|
232
|
-
return null;
|
|
233
|
-
try {
|
|
234
|
-
const data = JSON.parse(res.body);
|
|
235
|
-
if (!Array.isArray(data) || data.length === 0)
|
|
236
|
-
return null;
|
|
237
|
-
// The entry covering the site root governs the audited page.
|
|
238
|
-
const root = data.find((e) => e?.location === '/' || e?.location === '') ?? data[0];
|
|
239
|
-
const reserved = Number(root?.['tdm-reservation']);
|
|
240
|
-
if (reserved !== 0 && reserved !== 1)
|
|
241
|
-
return null;
|
|
242
|
-
return {
|
|
243
|
-
source: '/.well-known/tdmrep.json',
|
|
244
|
-
enforceable: false,
|
|
245
|
-
train: reserved === 1 ? 'deny' : 'allow',
|
|
246
|
-
ground: 'unstated',
|
|
247
|
-
index: 'unstated',
|
|
248
|
-
};
|
|
249
|
-
}
|
|
250
|
-
catch {
|
|
251
|
-
return null;
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
export default async function check(ctx) {
|
|
255
|
-
const start = performance.now();
|
|
256
|
-
const findings = [];
|
|
257
|
-
let score = 100;
|
|
258
|
-
const { declarations, notes } = await collect(ctx);
|
|
259
|
-
findings.push(...notes);
|
|
260
|
-
if (declarations.length === 0) {
|
|
261
|
-
findings.push({
|
|
262
|
-
status: 'warn',
|
|
263
|
-
message: 'No machine-readable usage policy declared',
|
|
264
|
-
detail: 'Checked robots.txt Content-Signal and Content-Usage, the Content-Usage and content-signal response headers, ' +
|
|
265
|
-
'an RSL licence, TDMRep, and the noai meta directive.',
|
|
266
|
-
hint: 'State your terms where they can be read without a lawyer. The lowest-effort option is a Content-Signal line ' +
|
|
267
|
-
'in robots.txt: Content-Signal: search=yes, ai-input=yes, ai-train=no. Absence is neutral, not permission — ' +
|
|
268
|
-
'but it also gives you nothing to point at.',
|
|
269
|
-
learnMoreUrl: guideUrl(meta.id, 'no-policy'),
|
|
270
|
-
});
|
|
271
|
-
return buildResult(meta, 40, findings, start);
|
|
272
|
-
}
|
|
273
|
-
findings.push({
|
|
274
|
-
status: 'pass',
|
|
275
|
-
message: `${declarations.length} usage declaration(s) found`,
|
|
276
|
-
detail: declarations.map((d) => d.source).join('\n'),
|
|
277
|
-
});
|
|
278
|
-
score = reportConsistency(declarations, findings, score);
|
|
279
|
-
reportCoverage(declarations, findings);
|
|
280
|
-
findings.push({
|
|
281
|
-
status: 'pass',
|
|
282
|
-
message: 'Enforcement note: only robots.txt access rules are documented as honored by major AI operators',
|
|
283
|
-
detail: 'Content Signals, AIPREF, RSL and TDMRep are declarations. Their weight is legal — the EU AI Act opt-out ' +
|
|
284
|
-
'provisions and the GPAI Code of Practice reference several of them — rather than technical. Pair any of them ' +
|
|
285
|
-
'with robots.txt rules if the intent is to be enforced.',
|
|
286
|
-
});
|
|
287
|
-
return buildResult(meta, score, findings, start);
|
|
288
|
-
}
|
|
289
|
-
/**
|
|
290
|
-
* Report where two declarations answer the same question differently. This is
|
|
291
|
-
* the finding this check exists for: contradictory published positions mean the
|
|
292
|
-
* terms a crawler honors depend on which file it happened to read.
|
|
293
|
-
*/
|
|
294
|
-
function reportConsistency(declarations, findings, score) {
|
|
295
|
-
let conflicts = 0;
|
|
296
|
-
for (const dimension of ['train', 'ground', 'index']) {
|
|
297
|
-
const allow = declarations.filter((d) => d[dimension] === 'allow');
|
|
298
|
-
const deny = declarations.filter((d) => d[dimension] === 'deny');
|
|
299
|
-
if (allow.length > 0 && deny.length > 0) {
|
|
300
|
-
conflicts++;
|
|
301
|
-
findings.push({
|
|
302
|
-
status: 'fail',
|
|
303
|
-
message: `Your signals disagree about ${DIMENSION_LABEL[dimension]}`,
|
|
304
|
-
detail: [
|
|
305
|
-
`Permitted by: ${allow.map((d) => d.source).join('; ')}`,
|
|
306
|
-
`Denied by: ${deny.map((d) => d.source).join('; ')}`,
|
|
307
|
-
].join('\n'),
|
|
308
|
-
hint: 'Two of your published documents state opposite terms, so which one applies depends on which file a crawler ' +
|
|
309
|
-
'reads. Pick the position you mean and make every mechanism say it.',
|
|
310
|
-
learnMoreUrl: guideUrl(meta.id, 'contradiction'),
|
|
311
|
-
});
|
|
312
|
-
continue;
|
|
313
|
-
}
|
|
314
|
-
if (allow.length + deny.length > 0) {
|
|
315
|
-
const stance = allow.length > 0 ? 'permitted' : 'denied';
|
|
316
|
-
findings.push({
|
|
317
|
-
status: 'pass',
|
|
318
|
-
message: `Consistent across ${allow.length + deny.length} declaration(s): ${DIMENSION_LABEL[dimension]} is ${stance}`,
|
|
319
|
-
});
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
return score - conflicts * 25;
|
|
323
|
-
}
|
|
324
|
-
/** Note the dimensions no declaration answers, and the vocabularies that cannot answer them. */
|
|
325
|
-
function reportCoverage(declarations, findings) {
|
|
326
|
-
const unstated = ['train', 'ground', 'index'].filter((d) => declarations.every((decl) => decl[d] === 'unstated'));
|
|
327
|
-
if (unstated.length === 0)
|
|
328
|
-
return;
|
|
329
|
-
findings.push({
|
|
330
|
-
status: 'warn',
|
|
331
|
-
message: `No declaration covers ${unstated.map((d) => DIMENSION_LABEL[d]).join(' or ')}`,
|
|
332
|
-
detail: unstated.includes('ground') && declarations.every((d) => !d.source.includes('Content-Signal'))
|
|
333
|
-
? 'Grounding is the dimension that decides whether an assistant may quote you in an answer. AIPREF has no category for it yet; Content Signals ai-input and RSL ai-input do.'
|
|
334
|
-
: undefined,
|
|
335
|
-
hint: 'An unstated dimension is "unknown", never "allowed". State it explicitly if you have a position on it.',
|
|
336
|
-
learnMoreUrl: guideUrl(meta.id, 'coverage'),
|
|
337
|
-
});
|
|
338
|
-
}
|
|
339
|
-
//# sourceMappingURL=usage-policy.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"usage-policy.js","sourceRoot":"","sources":["../../src/checks/usage-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,sBAAsB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEtH;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,cAAc;IAClB,IAAI,EAAE,cAAc;IACpB,WAAW,EAAE,gFAAgF;IAC7F,QAAQ,EAAE,QAAQ;CACnB,CAAC;AAiBF,MAAM,eAAe,GAA8B;IACjD,KAAK,EAAE,0BAA0B;IACjC,MAAM,EAAE,qCAAqC;IAC7C,KAAK,EAAE,kCAAkC;CAC1C,CAAC;AAEF,SAAS,KAAK,CAAC,KAAyB;IACtC,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,GAAG;QAAE,OAAO,OAAO,CAAC;IACrD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,GAAG;QAAE,OAAO,MAAM,CAAC;IACnD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,sDAAsD;AACtD,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,MAAc;IAC3D,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,uBAAuB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3C,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,CAAC,CAAC,EAAE,CAAC,CAAU,CAAC;IACzB,CAAC,CAAC,CACH,CAAC;IACF,OAAO;QACL,MAAM;QACN,WAAW,EAAE,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACnC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACpC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;KAClC,CAAC;AACJ,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,MAAc;IAC1D,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,sBAAsB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1C,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,CAAC,CAAC,EAAE,CAAC,CAAU,CAAC;IACzB,CAAC,CAAC,CACH,CAAC;IACF,OAAO;QACL,MAAM;QACN,WAAW,EAAE,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACnC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;KAClC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,QAAgB,EAAE,MAAc;IACtD,MAAM,IAAI,GAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IACnH,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,0CAA0C,CAAC,EAAE,CAAC;QAC1E,MAAM,MAAM,GAAW,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QAC3E,MAAM,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAC9D,IAAI,IAAI,KAAK,OAAO;YAAE,SAAS;QAE/B,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3E,QAAQ,KAAK,EAAE,CAAC;gBACd,KAAK,UAAU;oBACb,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;oBACpB,MAAM;gBACR,KAAK,UAAU;oBACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;oBACrB,MAAM;gBACR,KAAK,UAAU,CAAC;gBAChB,KAAK,QAAQ;oBACX,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;oBACpB,MAAM;gBACR,KAAK,QAAQ;oBACX,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;oBACpB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;oBACrB,MAAM;gBACR,KAAK,KAAK;oBACR,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;oBACpB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;oBACrB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;oBACpB,MAAM;gBACR;oBACE,MAAM;YACV,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,wDAAwD;AACxD,KAAK,UAAU,OAAO,CAAC,GAAiB;IACtC,MAAM,YAAY,GAAkB,EAAE,CAAC;IACvC,MAAM,KAAK,GAAc,EAAE,CAAC;IAE5B,8DAA8D;IAC9D,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEpE,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1C,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;gBACvC,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,0CAA0C,KAAK,GAAG,CAAC,CAAC,CAAC;YAChG,CAAC;YACD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;gBACrC,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,yCAAyC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC9F,CAAC;QACH,CAAC;IACH,CAAC;IAED,uFAAuF;IACvF,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;IAClC,IAAI,OAAO,CAAC,eAAe,CAAC,KAAK,SAAS,EAAE,CAAC;QAC3C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,+BAA+B,CAAC,CAAC,CAAC;QAC/F,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAC7B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CACxF,CAAC;QACF,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,yEAAyE;gBAClF,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxC,IAAI,EAAE,2FAA2F;gBACjG,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,mBAAmB,CAAC;aACrD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,gBAAgB,CAAC,KAAK,SAAS,EAAE,CAAC;QAC5C,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,gCAAgC,CAAC,CAAC,CAAC;IACpG,CAAC;IAED,qEAAqE;IACrE,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;IACvF,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,GAAG,CAAC,EAAE,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,gBAAgB,UAAU,GAAG,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,oFAAoF;IACpF,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,GAAG,KAAK,IAAI;QAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEzC,wEAAwE;IACxE,MAAM,UAAU,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAClF,IAAI,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAChC,YAAY,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,qCAAqC;YAC7C,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,UAAU;SAClB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AACjC,CAAC;AAED,2FAA2F;AAC3F,SAAS,cAAc,CAAC,GAAiB,EAAE,cAAwB;IACjE,MAAM,UAAU,GAAa,CAAC,GAAG,cAAc,CAAC,CAAC;IAEjD,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACjE,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzF,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,KAAK,IAAI;YAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACzC,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,iHAAiH;AACjH,KAAK,UAAU,UAAU,CAAC,GAAiB;IACzC,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,iBAAiB,CAAC,CAAC;IACpE,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;QACvB,OAAO;YACL,MAAM,EAAE,+BAA+B;YACvC,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;YAClD,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,UAAU;SAClB,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,iBAAiB,CAAC,CAAC;IAChD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,iCAAiC;YACzC,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;YAC/C,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,UAAU;SAClB,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,0BAA0B,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC3D,6DAA6D;QAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK,GAAG,IAAI,CAAC,EAAE,QAAQ,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;QACpF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC;QACnD,IAAI,QAAQ,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAClD,OAAO;YACL,MAAM,EAAE,0BAA0B;YAClC,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;YACxC,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,UAAU;SAClB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAG,GAAG,CAAC;IAEhB,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;IACnD,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;IAExB,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,2CAA2C;YACpD,MAAM,EACJ,8GAA8G;gBAC9G,sDAAsD;YACxD,IAAI,EACF,8GAA8G;gBAC9G,6GAA6G;gBAC7G,4CAA4C;YAC9C,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC;SAC7C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAChD,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC;QACZ,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,GAAG,YAAY,CAAC,MAAM,6BAA6B;QAC5D,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;KACrD,CAAC,CAAC;IAEH,KAAK,GAAG,iBAAiB,CAAC,YAAY,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACzD,cAAc,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IAEvC,QAAQ,CAAC,IAAI,CAAC;QACZ,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,gGAAgG;QACzG,MAAM,EACJ,0GAA0G;YAC1G,+GAA+G;YAC/G,wDAAwD;KAC3D,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,YAA2B,EAAE,QAAmB,EAAE,KAAa;IACxF,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,SAAS,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAgB,EAAE,CAAC;QACpE,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,OAAO,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,MAAM,CAAC,CAAC;QAEjE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxC,SAAS,EAAE,CAAC;YACZ,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,+BAA+B,eAAe,CAAC,SAAS,CAAC,EAAE;gBACpE,MAAM,EAAE;oBACN,iBAAiB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;oBACxD,cAAc,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;iBACrD,CAAC,IAAI,CAAC,IAAI,CAAC;gBACZ,IAAI,EACF,6GAA6G;oBAC7G,oEAAoE;gBACtE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,eAAe,CAAC;aACjD,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;YACzD,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,qBAAqB,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,oBAAoB,eAAe,CAAC,SAAS,CAAC,OAAO,MAAM,EAAE;aACtH,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,KAAK,GAAG,SAAS,GAAG,EAAE,CAAC;AAChC,CAAC;AAED,gGAAgG;AAChG,SAAS,cAAc,CAAC,YAA2B,EAAE,QAAmB;IACtE,MAAM,QAAQ,GAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1E,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CACrD,CAAC;IACF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAElC,QAAQ,CAAC,IAAI,CAAC;QACZ,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,yBAAyB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;QACxF,MAAM,EACJ,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;YAC5F,CAAC,CAAC,2KAA2K;YAC7K,CAAC,CAAC,SAAS;QACf,IAAI,EAAE,wGAAwG;QAC9G,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC;KAC5C,CAAC,CAAC;AACL,CAAC"}
|
package/dist/checks/utils.d.ts
DELETED
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
import type { CheckMeta, CheckResult, FetchResponse, Finding } from '../types.js';
|
|
2
|
-
export declare function clampScore(score: number): number;
|
|
3
|
-
export declare function buildResult(meta: CheckMeta, score: number, findings: Finding[], start: number, options?: {
|
|
4
|
-
applicable?: boolean;
|
|
5
|
-
}): CheckResult;
|
|
6
|
-
/**
|
|
7
|
-
* Result for a check whose subject the site does not have: no API to describe,
|
|
8
|
-
* no commerce surface, no MCP server. Reported as N/A and excluded from the
|
|
9
|
-
* score rather than counted as a failure.
|
|
10
|
-
*/
|
|
11
|
-
export declare function notApplicable(meta: CheckMeta, findings: Finding[], start: number): CheckResult;
|
|
12
|
-
/**
|
|
13
|
-
* Whether a response body is a web page rather than the machine-readable
|
|
14
|
-
* document that was asked for.
|
|
15
|
-
*
|
|
16
|
-
* Single-page applications answer every unmatched path with their index shell,
|
|
17
|
-
* status 200. Probing `/.well-known/agent-card.json` or `/mcp/server-card` on
|
|
18
|
-
* such a site returns HTML, and a check that trusts the status code reports a
|
|
19
|
-
* *malformed* document where the honest answer is that the document is absent.
|
|
20
|
-
* That mistake is worse than a miss: it sends an operator hunting for a bug in
|
|
21
|
-
* a file they never wrote.
|
|
22
|
-
*
|
|
23
|
-
* Detection is deliberately narrow — a doctype or an `<html>` tag near the top
|
|
24
|
-
* of the body. Anything else is handed to the real parser, which reports its own
|
|
25
|
-
* problems precisely.
|
|
26
|
-
*/
|
|
27
|
-
export declare function isHtmlDocument(body: string): boolean;
|
|
28
|
-
/**
|
|
29
|
-
* Validate the `Content-Type` of a fetched resource against a list of acceptable MIME types.
|
|
30
|
-
*
|
|
31
|
-
* Returns:
|
|
32
|
-
* - `null` when the content type is acceptable (no finding to add)
|
|
33
|
-
* - a `Finding` describing the mismatch otherwise (caller decides whether to apply a score penalty)
|
|
34
|
-
*/
|
|
35
|
-
export declare function checkContentType(res: FetchResponse, expected: string[], context: {
|
|
36
|
-
checkId: string;
|
|
37
|
-
resourceLabel: string;
|
|
38
|
-
anchor: string;
|
|
39
|
-
}): Finding | null;
|
|
40
|
-
//# sourceMappingURL=utils.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/checks/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAElF,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED,wBAAgB,WAAW,CACzB,IAAI,EAAE,SAAS,EACf,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,OAAO,EAAE,EACnB,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IAAE,UAAU,CAAC,EAAE,OAAO,CAAA;CAAO,GACrC,WAAW,CAab;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,CAE9F;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAGpD;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,aAAa,EAClB,QAAQ,EAAE,MAAM,EAAE,EAClB,OAAO,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAClE,OAAO,GAAG,IAAI,CAkBhB"}
|
package/dist/checks/utils.js
DELETED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
import { CHECK_CATEGORIES } from '../constants.js';
|
|
2
|
-
import { guideUrl } from '../guide-urls.js';
|
|
3
|
-
export function clampScore(score) {
|
|
4
|
-
return Math.max(0, Math.min(100, score));
|
|
5
|
-
}
|
|
6
|
-
export function buildResult(meta, score, findings, start, options = {}) {
|
|
7
|
-
return {
|
|
8
|
-
id: meta.id,
|
|
9
|
-
name: meta.name,
|
|
10
|
-
description: meta.description,
|
|
11
|
-
score: clampScore(score),
|
|
12
|
-
findings,
|
|
13
|
-
duration: Math.round(performance.now() - start),
|
|
14
|
-
// Falls back to the central map so every result carries its category,
|
|
15
|
-
// including in JSON output where no reporter is there to resolve it.
|
|
16
|
-
...((meta.category ?? CHECK_CATEGORIES[meta.id]) ? { category: meta.category ?? CHECK_CATEGORIES[meta.id] } : {}),
|
|
17
|
-
...(options.applicable === false ? { applicable: false } : {}),
|
|
18
|
-
};
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* Result for a check whose subject the site does not have: no API to describe,
|
|
22
|
-
* no commerce surface, no MCP server. Reported as N/A and excluded from the
|
|
23
|
-
* score rather than counted as a failure.
|
|
24
|
-
*/
|
|
25
|
-
export function notApplicable(meta, findings, start) {
|
|
26
|
-
return buildResult(meta, 0, findings, start, { applicable: false });
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Whether a response body is a web page rather than the machine-readable
|
|
30
|
-
* document that was asked for.
|
|
31
|
-
*
|
|
32
|
-
* Single-page applications answer every unmatched path with their index shell,
|
|
33
|
-
* status 200. Probing `/.well-known/agent-card.json` or `/mcp/server-card` on
|
|
34
|
-
* such a site returns HTML, and a check that trusts the status code reports a
|
|
35
|
-
* *malformed* document where the honest answer is that the document is absent.
|
|
36
|
-
* That mistake is worse than a miss: it sends an operator hunting for a bug in
|
|
37
|
-
* a file they never wrote.
|
|
38
|
-
*
|
|
39
|
-
* Detection is deliberately narrow — a doctype or an `<html>` tag near the top
|
|
40
|
-
* of the body. Anything else is handed to the real parser, which reports its own
|
|
41
|
-
* problems precisely.
|
|
42
|
-
*/
|
|
43
|
-
export function isHtmlDocument(body) {
|
|
44
|
-
const head = body.slice(0, 1024).trimStart();
|
|
45
|
-
return /^<!doctype\s+html/i.test(head) || /^<html[\s>]/i.test(head) || /<html[\s>]/i.test(head.slice(0, 512));
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* Validate the `Content-Type` of a fetched resource against a list of acceptable MIME types.
|
|
49
|
-
*
|
|
50
|
-
* Returns:
|
|
51
|
-
* - `null` when the content type is acceptable (no finding to add)
|
|
52
|
-
* - a `Finding` describing the mismatch otherwise (caller decides whether to apply a score penalty)
|
|
53
|
-
*/
|
|
54
|
-
export function checkContentType(res, expected, context) {
|
|
55
|
-
const ct = (res.headers['content-type'] ?? '').toLowerCase();
|
|
56
|
-
if (!ct) {
|
|
57
|
-
return {
|
|
58
|
-
status: 'warn',
|
|
59
|
-
message: `${context.resourceLabel} has no Content-Type header`,
|
|
60
|
-
hint: `Serve ${context.resourceLabel} with one of: ${expected.join(', ')}.`,
|
|
61
|
-
learnMoreUrl: guideUrl(context.checkId, context.anchor),
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
|
-
if (expected.some((mime) => ct.includes(mime)))
|
|
65
|
-
return null;
|
|
66
|
-
return {
|
|
67
|
-
status: 'warn',
|
|
68
|
-
message: `${context.resourceLabel} Content-Type is "${ct.split(';')[0]}"`,
|
|
69
|
-
detail: `Expected one of: ${expected.join(', ')}`,
|
|
70
|
-
hint: `Configure your server to serve ${context.resourceLabel} as ${expected[0]} so AI agents parse it correctly.`,
|
|
71
|
-
learnMoreUrl: guideUrl(context.checkId, context.anchor),
|
|
72
|
-
};
|
|
73
|
-
}
|
|
74
|
-
//# sourceMappingURL=utils.js.map
|
package/dist/checks/utils.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/checks/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAG5C,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,IAAe,EACf,KAAa,EACb,QAAmB,EACnB,KAAa,EACb,UAAoC,EAAE;IAEtC,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC;QACxB,QAAQ;QACR,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAC/C,sEAAsE;QACtE,qEAAqE;QACrE,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/D,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,IAAe,EAAE,QAAmB,EAAE,KAAa;IAC/E,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;AACtE,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;IAC7C,OAAO,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AAChH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,GAAkB,EAClB,QAAkB,EAClB,OAAmE;IAEnE,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7D,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,OAAO;YACL,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,OAAO,CAAC,aAAa,6BAA6B;YAC9D,IAAI,EAAE,SAAS,OAAO,CAAC,aAAa,iBAAiB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YAC3E,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;SACxD,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5D,OAAO;QACL,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,GAAG,OAAO,CAAC,aAAa,qBAAqB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;QACzE,MAAM,EAAE,oBAAoB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACjD,IAAI,EAAE,kCAAkC,OAAO,CAAC,aAAa,OAAO,QAAQ,CAAC,CAAC,CAAC,mCAAmC;QAClH,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;KACxD,CAAC;AACJ,CAAC"}
|
package/dist/checks/waf.d.ts
DELETED
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Classify what actually happened to a request, beyond `res.ok`.
|
|
3
|
-
*
|
|
4
|
-
* An audit that probes a site with a crawler user agent gets back a status code
|
|
5
|
-
* and has to answer one question honestly: *does this site block AI crawlers?*
|
|
6
|
-
* A bare 403 does not answer it. The same 403 can mean:
|
|
7
|
-
*
|
|
8
|
-
* - a deliberate AI-crawler block in the WAF (a real finding);
|
|
9
|
-
* - a JavaScript challenge that any fetch-only client fails, real crawler
|
|
10
|
-
* included (a different, usually accidental, finding);
|
|
11
|
-
* - anti-spoofing: the edge verifies crawlers by IP range or by Web Bot Auth
|
|
12
|
-
* signature, so it rejects *this* probe precisely because the probe is
|
|
13
|
-
* unsigned and comes from the wrong network, while admitting the genuine
|
|
14
|
-
* crawler (not a finding at all — the site is behaving correctly).
|
|
15
|
-
*
|
|
16
|
-
* Reporting all three as "blocks AI crawlers" would be wrong often enough to
|
|
17
|
-
* make the check untrustworthy. So every classification carries the evidence
|
|
18
|
-
* that produced it and an `inconclusive` flag for the cases an unauthenticated
|
|
19
|
-
* probe genuinely cannot settle.
|
|
20
|
-
*
|
|
21
|
-
* Header signatures verified 2026-09-04:
|
|
22
|
-
* - Cloudflare challenge: `cf-mitigated: challenge`, always `text/html`.
|
|
23
|
-
* https://developers.cloudflare.com/cloudflare-challenges/challenge-types/challenge-pages/detect-response/
|
|
24
|
-
* - AWS WAF challenge: HTTP **202** with `x-amzn-waf-action: challenge`.
|
|
25
|
-
* https://docs.aws.amazon.com/waf/latest/developerguide/waf-captcha-and-challenge-actions.html
|
|
26
|
-
* - Vercel challenge: `x-vercel-mitigated: challenge` plus `x-vercel-challenge-token`.
|
|
27
|
-
* - Cloudflare pay-per-crawl: 402 with `crawler-price`.
|
|
28
|
-
* https://developers.cloudflare.com/ai-crawl-control/features/pay-per-crawl/
|
|
29
|
-
* - AWS WAF / x402 monetization: 402 with `payment-signature` or an x402 body.
|
|
30
|
-
* - Web Bot Auth required: 403 with `Accept-Signature`.
|
|
31
|
-
* https://datatracker.ietf.org/doc/html/draft-ietf-webbotauth-httpsig-protocol
|
|
32
|
-
* - RSL Open Licensing Protocol: 401/402 with `WWW-Authenticate: License`.
|
|
33
|
-
*/
|
|
34
|
-
import type { FetchResponse } from '../types.js';
|
|
35
|
-
export type ResponseKind = 'ok'
|
|
36
|
-
/** An interstitial that requires running JavaScript. Fetch-only clients never get past it. */
|
|
37
|
-
| 'challenge'
|
|
38
|
-
/** 402 with a crawler price attached: content is monetised, not blocked. */
|
|
39
|
-
| 'paywall'
|
|
40
|
-
/** 403 asking the client to re-request with an HTTP Message Signature. */
|
|
41
|
-
| 'needs-signature'
|
|
42
|
-
/** 401/402 pointing at a machine-readable licence (RSL Open Licensing Protocol). */
|
|
43
|
-
| 'license-required' | 'rate-limited'
|
|
44
|
-
/** A plain refusal with no negotiation offered. */
|
|
45
|
-
| 'blocked' | 'not-found' | 'server-error' | 'network-error';
|
|
46
|
-
export interface ResponseClass {
|
|
47
|
-
kind: ResponseKind;
|
|
48
|
-
/** Edge or WAF vendor implicated, when the response identifies one. */
|
|
49
|
-
vendor?: string;
|
|
50
|
-
/** One sentence describing the outcome, written for a report. */
|
|
51
|
-
label: string;
|
|
52
|
-
/** Header names, values or body markers that produced this classification. */
|
|
53
|
-
evidence: string[];
|
|
54
|
-
/**
|
|
55
|
-
* True when this response cannot settle whether the site blocks AI crawlers.
|
|
56
|
-
* A challenge and a signature demand both reject an unsigned probe from a
|
|
57
|
-
* non-vendor IP while potentially admitting the genuine crawler.
|
|
58
|
-
*/
|
|
59
|
-
inconclusive: boolean;
|
|
60
|
-
/** Price string from a pay-per-crawl response, e.g. `USD 0.01`. */
|
|
61
|
-
price?: string;
|
|
62
|
-
}
|
|
63
|
-
/** Identify the CDN or edge from response headers, for context in findings. */
|
|
64
|
-
export declare function detectEdgeVendor(headers: Record<string, string>): string | undefined;
|
|
65
|
-
/**
|
|
66
|
-
* Classify a fetch result. Pure and total: any response produces a
|
|
67
|
-
* classification, and unknown shapes fall back to status-based buckets.
|
|
68
|
-
*/
|
|
69
|
-
export declare function classifyResponse(res: FetchResponse): ResponseClass;
|
|
70
|
-
/**
|
|
71
|
-
* Standard caveat for classifications an unauthenticated probe cannot settle.
|
|
72
|
-
* Appended to hints so a report never overstates what it observed.
|
|
73
|
-
*/
|
|
74
|
-
export declare const INCONCLUSIVE_CAVEAT: string;
|
|
75
|
-
//# sourceMappingURL=waf.d.ts.map
|
package/dist/checks/waf.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"waf.d.ts","sourceRoot":"","sources":["../../src/checks/waf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,MAAM,MAAM,YAAY,GACpB,IAAI;AACN,8FAA8F;GAC5F,WAAW;AACb,4EAA4E;GAC1E,SAAS;AACX,0EAA0E;GACxE,iBAAiB;AACnB,oFAAoF;GAClF,kBAAkB,GAClB,cAAc;AAChB,mDAAmD;GACjD,SAAS,GACT,WAAW,GACX,cAAc,GACd,eAAe,CAAC;AAEpB,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,YAAY,CAAC;IACnB,uEAAuE;IACvE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;OAIG;IACH,YAAY,EAAE,OAAO,CAAC;IACtB,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAmBD,+EAA+E;AAC/E,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,GAAG,SAAS,CAQpF;AAMD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,aAAa,GAAG,aAAa,CA6KlE;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAGM,CAAC"}
|