ax-audit 3.1.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +198 -0
- package/LICENSE +1 -1
- package/README.md +89 -231
- package/dist/baseline.d.ts +2 -0
- package/dist/baseline.d.ts.map +1 -1
- package/dist/baseline.js +42 -4
- package/dist/baseline.js.map +1 -1
- package/dist/check-ids.d.ts +19 -0
- package/dist/check-ids.d.ts.map +1 -0
- package/dist/check-ids.js +53 -0
- package/dist/check-ids.js.map +1 -0
- package/dist/checks/agent-access.d.ts +33 -0
- package/dist/checks/agent-access.d.ts.map +1 -0
- package/dist/checks/agent-access.js +256 -0
- package/dist/checks/agent-access.js.map +1 -0
- package/dist/checks/agent-card.d.ts +37 -0
- package/dist/checks/agent-card.d.ts.map +1 -0
- package/dist/checks/agent-card.js +352 -0
- package/dist/checks/agent-card.js.map +1 -0
- package/dist/checks/agent-operability.d.ts +66 -0
- package/dist/checks/agent-operability.d.ts.map +1 -0
- package/dist/checks/agent-operability.js +383 -0
- package/dist/checks/agent-operability.js.map +1 -0
- package/dist/checks/agent-skills.d.ts +24 -0
- package/dist/checks/agent-skills.d.ts.map +1 -0
- package/dist/checks/agent-skills.js +316 -0
- package/dist/checks/agent-skills.js.map +1 -0
- package/dist/checks/ai-catalog.d.ts +28 -0
- package/dist/checks/ai-catalog.d.ts.map +1 -0
- package/dist/checks/ai-catalog.js +254 -0
- package/dist/checks/ai-catalog.js.map +1 -0
- package/dist/checks/ai-directives.d.ts +57 -0
- package/dist/checks/ai-directives.d.ts.map +1 -0
- package/dist/checks/ai-directives.js +263 -0
- package/dist/checks/ai-directives.js.map +1 -0
- package/dist/checks/api-discovery.d.ts +26 -0
- package/dist/checks/api-discovery.d.ts.map +1 -0
- package/dist/checks/api-discovery.js +432 -0
- package/dist/checks/api-discovery.js.map +1 -0
- package/dist/checks/auth-discovery.d.ts +28 -0
- package/dist/checks/auth-discovery.d.ts.map +1 -0
- package/dist/checks/auth-discovery.js +213 -0
- package/dist/checks/auth-discovery.js.map +1 -0
- package/dist/checks/commerce-discovery.d.ts +40 -0
- package/dist/checks/commerce-discovery.d.ts.map +1 -0
- package/dist/checks/commerce-discovery.js +295 -0
- package/dist/checks/commerce-discovery.js.map +1 -0
- package/dist/checks/content-negotiation.d.ts.map +1 -1
- package/dist/checks/content-negotiation.js +135 -20
- package/dist/checks/content-negotiation.js.map +1 -1
- package/dist/checks/crawl-efficiency.d.ts +16 -0
- package/dist/checks/crawl-efficiency.d.ts.map +1 -0
- package/dist/checks/crawl-efficiency.js +186 -0
- package/dist/checks/crawl-efficiency.js.map +1 -0
- package/dist/checks/frontmatter.d.ts +34 -0
- package/dist/checks/frontmatter.d.ts.map +1 -0
- package/dist/checks/frontmatter.js +100 -0
- package/dist/checks/frontmatter.js.map +1 -0
- package/dist/checks/html-rendering.d.ts.map +1 -1
- package/dist/checks/html-rendering.js +0 -1
- package/dist/checks/html-rendering.js.map +1 -1
- package/dist/checks/html-utils.d.ts +10 -0
- package/dist/checks/html-utils.d.ts.map +1 -1
- package/dist/checks/html-utils.js +19 -0
- package/dist/checks/html-utils.js.map +1 -1
- package/dist/checks/http-headers.d.ts.map +1 -1
- package/dist/checks/http-headers.js +82 -10
- package/dist/checks/http-headers.js.map +1 -1
- package/dist/checks/http-hygiene.d.ts +26 -0
- package/dist/checks/http-hygiene.d.ts.map +1 -0
- package/dist/checks/http-hygiene.js +257 -0
- package/dist/checks/http-hygiene.js.map +1 -0
- package/dist/checks/index.d.ts.map +1 -1
- package/dist/checks/index.js +30 -8
- package/dist/checks/index.js.map +1 -1
- package/dist/checks/llms-txt.d.ts +15 -0
- package/dist/checks/llms-txt.d.ts.map +1 -1
- package/dist/checks/llms-txt.js +162 -2
- package/dist/checks/llms-txt.js.map +1 -1
- package/dist/checks/mcp-discovery.d.ts +30 -0
- package/dist/checks/mcp-discovery.d.ts.map +1 -0
- package/dist/checks/mcp-discovery.js +523 -0
- package/dist/checks/mcp-discovery.js.map +1 -0
- package/dist/checks/meta-tags.d.ts.map +1 -1
- package/dist/checks/meta-tags.js +6 -5
- package/dist/checks/meta-tags.js.map +1 -1
- package/dist/checks/robots-parser.d.ts +110 -0
- package/dist/checks/robots-parser.d.ts.map +1 -0
- package/dist/checks/robots-parser.js +277 -0
- package/dist/checks/robots-parser.js.map +1 -0
- package/dist/checks/robots-txt.d.ts +2 -0
- package/dist/checks/robots-txt.d.ts.map +1 -1
- package/dist/checks/robots-txt.js +252 -45
- package/dist/checks/robots-txt.js.map +1 -1
- package/dist/checks/{mcp.d.ts → rsl.d.ts} +1 -1
- package/dist/checks/rsl.d.ts.map +1 -0
- package/dist/checks/rsl.js +242 -0
- package/dist/checks/rsl.js.map +1 -0
- package/dist/checks/security-txt.d.ts.map +1 -1
- package/dist/checks/security-txt.js +0 -1
- package/dist/checks/security-txt.js.map +1 -1
- package/dist/checks/seo-basics.d.ts.map +1 -1
- package/dist/checks/seo-basics.js +0 -1
- package/dist/checks/seo-basics.js.map +1 -1
- package/dist/checks/sitemap.d.ts.map +1 -1
- package/dist/checks/sitemap.js +0 -1
- package/dist/checks/sitemap.js.map +1 -1
- package/dist/checks/structured-data.d.ts.map +1 -1
- package/dist/checks/structured-data.js +215 -4
- package/dist/checks/structured-data.js.map +1 -1
- package/dist/checks/structured-fields.d.ts +46 -0
- package/dist/checks/structured-fields.d.ts.map +1 -0
- package/dist/checks/structured-fields.js +112 -0
- package/dist/checks/structured-fields.js.map +1 -0
- package/dist/checks/surface.d.ts +59 -0
- package/dist/checks/surface.d.ts.map +1 -0
- package/dist/checks/surface.js +106 -0
- package/dist/checks/surface.js.map +1 -0
- package/dist/checks/tls-https.d.ts.map +1 -1
- package/dist/checks/tls-https.js +0 -1
- package/dist/checks/tls-https.js.map +1 -1
- package/dist/checks/usage-policy.d.ts +53 -0
- package/dist/checks/usage-policy.d.ts.map +1 -0
- package/dist/checks/usage-policy.js +339 -0
- package/dist/checks/usage-policy.js.map +1 -0
- package/dist/checks/utils.d.ts +25 -1
- package/dist/checks/utils.d.ts.map +1 -1
- package/dist/checks/utils.js +33 -1
- package/dist/checks/utils.js.map +1 -1
- package/dist/checks/waf.d.ts +75 -0
- package/dist/checks/waf.d.ts.map +1 -0
- package/dist/checks/waf.js +203 -0
- package/dist/checks/waf.js.map +1 -0
- package/dist/checks/webmcp.d.ts +55 -0
- package/dist/checks/webmcp.d.ts.map +1 -0
- package/dist/checks/webmcp.js +209 -0
- package/dist/checks/webmcp.js.map +1 -0
- package/dist/checks/well-known.d.ts +38 -0
- package/dist/checks/well-known.d.ts.map +1 -0
- package/dist/checks/well-known.js +202 -0
- package/dist/checks/well-known.js.map +1 -0
- package/dist/cli.d.ts +14 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +144 -6
- package/dist/cli.js.map +1 -1
- package/dist/constants.d.ts +212 -14
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +625 -59
- package/dist/constants.js.map +1 -1
- package/dist/fetcher.d.ts +5 -1
- package/dist/fetcher.d.ts.map +1 -1
- package/dist/fetcher.js +62 -27
- package/dist/fetcher.js.map +1 -1
- package/dist/guide-urls.js +1 -1
- package/dist/guide-urls.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/orchestrator.d.ts +2 -2
- package/dist/orchestrator.d.ts.map +1 -1
- package/dist/orchestrator.js +18 -7
- package/dist/orchestrator.js.map +1 -1
- package/dist/reporter/html.d.ts +9 -0
- package/dist/reporter/html.d.ts.map +1 -1
- package/dist/reporter/html.js +49 -11
- package/dist/reporter/html.js.map +1 -1
- package/dist/reporter/index.d.ts.map +1 -1
- package/dist/reporter/index.js +7 -0
- package/dist/reporter/index.js.map +1 -1
- package/dist/reporter/markdown.d.ts +8 -0
- package/dist/reporter/markdown.d.ts.map +1 -0
- package/dist/reporter/markdown.js +106 -0
- package/dist/reporter/markdown.js.map +1 -0
- package/dist/reporter/terminal.d.ts.map +1 -1
- package/dist/reporter/terminal.js +36 -1
- package/dist/reporter/terminal.js.map +1 -1
- package/dist/scorer.d.ts +10 -0
- package/dist/scorer.d.ts.map +1 -1
- package/dist/scorer.js +22 -5
- package/dist/scorer.js.map +1 -1
- package/dist/types.d.ts +96 -3
- package/dist/types.d.ts.map +1 -1
- package/docs/api.md +200 -0
- package/docs/architecture.md +104 -0
- package/docs/checks.md +555 -0
- package/docs/ci.md +89 -0
- package/docs/cli.md +103 -0
- package/docs/concepts.md +101 -0
- package/docs/faq.md +89 -0
- package/docs/getting-started.md +108 -0
- package/docs/roadmap.md +367 -0
- package/package.json +14 -5
- package/dist/checks/agent-json.d.ts +0 -14
- package/dist/checks/agent-json.d.ts.map +0 -1
- package/dist/checks/agent-json.js +0 -167
- package/dist/checks/agent-json.js.map +0 -1
- package/dist/checks/mcp.d.ts.map +0 -1
- package/dist/checks/mcp.js +0 -162
- package/dist/checks/mcp.js.map +0 -1
- package/dist/checks/openapi.d.ts +0 -4
- package/dist/checks/openapi.d.ts.map +0 -1
- package/dist/checks/openapi.js +0 -121
- package/dist/checks/openapi.js.map +0 -1
- package/dist/checks/well-known-ai.d.ts +0 -17
- package/dist/checks/well-known-ai.d.ts.map +0 -1
- package/dist/checks/well-known-ai.js +0 -123
- package/dist/checks/well-known-ai.js.map +0 -1
package/dist/baseline.js
CHANGED
|
@@ -1,19 +1,28 @@
|
|
|
1
1
|
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
2
2
|
import { dirname } from 'node:path';
|
|
3
|
+
import { checks as allChecks } from './checks/index.js';
|
|
4
|
+
import { allIdsFor } from './check-ids.js';
|
|
3
5
|
/**
|
|
4
6
|
* Extract a minimal, stable snapshot from an AuditReport suitable for
|
|
5
7
|
* persistence and future comparison.
|
|
6
8
|
*/
|
|
9
|
+
/** Scoring model this build produces. Bumped whenever weights or the scoring rules change. */
|
|
10
|
+
export const BASELINE_SCHEMA_VERSION = 2;
|
|
7
11
|
export function toBaselineData(report) {
|
|
8
12
|
const checks = {};
|
|
13
|
+
const notApplicable = [];
|
|
9
14
|
for (const r of report.results) {
|
|
10
15
|
checks[r.id] = r.score;
|
|
16
|
+
if (r.applicable === false)
|
|
17
|
+
notApplicable.push(r.id);
|
|
11
18
|
}
|
|
12
19
|
return {
|
|
20
|
+
schemaVersion: BASELINE_SCHEMA_VERSION,
|
|
13
21
|
url: report.url,
|
|
14
22
|
timestamp: report.timestamp,
|
|
15
23
|
overallScore: report.overallScore,
|
|
16
24
|
checks,
|
|
25
|
+
...(notApplicable.length > 0 ? { notApplicable } : {}),
|
|
17
26
|
};
|
|
18
27
|
}
|
|
19
28
|
/**
|
|
@@ -58,19 +67,39 @@ export function loadBaseline(path) {
|
|
|
58
67
|
* per-check deltas and overall regression/improvement lists.
|
|
59
68
|
*/
|
|
60
69
|
export function diffBaseline(baseline, report) {
|
|
70
|
+
// A baseline saved before a check was renamed still holds the former id, so
|
|
71
|
+
// look each score up under every id that check answers to. Without this a
|
|
72
|
+
// rename reads as one check removed (a full regression) and one added at
|
|
73
|
+
// zero, which would fire `--fail-on-regression` on an unchanged site.
|
|
74
|
+
const consumedIds = new Set();
|
|
75
|
+
const previousScoreFor = (id) => {
|
|
76
|
+
for (const candidate of idsForCheck(id)) {
|
|
77
|
+
if (Object.hasOwn(baseline.checks, candidate)) {
|
|
78
|
+
consumedIds.add(candidate);
|
|
79
|
+
return baseline.checks[candidate];
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return undefined;
|
|
83
|
+
};
|
|
84
|
+
const wasNotApplicable = new Set(baseline.notApplicable ?? []);
|
|
61
85
|
const checks = report.results.map((r) => {
|
|
62
|
-
const previous =
|
|
86
|
+
const previous = previousScoreFor(r.id) ?? 0;
|
|
87
|
+
const isNotApplicable = r.applicable === false;
|
|
88
|
+
// A check that is N/A on one side has no comparable score, so its delta
|
|
89
|
+
// reflects a change in what the site offers rather than in its quality.
|
|
90
|
+
const applicabilityChanged = isNotApplicable !== wasNotApplicable.has(r.id);
|
|
63
91
|
return {
|
|
64
92
|
id: r.id,
|
|
65
93
|
name: r.name,
|
|
66
94
|
previous,
|
|
67
95
|
current: r.score,
|
|
68
96
|
delta: r.score - previous,
|
|
97
|
+
...(applicabilityChanged ? { applicabilityChanged: true } : {}),
|
|
69
98
|
};
|
|
70
99
|
});
|
|
71
100
|
// Include checks that existed in the baseline but were removed from the current run
|
|
72
101
|
for (const [id, score] of Object.entries(baseline.checks)) {
|
|
73
|
-
if (!checks.some((c) => c.id === id)) {
|
|
102
|
+
if (!consumedIds.has(id) && !checks.some((c) => c.id === id)) {
|
|
74
103
|
checks.push({
|
|
75
104
|
id,
|
|
76
105
|
name: id, // no human-readable name available for removed checks
|
|
@@ -81,7 +110,9 @@ export function diffBaseline(baseline, report) {
|
|
|
81
110
|
}
|
|
82
111
|
}
|
|
83
112
|
const overallDelta = report.overallScore - baseline.overallScore;
|
|
113
|
+
const scoringModelChanged = (baseline.schemaVersion ?? 1) !== BASELINE_SCHEMA_VERSION;
|
|
84
114
|
return {
|
|
115
|
+
...(scoringModelChanged ? { scoringModelChanged: true } : {}),
|
|
85
116
|
url: report.url,
|
|
86
117
|
baselineTimestamp: baseline.timestamp,
|
|
87
118
|
currentTimestamp: report.timestamp,
|
|
@@ -89,11 +120,18 @@ export function diffBaseline(baseline, report) {
|
|
|
89
120
|
overallCurrent: report.overallScore,
|
|
90
121
|
overallDelta,
|
|
91
122
|
checks,
|
|
92
|
-
|
|
93
|
-
|
|
123
|
+
// A regression must be something the site did. A check that changed
|
|
124
|
+
// applicability, or a baseline from a different scoring model, is neither.
|
|
125
|
+
regressions: scoringModelChanged ? [] : checks.filter((c) => c.delta < 0 && !c.applicabilityChanged),
|
|
126
|
+
improvements: checks.filter((c) => c.delta > 0 && !c.applicabilityChanged),
|
|
94
127
|
};
|
|
95
128
|
}
|
|
96
129
|
/* ── Internal helpers ─────────────────────────────────────── */
|
|
130
|
+
/** Current id plus every former id a check answers to, for baseline lookup. */
|
|
131
|
+
function idsForCheck(id) {
|
|
132
|
+
const meta = allChecks.find((c) => c.meta.id === id)?.meta;
|
|
133
|
+
return meta === undefined ? [id] : allIdsFor(meta);
|
|
134
|
+
}
|
|
97
135
|
function isBaselineData(value) {
|
|
98
136
|
if (typeof value !== 'object' || value === null)
|
|
99
137
|
return false;
|
package/dist/baseline.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"baseline.js","sourceRoot":"","sources":["../src/baseline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"baseline.js","sourceRoot":"","sources":["../src/baseline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,IAAI,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG3C;;;GAGG;AACH,8FAA8F;AAC9F,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAEzC,MAAM,UAAU,cAAc,CAAC,MAAmB;IAChD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;QACvB,IAAI,CAAC,CAAC,UAAU,KAAK,KAAK;YAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,OAAO;QACL,aAAa,EAAE,uBAAuB;QACtC,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,MAAM;QACN,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,MAAmB;IAC5D,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACpC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;QACjD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,wFAAwF,IAAI,EAAE,CAAC,CAAC;IAClH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,QAAsB,EAAE,MAAmB;IACtE,4EAA4E;IAC5E,0EAA0E;IAC1E,yEAAyE;IACzE,sEAAsE;IACtE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC,MAAM,gBAAgB,GAAG,CAAC,EAAU,EAAsB,EAAE;QAC1D,KAAK,MAAM,SAAS,IAAI,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACxC,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC;gBAC9C,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC3B,OAAO,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC;IAEF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;IAE/D,MAAM,MAAM,GAAgB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACnD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,eAAe,GAAG,CAAC,CAAC,UAAU,KAAK,KAAK,CAAC;QAC/C,wEAAwE;QACxE,wEAAwE;QACxE,MAAM,oBAAoB,GAAG,eAAe,KAAK,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAE5E,OAAO;YACL,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,QAAQ;YACR,OAAO,EAAE,CAAC,CAAC,KAAK;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,QAAQ;YACzB,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,oFAAoF;IACpF,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE;gBACF,IAAI,EAAE,EAAE,EAAE,sDAAsD;gBAChE,QAAQ,EAAE,KAAK;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,CAAC,KAAK;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC;IACjE,MAAM,mBAAmB,GAAG,CAAC,QAAQ,CAAC,aAAa,IAAI,CAAC,CAAC,KAAK,uBAAuB,CAAC;IAEtF,OAAO;QACL,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,iBAAiB,EAAE,QAAQ,CAAC,SAAS;QACrC,gBAAgB,EAAE,MAAM,CAAC,SAAS;QAClC,eAAe,EAAE,QAAQ,CAAC,YAAY;QACtC,cAAc,EAAE,MAAM,CAAC,YAAY;QACnC,YAAY;QACZ,MAAM;QACN,oEAAoE;QACpE,2EAA2E;QAC3E,WAAW,EAAE,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,oBAAoB,CAAC;QACpG,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,oBAAoB,CAAC;KAC3E,CAAC;AACJ,CAAC;AAED,iEAAiE;AAEjE,+EAA+E;AAC/E,SAAS,WAAW,CAAC,EAAU;IAC7B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC;IAC3D,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,OAAO,CACL,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ;QAC3B,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ;QACjC,OAAO,GAAG,CAAC,YAAY,KAAK,QAAQ;QACpC,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ;QAC9B,GAAG,CAAC,MAAM,KAAK,IAAI;QACnB,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAC3B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { CheckMeta } from './types.js';
|
|
2
|
+
/** Map of alias → current id, built from the registered checks. */
|
|
3
|
+
export declare function buildAliasMap(metas?: CheckMeta[]): Map<string, string>;
|
|
4
|
+
/**
|
|
5
|
+
* Resolve a possibly-former check id to its current one. Unknown ids are
|
|
6
|
+
* returned unchanged so callers can report them as invalid with the name the
|
|
7
|
+
* user actually typed.
|
|
8
|
+
*/
|
|
9
|
+
export declare function resolveCheckId(id: string, metas?: CheckMeta[]): string;
|
|
10
|
+
/** Every id a check answers to, current name first. */
|
|
11
|
+
export declare function allIdsFor(meta: CheckMeta): string[];
|
|
12
|
+
/** Every selectable id across all checks, aliases included. */
|
|
13
|
+
export declare function allSelectableIds(metas?: CheckMeta[]): string[];
|
|
14
|
+
/**
|
|
15
|
+
* Whether a check should run given a `--checks` selection, matching on its
|
|
16
|
+
* current id or any alias.
|
|
17
|
+
*/
|
|
18
|
+
export declare function isSelected(meta: CheckMeta, selection: string[]): boolean;
|
|
19
|
+
//# sourceMappingURL=check-ids.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"check-ids.d.ts","sourceRoot":"","sources":["../src/check-ids.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,mEAAmE;AACnE,wBAAgB,aAAa,CAAC,KAAK,GAAE,SAAS,EAAiC,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAQpG;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,SAAS,EAAE,GAAG,MAAM,CAKtE;AAED,uDAAuD;AACvD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,EAAE,CAEnD;AAED,+DAA+D;AAC/D,wBAAgB,gBAAgB,CAAC,KAAK,GAAE,SAAS,EAAiC,GAAG,MAAM,EAAE,CAE5F;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAGxE"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Check-id resolution across renames.
|
|
3
|
+
*
|
|
4
|
+
* Check ids are a public interface twice over: they appear in `--checks`
|
|
5
|
+
* invocations that live in CI configs, and as keys in saved baseline files. So
|
|
6
|
+
* a rename cannot be a silent break — `--checks mcp` must keep working, and a
|
|
7
|
+
* baseline recorded when the check was called `mcp` must still compare against
|
|
8
|
+
* the check now called `mcp-discovery` rather than reporting one check removed
|
|
9
|
+
* and another added.
|
|
10
|
+
*
|
|
11
|
+
* A check declares its former names in `meta.aliases`; everything here reads
|
|
12
|
+
* from that, so adding an alias is a one-line change on the check itself.
|
|
13
|
+
*/
|
|
14
|
+
import { checks as allChecks } from './checks/index.js';
|
|
15
|
+
/** Map of alias → current id, built from the registered checks. */
|
|
16
|
+
export function buildAliasMap(metas = allChecks.map((c) => c.meta)) {
|
|
17
|
+
const map = new Map();
|
|
18
|
+
for (const meta of metas) {
|
|
19
|
+
for (const alias of meta.aliases ?? []) {
|
|
20
|
+
map.set(alias.toLowerCase(), meta.id);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return map;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Resolve a possibly-former check id to its current one. Unknown ids are
|
|
27
|
+
* returned unchanged so callers can report them as invalid with the name the
|
|
28
|
+
* user actually typed.
|
|
29
|
+
*/
|
|
30
|
+
export function resolveCheckId(id, metas) {
|
|
31
|
+
const current = metas ?? allChecks.map((c) => c.meta);
|
|
32
|
+
const lower = id.toLowerCase();
|
|
33
|
+
if (current.some((m) => m.id.toLowerCase() === lower))
|
|
34
|
+
return current.find((m) => m.id.toLowerCase() === lower).id;
|
|
35
|
+
return buildAliasMap(current).get(lower) ?? id;
|
|
36
|
+
}
|
|
37
|
+
/** Every id a check answers to, current name first. */
|
|
38
|
+
export function allIdsFor(meta) {
|
|
39
|
+
return [meta.id, ...(meta.aliases ?? [])];
|
|
40
|
+
}
|
|
41
|
+
/** Every selectable id across all checks, aliases included. */
|
|
42
|
+
export function allSelectableIds(metas = allChecks.map((c) => c.meta)) {
|
|
43
|
+
return metas.flatMap(allIdsFor);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Whether a check should run given a `--checks` selection, matching on its
|
|
47
|
+
* current id or any alias.
|
|
48
|
+
*/
|
|
49
|
+
export function isSelected(meta, selection) {
|
|
50
|
+
const wanted = new Set(selection.map((s) => s.toLowerCase()));
|
|
51
|
+
return allIdsFor(meta).some((id) => wanted.has(id.toLowerCase()));
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=check-ids.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"check-ids.js","sourceRoot":"","sources":["../src/check-ids.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,EAAE,MAAM,IAAI,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGxD,mEAAmE;AACnE,MAAM,UAAU,aAAa,CAAC,QAAqB,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7E,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,EAAU,EAAE,KAAmB;IAC5D,MAAM,OAAO,GAAG,KAAK,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,KAAK,CAAE,CAAC,EAAE,CAAC;IACpH,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;AACjD,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,SAAS,CAAC,IAAe;IACvC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,gBAAgB,CAAC,QAAqB,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAChF,OAAO,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,IAAe,EAAE,SAAmB;IAC7D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC9D,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACpE,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { CheckContext, CheckResult, CheckMeta } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* "agent-access" — the gap between what robots.txt permits and what the server
|
|
4
|
+
* actually does.
|
|
5
|
+
*
|
|
6
|
+
* The failure this catches is invisible from the inside: robots.txt says
|
|
7
|
+
* `Allow: /` for GPTBot, and the WAF returns 403 to anything whose user agent
|
|
8
|
+
* says GPTBot. The operator sees a permissive robots.txt and assumes the site
|
|
9
|
+
* is reachable. Cloudflare's "Block AI Crawlers" toggle produces exactly this,
|
|
10
|
+
* and so does a hand-written rule that outlived its reason.
|
|
11
|
+
*
|
|
12
|
+
* The check probes the homepage once per core crawler and compares the result
|
|
13
|
+
* against the default-user-agent baseline. Since 3.7 it classifies *how* a
|
|
14
|
+
* request was turned away, because the remedies are completely different: a
|
|
15
|
+
* JavaScript challenge needs a bot-management exception, a hard 403 needs a
|
|
16
|
+
* firewall rule change, a 402 is a deliberate price, and a signature demand is
|
|
17
|
+
* the site working as designed.
|
|
18
|
+
*
|
|
19
|
+
* Honesty constraint: this probe is unsigned and comes from the auditor's own
|
|
20
|
+
* network. An edge that verifies crawlers by IP range or Web Bot Auth signature
|
|
21
|
+
* will reject it while admitting the genuine crawler. Those outcomes are scored
|
|
22
|
+
* as inconclusive and labelled as such, never as "blocks AI crawlers".
|
|
23
|
+
*/
|
|
24
|
+
export declare const meta: CheckMeta;
|
|
25
|
+
/**
|
|
26
|
+
* Build a realistic crawler User-Agent for a given bot token. WAF and
|
|
27
|
+
* bot-management rules match on the token substring, which is what we need
|
|
28
|
+
* to trigger the same code path the real crawler would hit.
|
|
29
|
+
*/
|
|
30
|
+
export declare function crawlerUserAgent(token: string): string;
|
|
31
|
+
export { intentBlocked } from './robots-parser.js';
|
|
32
|
+
export default function check(ctx: CheckContext): Promise<CheckResult>;
|
|
33
|
+
//# sourceMappingURL=agent-access.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-access.d.ts","sourceRoot":"","sources":["../../src/checks/agent-access.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAMjF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;AA+BF;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEtD;AAGD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAoCnD,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA8E3E"}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
import { PROBEABLE_CORE_CRAWLERS, crawlerInfo } from '../constants.js';
|
|
2
|
+
import { guideUrl } from '../guide-urls.js';
|
|
3
|
+
import { buildResult } from './utils.js';
|
|
4
|
+
import { extractVisibleText, findJsonLdBlocks } from './html-utils.js';
|
|
5
|
+
import { parseUserAgents, intentBlocked } from './robots-parser.js';
|
|
6
|
+
import { classifyResponse, INCONCLUSIVE_CAVEAT } from './waf.js';
|
|
7
|
+
/**
|
|
8
|
+
* "agent-access" — the gap between what robots.txt permits and what the server
|
|
9
|
+
* actually does.
|
|
10
|
+
*
|
|
11
|
+
* The failure this catches is invisible from the inside: robots.txt says
|
|
12
|
+
* `Allow: /` for GPTBot, and the WAF returns 403 to anything whose user agent
|
|
13
|
+
* says GPTBot. The operator sees a permissive robots.txt and assumes the site
|
|
14
|
+
* is reachable. Cloudflare's "Block AI Crawlers" toggle produces exactly this,
|
|
15
|
+
* and so does a hand-written rule that outlived its reason.
|
|
16
|
+
*
|
|
17
|
+
* The check probes the homepage once per core crawler and compares the result
|
|
18
|
+
* against the default-user-agent baseline. Since 3.7 it classifies *how* a
|
|
19
|
+
* request was turned away, because the remedies are completely different: a
|
|
20
|
+
* JavaScript challenge needs a bot-management exception, a hard 403 needs a
|
|
21
|
+
* firewall rule change, a 402 is a deliberate price, and a signature demand is
|
|
22
|
+
* the site working as designed.
|
|
23
|
+
*
|
|
24
|
+
* Honesty constraint: this probe is unsigned and comes from the auditor's own
|
|
25
|
+
* network. An edge that verifies crawlers by IP range or Web Bot Auth signature
|
|
26
|
+
* will reject it while admitting the genuine crawler. Those outcomes are scored
|
|
27
|
+
* as inconclusive and labelled as such, never as "blocks AI crawlers".
|
|
28
|
+
*/
|
|
29
|
+
export const meta = {
|
|
30
|
+
id: 'agent-access',
|
|
31
|
+
name: 'Agent Access',
|
|
32
|
+
description: 'Checks that AI crawler user-agents are not blocked or served reduced content (cloaking)',
|
|
33
|
+
category: 'access',
|
|
34
|
+
};
|
|
35
|
+
/** Content below this fraction of the baseline visible text counts as "reduced". */
|
|
36
|
+
const REDUCED_CONTENT_RATIO = 0.5;
|
|
37
|
+
/** Baselines with less visible text than this are too small for meaningful content comparison. */
|
|
38
|
+
const MIN_BASELINE_TEXT = 200;
|
|
39
|
+
/** Credit each outcome contributes to the access score. */
|
|
40
|
+
const OUTCOME_CREDIT = {
|
|
41
|
+
ok: 1,
|
|
42
|
+
'blocked-consistent': 1,
|
|
43
|
+
conditional: 1,
|
|
44
|
+
inconclusive: 0.75,
|
|
45
|
+
reduced: 0.5,
|
|
46
|
+
blocked: 0,
|
|
47
|
+
};
|
|
48
|
+
/**
|
|
49
|
+
* Build a realistic crawler User-Agent for a given bot token. WAF and
|
|
50
|
+
* bot-management rules match on the token substring, which is what we need
|
|
51
|
+
* to trigger the same code path the real crawler would hit.
|
|
52
|
+
*/
|
|
53
|
+
export function crawlerUserAgent(token) {
|
|
54
|
+
return `Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ${token}/1.0)`;
|
|
55
|
+
}
|
|
56
|
+
// `intentBlocked` moved to robots-parser.ts in 3.7; re-exported for compatibility.
|
|
57
|
+
export { intentBlocked } from './robots-parser.js';
|
|
58
|
+
function shapeOf(html) {
|
|
59
|
+
const title = html.match(/<title\b[^>]*>([\s\S]*?)<\/title>/i);
|
|
60
|
+
const h1 = html.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i);
|
|
61
|
+
return {
|
|
62
|
+
textLength: extractVisibleText(html).length,
|
|
63
|
+
title: title ? extractVisibleText(title[1]) : null,
|
|
64
|
+
h1: h1 ? extractVisibleText(h1[1]) : null,
|
|
65
|
+
jsonLdBlocks: findJsonLdBlocks(html).length,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
/** Differences between a crawler's page and the baseline, in reader-facing terms. */
|
|
69
|
+
function shapeDiff(baseline, actual) {
|
|
70
|
+
const diffs = [];
|
|
71
|
+
if (baseline.title !== null && actual.title !== baseline.title) {
|
|
72
|
+
diffs.push(`title differs ("${actual.title ?? 'none'}" vs "${baseline.title}")`);
|
|
73
|
+
}
|
|
74
|
+
if (baseline.h1 !== null && actual.h1 !== baseline.h1) {
|
|
75
|
+
diffs.push(`h1 differs ("${actual.h1 ?? 'none'}" vs "${baseline.h1}")`);
|
|
76
|
+
}
|
|
77
|
+
if (baseline.jsonLdBlocks > 0 && actual.jsonLdBlocks < baseline.jsonLdBlocks) {
|
|
78
|
+
diffs.push(`${baseline.jsonLdBlocks - actual.jsonLdBlocks} JSON-LD block(s) missing`);
|
|
79
|
+
}
|
|
80
|
+
return diffs;
|
|
81
|
+
}
|
|
82
|
+
export default async function check(ctx) {
|
|
83
|
+
const start = performance.now();
|
|
84
|
+
const findings = [];
|
|
85
|
+
const baseline = await ctx.fetch(ctx.url);
|
|
86
|
+
if (!baseline.ok) {
|
|
87
|
+
findings.push({
|
|
88
|
+
status: 'fail',
|
|
89
|
+
message: 'Baseline homepage request failed — cannot compare crawler access',
|
|
90
|
+
detail: `HTTP ${baseline.status || 'network error'}`,
|
|
91
|
+
learnMoreUrl: guideUrl(meta.id, 'baseline-unavailable'),
|
|
92
|
+
});
|
|
93
|
+
return buildResult(meta, 0, findings, start);
|
|
94
|
+
}
|
|
95
|
+
const baselineShape = shapeOf(baseline.body);
|
|
96
|
+
const baselineText = baselineShape.textLength;
|
|
97
|
+
const robotsRes = await ctx.fetch(`${ctx.url}/robots.txt`);
|
|
98
|
+
const robotsEntries = robotsRes.ok ? parseUserAgents(robotsRes.body) : [];
|
|
99
|
+
const outcomes = new Map();
|
|
100
|
+
for (const crawler of PROBEABLE_CORE_CRAWLERS) {
|
|
101
|
+
const res = await ctx.fetch(ctx.url, { headers: { 'User-Agent': crawlerUserAgent(crawler) } });
|
|
102
|
+
const cls = classifyResponse(res);
|
|
103
|
+
const blockedByRobots = intentBlocked(robotsEntries, crawler);
|
|
104
|
+
if (cls.kind === 'ok') {
|
|
105
|
+
const shape = shapeOf(res.body);
|
|
106
|
+
const diffs = shapeDiff(baselineShape, shape);
|
|
107
|
+
const textDropped = baselineText >= MIN_BASELINE_TEXT && shape.textLength < baselineText * REDUCED_CONTENT_RATIO;
|
|
108
|
+
if (textDropped || diffs.length > 0) {
|
|
109
|
+
outcomes.set(crawler, 'reduced');
|
|
110
|
+
findings.push({
|
|
111
|
+
status: 'warn',
|
|
112
|
+
message: `${crawler} receives a different page than a regular client`,
|
|
113
|
+
detail: [textDropped ? `${shape.textLength} vs ${baselineText} chars of visible text` : null, ...diffs]
|
|
114
|
+
.filter(Boolean)
|
|
115
|
+
.join('; '),
|
|
116
|
+
hint: 'The server returns 200 but serves this crawler different content — often an interstitial, a consent ' +
|
|
117
|
+
'wall, or conditional rendering. Agents index exactly what they receive, so the version they see is the ' +
|
|
118
|
+
'version that gets cited.',
|
|
119
|
+
learnMoreUrl: guideUrl(meta.id, 'reduced-content'),
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
outcomes.set(crawler, 'ok');
|
|
124
|
+
}
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
outcomes.set(crawler, recordTurnedAway(crawler, cls, blockedByRobots, robotsRes.ok, findings));
|
|
128
|
+
}
|
|
129
|
+
const okCount = [...outcomes.values()].filter((o) => o === 'ok').length;
|
|
130
|
+
if (okCount === PROBEABLE_CORE_CRAWLERS.length) {
|
|
131
|
+
findings.unshift({
|
|
132
|
+
status: 'pass',
|
|
133
|
+
message: `All ${PROBEABLE_CORE_CRAWLERS.length} core AI crawler user-agents receive the same page as a regular client`,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
const inconclusive = [...outcomes.entries()].filter(([, o]) => o === 'inconclusive');
|
|
137
|
+
if (inconclusive.length > 0) {
|
|
138
|
+
findings.push({
|
|
139
|
+
status: 'warn',
|
|
140
|
+
message: `${inconclusive.length} crawler probe(s) could not be settled from outside`,
|
|
141
|
+
detail: inconclusive.map(([name]) => name).join(', '),
|
|
142
|
+
hint: INCONCLUSIVE_CAVEAT,
|
|
143
|
+
learnMoreUrl: guideUrl(meta.id, 'inconclusive-probe'),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
const credit = [...outcomes.values()].reduce((acc, o) => acc + OUTCOME_CREDIT[o], 0);
|
|
147
|
+
const score = Math.round((credit / PROBEABLE_CORE_CRAWLERS.length) * 100);
|
|
148
|
+
return buildResult(meta, score, findings, start);
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Turn a non-200 crawler response into a finding and an outcome. The
|
|
152
|
+
* classification decides both the wording and the credit: a challenge is not a
|
|
153
|
+
* block, a price is not a refusal, and a signature demand is a site working as
|
|
154
|
+
* designed.
|
|
155
|
+
*/
|
|
156
|
+
function recordTurnedAway(crawler, cls, blockedByRobots, robotsAvailable, findings) {
|
|
157
|
+
const info = crawlerInfo(crawler);
|
|
158
|
+
const evidence = cls.evidence.length > 0 ? cls.evidence.join('; ') : cls.label;
|
|
159
|
+
switch (cls.kind) {
|
|
160
|
+
case 'challenge':
|
|
161
|
+
findings.push({
|
|
162
|
+
status: 'warn',
|
|
163
|
+
message: `${crawler} receives a ${cls.vendor ?? 'JavaScript'} challenge instead of the page`,
|
|
164
|
+
detail: evidence,
|
|
165
|
+
hint: 'Challenge pages require running JavaScript. Crawlers that only fetch HTML — which is most of them — never ' +
|
|
166
|
+
'get past one, so the page is effectively unavailable to them even though nothing is "blocked". ' +
|
|
167
|
+
'Add a bot-management exception for verified AI crawlers. ' +
|
|
168
|
+
INCONCLUSIVE_CAVEAT,
|
|
169
|
+
learnMoreUrl: guideUrl(meta.id, 'challenge-page'),
|
|
170
|
+
});
|
|
171
|
+
return 'inconclusive';
|
|
172
|
+
case 'needs-signature':
|
|
173
|
+
findings.push({
|
|
174
|
+
status: 'pass',
|
|
175
|
+
message: `${crawler} must present a Web Bot Auth signature`,
|
|
176
|
+
detail: evidence,
|
|
177
|
+
hint: 'The origin asks unverified clients to re-request with an HTTP Message Signature. Vendors that sign their ' +
|
|
178
|
+
'requests will pass; this probe does not sign, so it cannot confirm the outcome for the real crawler.',
|
|
179
|
+
learnMoreUrl: guideUrl(meta.id, 'needs-signature'),
|
|
180
|
+
});
|
|
181
|
+
return 'inconclusive';
|
|
182
|
+
case 'paywall':
|
|
183
|
+
findings.push({
|
|
184
|
+
status: 'pass',
|
|
185
|
+
message: `${crawler} is offered priced access${cls.price ? ` at ${cls.price}` : ''}`,
|
|
186
|
+
detail: evidence,
|
|
187
|
+
hint: 'Content is monetised for crawlers rather than blocked. Crawlers that support the payment flow can still reach it.',
|
|
188
|
+
learnMoreUrl: guideUrl(meta.id, 'pay-per-crawl'),
|
|
189
|
+
});
|
|
190
|
+
return 'conditional';
|
|
191
|
+
case 'license-required':
|
|
192
|
+
findings.push({
|
|
193
|
+
status: 'pass',
|
|
194
|
+
message: `${crawler} is asked to obtain a licence before use`,
|
|
195
|
+
detail: evidence,
|
|
196
|
+
hint: 'The origin implements the RSL Open Licensing Protocol. Clients that negotiate a licence can reach the content.',
|
|
197
|
+
learnMoreUrl: guideUrl(meta.id, 'license-required'),
|
|
198
|
+
});
|
|
199
|
+
return 'conditional';
|
|
200
|
+
case 'rate-limited':
|
|
201
|
+
findings.push({
|
|
202
|
+
status: 'warn',
|
|
203
|
+
message: `${crawler} is rate limited on a single request`,
|
|
204
|
+
detail: evidence,
|
|
205
|
+
hint: 'One probe should not hit a rate limit. A limit this tight will stall any crawl. Always answer 429 with a ' +
|
|
206
|
+
'Retry-After header so well-behaved crawlers back off correctly instead of giving up.',
|
|
207
|
+
learnMoreUrl: guideUrl(meta.id, 'rate-limited'),
|
|
208
|
+
});
|
|
209
|
+
return 'inconclusive';
|
|
210
|
+
case 'server-error':
|
|
211
|
+
case 'network-error':
|
|
212
|
+
findings.push({
|
|
213
|
+
status: 'warn',
|
|
214
|
+
message: `${crawler} probe failed: ${cls.label}`,
|
|
215
|
+
detail: evidence,
|
|
216
|
+
hint: 'The request did not complete, so access for this crawler is unknown. Re-run the audit; if it persists, check origin health for this user agent.',
|
|
217
|
+
learnMoreUrl: guideUrl(meta.id, 'probe-failed'),
|
|
218
|
+
});
|
|
219
|
+
return 'inconclusive';
|
|
220
|
+
default: {
|
|
221
|
+
// A plain refusal. Whether it is a finding depends on robots.txt intent.
|
|
222
|
+
if (blockedByRobots) {
|
|
223
|
+
findings.push({
|
|
224
|
+
status: 'pass',
|
|
225
|
+
message: `${crawler} blocked at the server — consistent with its robots.txt Disallow`,
|
|
226
|
+
detail: evidence,
|
|
227
|
+
});
|
|
228
|
+
return 'blocked-consistent';
|
|
229
|
+
}
|
|
230
|
+
if (cls.inconclusive) {
|
|
231
|
+
findings.push({
|
|
232
|
+
status: 'warn',
|
|
233
|
+
message: `${crawler} is refused by ${cls.vendor ?? 'the origin'} while robots.txt permits it`,
|
|
234
|
+
detail: evidence,
|
|
235
|
+
hint: (info ? `${info.impact} ` : '') +
|
|
236
|
+
'This edge verifies crawlers by IP range or signature, so the refusal may be correct anti-spoofing rather ' +
|
|
237
|
+
'than a policy block. ' +
|
|
238
|
+
INCONCLUSIVE_CAVEAT,
|
|
239
|
+
learnMoreUrl: guideUrl(meta.id, 'blocked-crawler'),
|
|
240
|
+
});
|
|
241
|
+
return 'inconclusive';
|
|
242
|
+
}
|
|
243
|
+
findings.push({
|
|
244
|
+
status: 'warn',
|
|
245
|
+
message: `${crawler} is ${robotsAvailable ? 'allowed in robots.txt' : 'not restricted'} but its User-Agent is refused`,
|
|
246
|
+
detail: evidence,
|
|
247
|
+
hint: (info ? `${info.impact} ` : '') +
|
|
248
|
+
'Your firewall rejects this crawler token even though robots.txt permits it — the block is invisible to you ' +
|
|
249
|
+
'but fatal for the agent. Check your firewall rules and AI-bot toggles (for example Cloudflare "Block AI Crawlers").',
|
|
250
|
+
learnMoreUrl: guideUrl(meta.id, 'blocked-crawler'),
|
|
251
|
+
});
|
|
252
|
+
return 'blocked';
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
//# sourceMappingURL=agent-access.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-access.js","sourceRoot":"","sources":["../../src/checks/agent-access.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAsB,MAAM,UAAU,CAAC;AAErF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,cAAc;IAClB,IAAI,EAAE,cAAc;IACpB,WAAW,EAAE,yFAAyF;IACtG,QAAQ,EAAE,QAAQ;CACnB,CAAC;AAEF,oFAAoF;AACpF,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,kGAAkG;AAClG,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAgB9B,2DAA2D;AAC3D,MAAM,cAAc,GAA4B;IAC9C,EAAE,EAAE,CAAC;IACL,oBAAoB,EAAE,CAAC;IACvB,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,IAAI;IAClB,OAAO,EAAE,GAAG;IACZ,OAAO,EAAE,CAAC;CACX,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,kEAAkE,KAAK,OAAO,CAAC;AACxF,CAAC;AAED,mFAAmF;AACnF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAUnD,SAAS,OAAO,CAAC,IAAY;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAC/D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IACtD,OAAO;QACL,UAAU,EAAE,kBAAkB,CAAC,IAAI,CAAC,CAAC,MAAM;QAC3C,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QAClD,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QACzC,YAAY,EAAE,gBAAgB,CAAC,IAAI,CAAC,CAAC,MAAM;KAC5C,CAAC;AACJ,CAAC;AAED,qFAAqF;AACrF,SAAS,SAAS,CAAC,QAAmB,EAAE,MAAiB;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,QAAQ,CAAC,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC/D,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,KAAK,IAAI,MAAM,SAAS,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,QAAQ,CAAC,EAAE,KAAK,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK,QAAQ,CAAC,EAAE,EAAE,CAAC;QACtD,KAAK,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,EAAE,IAAI,MAAM,SAAS,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,QAAQ,CAAC,YAAY,GAAG,CAAC,IAAI,MAAM,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,EAAE,CAAC;QAC7E,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,2BAA2B,CAAC,CAAC;IACxF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,kEAAkE;YAC3E,MAAM,EAAE,QAAQ,QAAQ,CAAC,MAAM,IAAI,eAAe,EAAE;YACpD,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,sBAAsB,CAAC;SACxD,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,aAAa,CAAC,UAAU,CAAC;IAE9C,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE1E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;IAE5C,KAAK,MAAM,OAAO,IAAI,uBAAuB,EAAE,CAAC;QAC9C,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/F,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,eAAe,GAAG,aAAa,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QAE9D,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YAC9C,MAAM,WAAW,GAAG,YAAY,IAAI,iBAAiB,IAAI,KAAK,CAAC,UAAU,GAAG,YAAY,GAAG,qBAAqB,CAAC;YAEjH,IAAI,WAAW,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;gBACjC,QAAQ,CAAC,IAAI,CAAC;oBACZ,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,GAAG,OAAO,kDAAkD;oBACrE,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU,OAAO,YAAY,wBAAwB,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC;yBACpG,MAAM,CAAC,OAAO,CAAC;yBACf,IAAI,CAAC,IAAI,CAAC;oBACb,IAAI,EACF,sGAAsG;wBACtG,yGAAyG;wBACzG,0BAA0B;oBAC5B,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,iBAAiB,CAAC;iBACnD,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAC9B,CAAC;YACD,SAAS;QACX,CAAC;QAED,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,eAAe,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACxE,IAAI,OAAO,KAAK,uBAAuB,CAAC,MAAM,EAAE,CAAC;QAC/C,QAAQ,CAAC,OAAO,CAAC;YACf,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,OAAO,uBAAuB,CAAC,MAAM,wEAAwE;SACvH,CAAC,CAAC;IACL,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,cAAc,CAAC,CAAC;IACrF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,YAAY,CAAC,MAAM,qDAAqD;YACpF,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACrD,IAAI,EAAE,mBAAmB;YACzB,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,CAAC;SACtD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,uBAAuB,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;IAE1E,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CACvB,OAAe,EACf,GAAkB,EAClB,eAAwB,EACxB,eAAwB,EACxB,QAAmB;IAEnB,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;IAE/E,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,OAAO,eAAe,GAAG,CAAC,MAAM,IAAI,YAAY,gCAAgC;gBAC5F,MAAM,EAAE,QAAQ;gBAChB,IAAI,EACF,4GAA4G;oBAC5G,iGAAiG;oBACjG,2DAA2D;oBAC3D,mBAAmB;gBACrB,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC;aAClD,CAAC,CAAC;YACH,OAAO,cAAc,CAAC;QAExB,KAAK,iBAAiB;YACpB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,OAAO,wCAAwC;gBAC3D,MAAM,EAAE,QAAQ;gBAChB,IAAI,EACF,2GAA2G;oBAC3G,sGAAsG;gBACxG,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,iBAAiB,CAAC;aACnD,CAAC,CAAC;YACH,OAAO,cAAc,CAAC;QAExB,KAAK,SAAS;YACZ,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,OAAO,4BAA4B,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpF,MAAM,EAAE,QAAQ;gBAChB,IAAI,EAAE,mHAAmH;gBACzH,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,eAAe,CAAC;aACjD,CAAC,CAAC;YACH,OAAO,aAAa,CAAC;QAEvB,KAAK,kBAAkB;YACrB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,OAAO,0CAA0C;gBAC7D,MAAM,EAAE,QAAQ;gBAChB,IAAI,EAAE,gHAAgH;gBACtH,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,kBAAkB,CAAC;aACpD,CAAC,CAAC;YACH,OAAO,aAAa,CAAC;QAEvB,KAAK,cAAc;YACjB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,OAAO,sCAAsC;gBACzD,MAAM,EAAE,QAAQ;gBAChB,IAAI,EACF,2GAA2G;oBAC3G,sFAAsF;gBACxF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;aAChD,CAAC,CAAC;YACH,OAAO,cAAc,CAAC;QAExB,KAAK,cAAc,CAAC;QACpB,KAAK,eAAe;YAClB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,OAAO,kBAAkB,GAAG,CAAC,KAAK,EAAE;gBAChD,MAAM,EAAE,QAAQ;gBAChB,IAAI,EAAE,iJAAiJ;gBACvJ,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;aAChD,CAAC,CAAC;YACH,OAAO,cAAc,CAAC;QAExB,OAAO,CAAC,CAAC,CAAC;YACR,yEAAyE;YACzE,IAAI,eAAe,EAAE,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC;oBACZ,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,GAAG,OAAO,kEAAkE;oBACrF,MAAM,EAAE,QAAQ;iBACjB,CAAC,CAAC;gBACH,OAAO,oBAAoB,CAAC;YAC9B,CAAC;YAED,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC;oBACZ,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,GAAG,OAAO,kBAAkB,GAAG,CAAC,MAAM,IAAI,YAAY,8BAA8B;oBAC7F,MAAM,EAAE,QAAQ;oBAChB,IAAI,EACF,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC/B,2GAA2G;wBAC3G,uBAAuB;wBACvB,mBAAmB;oBACrB,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,iBAAiB,CAAC;iBACnD,CAAC,CAAC;gBACH,OAAO,cAAc,CAAC;YACxB,CAAC;YAED,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,OAAO,OAAO,eAAe,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,gBAAgB,gCAAgC;gBACtH,MAAM,EAAE,QAAQ;gBAChB,IAAI,EACF,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC/B,6GAA6G;oBAC7G,qHAAqH;gBACvH,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,iBAAiB,CAAC;aACnD,CAAC,CAAC;YACH,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { CheckContext, CheckResult, CheckMeta } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* "agent-card" — the A2A (Agent2Agent) Agent Card.
|
|
4
|
+
*
|
|
5
|
+
* Two things changed since this check was written, and both matter:
|
|
6
|
+
*
|
|
7
|
+
* 1. **The path moved.** A2A v0.3.0 (2025-07-30) relocated the card from
|
|
8
|
+
* `/.well-known/agent.json` to `/.well-known/agent-card.json`, which is now
|
|
9
|
+
* an IANA-registered well-known URI. A site serving only the old path is
|
|
10
|
+
* invisible to a current client.
|
|
11
|
+
*
|
|
12
|
+
* 2. **The shape changed.** A2A 1.0 (2026-03-12, breaking) folded the
|
|
13
|
+
* top-level `url`, `protocolVersion`, `preferredTransport` and
|
|
14
|
+
* `additionalInterfaces` fields into a single `supportedInterfaces[]` array,
|
|
15
|
+
* and renamed `supportsAuthenticatedExtendedCard` to
|
|
16
|
+
* `capabilities.extendedAgentCard`. Most deployed cards are still 0.3-shaped.
|
|
17
|
+
*
|
|
18
|
+
* Validating a 1.0 card against 0.3 rules (or the reverse) produces nonsense,
|
|
19
|
+
* so the check detects which generation a card belongs to from its own
|
|
20
|
+
* structure and applies the matching rules. `authentication`, dropped back in
|
|
21
|
+
* 0.2.x in favour of `securitySchemes`, is flagged wherever it appears.
|
|
22
|
+
*
|
|
23
|
+
* Spec: https://a2a-protocol.org/latest/specification/
|
|
24
|
+
* Normative field list: specification/a2a.proto in github.com/a2aproject/A2A
|
|
25
|
+
*/
|
|
26
|
+
export declare const meta: CheckMeta;
|
|
27
|
+
/** Which generation of the A2A spec a card was written against. */
|
|
28
|
+
type Generation = 'v1' | 'v0.3' | 'unknown';
|
|
29
|
+
/**
|
|
30
|
+
* Detect the card generation from its own structure rather than from a version
|
|
31
|
+
* field, because 0.3 cards state `protocolVersion` at the top level and 1.0
|
|
32
|
+
* cards state it per interface.
|
|
33
|
+
*/
|
|
34
|
+
export declare function detectGeneration(data: Record<string, unknown>): Generation;
|
|
35
|
+
export default function check(ctx: CheckContext): Promise<CheckResult>;
|
|
36
|
+
export {};
|
|
37
|
+
//# sourceMappingURL=agent-card.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-card.d.ts","sourceRoot":"","sources":["../../src/checks/agent-card.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAA0B,MAAM,aAAa,CAAC;AAKhG;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,eAAO,MAAM,IAAI,EAAE,SAMlB,CAAC;AAKF,mEAAmE;AACnE,KAAK,UAAU,GAAG,IAAI,GAAG,MAAM,GAAG,SAAS,CAAC;AAE5C;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,UAAU,CAI1E;AAqBD,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAkF3E"}
|