ax-audit 5.0.0 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/CHANGELOG.md +10 -0
  2. package/README.md +33 -24
  3. package/dist/baseline.js +1 -1
  4. package/dist/baseline.js.map +1 -1
  5. package/dist/check-ids.js +1 -1
  6. package/dist/check-ids.js.map +1 -1
  7. package/dist/cli.js +1 -1
  8. package/dist/cli.js.map +1 -1
  9. package/dist/constants.d.ts +1 -85
  10. package/dist/constants.d.ts.map +1 -1
  11. package/dist/constants.js +0 -135
  12. package/dist/constants.js.map +1 -1
  13. package/dist/index.d.ts +1 -1
  14. package/dist/index.d.ts.map +1 -1
  15. package/dist/index.js +1 -1
  16. package/dist/index.js.map +1 -1
  17. package/dist/license.d.ts +3 -2
  18. package/dist/license.d.ts.map +1 -1
  19. package/dist/license.js +5 -58
  20. package/dist/license.js.map +1 -1
  21. package/dist/metadata.d.ts +6 -0
  22. package/dist/metadata.d.ts.map +1 -0
  23. package/dist/metadata.js +205 -0
  24. package/dist/metadata.js.map +1 -0
  25. package/dist/orchestrator.d.ts +1 -0
  26. package/dist/orchestrator.d.ts.map +1 -1
  27. package/dist/orchestrator.js +134 -63
  28. package/dist/orchestrator.js.map +1 -1
  29. package/dist/scorer.js +2 -2
  30. package/dist/scorer.js.map +1 -1
  31. package/dist/types.d.ts +2 -0
  32. package/dist/types.d.ts.map +1 -1
  33. package/docs/api.md +12 -4
  34. package/docs/architecture.md +9 -98
  35. package/docs/ci.md +7 -3
  36. package/docs/cli.md +7 -3
  37. package/docs/faq.md +18 -0
  38. package/docs/getting-started.md +32 -26
  39. package/package.json +5 -5
  40. package/dist/checks/agent-access.d.ts +0 -33
  41. package/dist/checks/agent-access.d.ts.map +0 -1
  42. package/dist/checks/agent-access.js +0 -256
  43. package/dist/checks/agent-access.js.map +0 -1
  44. package/dist/checks/agent-card.d.ts +0 -37
  45. package/dist/checks/agent-card.d.ts.map +0 -1
  46. package/dist/checks/agent-card.js +0 -352
  47. package/dist/checks/agent-card.js.map +0 -1
  48. package/dist/checks/agent-operability.d.ts +0 -66
  49. package/dist/checks/agent-operability.d.ts.map +0 -1
  50. package/dist/checks/agent-operability.js +0 -383
  51. package/dist/checks/agent-operability.js.map +0 -1
  52. package/dist/checks/agent-skills.d.ts +0 -24
  53. package/dist/checks/agent-skills.d.ts.map +0 -1
  54. package/dist/checks/agent-skills.js +0 -316
  55. package/dist/checks/agent-skills.js.map +0 -1
  56. package/dist/checks/ai-catalog.d.ts +0 -28
  57. package/dist/checks/ai-catalog.d.ts.map +0 -1
  58. package/dist/checks/ai-catalog.js +0 -254
  59. package/dist/checks/ai-catalog.js.map +0 -1
  60. package/dist/checks/ai-directives.d.ts +0 -57
  61. package/dist/checks/ai-directives.d.ts.map +0 -1
  62. package/dist/checks/ai-directives.js +0 -263
  63. package/dist/checks/ai-directives.js.map +0 -1
  64. package/dist/checks/api-discovery.d.ts +0 -26
  65. package/dist/checks/api-discovery.d.ts.map +0 -1
  66. package/dist/checks/api-discovery.js +0 -432
  67. package/dist/checks/api-discovery.js.map +0 -1
  68. package/dist/checks/auth-discovery.d.ts +0 -28
  69. package/dist/checks/auth-discovery.d.ts.map +0 -1
  70. package/dist/checks/auth-discovery.js +0 -302
  71. package/dist/checks/auth-discovery.js.map +0 -1
  72. package/dist/checks/commerce-discovery.d.ts +0 -40
  73. package/dist/checks/commerce-discovery.d.ts.map +0 -1
  74. package/dist/checks/commerce-discovery.js +0 -295
  75. package/dist/checks/commerce-discovery.js.map +0 -1
  76. package/dist/checks/content-negotiation.d.ts +0 -4
  77. package/dist/checks/content-negotiation.d.ts.map +0 -1
  78. package/dist/checks/content-negotiation.js +0 -253
  79. package/dist/checks/content-negotiation.js.map +0 -1
  80. package/dist/checks/crawl-efficiency.d.ts +0 -16
  81. package/dist/checks/crawl-efficiency.d.ts.map +0 -1
  82. package/dist/checks/crawl-efficiency.js +0 -186
  83. package/dist/checks/crawl-efficiency.js.map +0 -1
  84. package/dist/checks/frontmatter.d.ts +0 -34
  85. package/dist/checks/frontmatter.d.ts.map +0 -1
  86. package/dist/checks/frontmatter.js +0 -100
  87. package/dist/checks/frontmatter.js.map +0 -1
  88. package/dist/checks/html-rendering.d.ts +0 -21
  89. package/dist/checks/html-rendering.d.ts.map +0 -1
  90. package/dist/checks/html-rendering.js +0 -221
  91. package/dist/checks/html-rendering.js.map +0 -1
  92. package/dist/checks/html-utils.d.ts +0 -66
  93. package/dist/checks/html-utils.d.ts.map +0 -1
  94. package/dist/checks/html-utils.js +0 -139
  95. package/dist/checks/html-utils.js.map +0 -1
  96. package/dist/checks/http-headers.d.ts +0 -11
  97. package/dist/checks/http-headers.d.ts.map +0 -1
  98. package/dist/checks/http-headers.js +0 -249
  99. package/dist/checks/http-headers.js.map +0 -1
  100. package/dist/checks/http-hygiene.d.ts +0 -26
  101. package/dist/checks/http-hygiene.d.ts.map +0 -1
  102. package/dist/checks/http-hygiene.js +0 -257
  103. package/dist/checks/http-hygiene.js.map +0 -1
  104. package/dist/checks/index.d.ts +0 -3
  105. package/dist/checks/index.d.ts.map +0 -1
  106. package/dist/checks/index.js +0 -55
  107. package/dist/checks/index.js.map +0 -1
  108. package/dist/checks/llms-txt.d.ts +0 -19
  109. package/dist/checks/llms-txt.d.ts.map +0 -1
  110. package/dist/checks/llms-txt.js +0 -281
  111. package/dist/checks/llms-txt.js.map +0 -1
  112. package/dist/checks/mcp-discovery.d.ts +0 -30
  113. package/dist/checks/mcp-discovery.d.ts.map +0 -1
  114. package/dist/checks/mcp-discovery.js +0 -523
  115. package/dist/checks/mcp-discovery.js.map +0 -1
  116. package/dist/checks/meta-tags.d.ts +0 -17
  117. package/dist/checks/meta-tags.d.ts.map +0 -1
  118. package/dist/checks/meta-tags.js +0 -188
  119. package/dist/checks/meta-tags.js.map +0 -1
  120. package/dist/checks/robots-parser.d.ts +0 -110
  121. package/dist/checks/robots-parser.d.ts.map +0 -1
  122. package/dist/checks/robots-parser.js +0 -277
  123. package/dist/checks/robots-parser.js.map +0 -1
  124. package/dist/checks/robots-txt.d.ts +0 -6
  125. package/dist/checks/robots-txt.d.ts.map +0 -1
  126. package/dist/checks/robots-txt.js +0 -367
  127. package/dist/checks/robots-txt.js.map +0 -1
  128. package/dist/checks/rsl.d.ts +0 -4
  129. package/dist/checks/rsl.d.ts.map +0 -1
  130. package/dist/checks/rsl.js +0 -242
  131. package/dist/checks/rsl.js.map +0 -1
  132. package/dist/checks/security-txt.d.ts +0 -4
  133. package/dist/checks/security-txt.d.ts.map +0 -1
  134. package/dist/checks/security-txt.js +0 -81
  135. package/dist/checks/security-txt.js.map +0 -1
  136. package/dist/checks/seo-basics.d.ts +0 -13
  137. package/dist/checks/seo-basics.d.ts.map +0 -1
  138. package/dist/checks/seo-basics.js +0 -221
  139. package/dist/checks/seo-basics.js.map +0 -1
  140. package/dist/checks/sitemap.d.ts +0 -12
  141. package/dist/checks/sitemap.d.ts.map +0 -1
  142. package/dist/checks/sitemap.js +0 -240
  143. package/dist/checks/sitemap.js.map +0 -1
  144. package/dist/checks/structured-data.d.ts +0 -4
  145. package/dist/checks/structured-data.d.ts.map +0 -1
  146. package/dist/checks/structured-data.js +0 -408
  147. package/dist/checks/structured-data.js.map +0 -1
  148. package/dist/checks/structured-fields.d.ts +0 -46
  149. package/dist/checks/structured-fields.d.ts.map +0 -1
  150. package/dist/checks/structured-fields.js +0 -112
  151. package/dist/checks/structured-fields.js.map +0 -1
  152. package/dist/checks/surface.d.ts +0 -59
  153. package/dist/checks/surface.d.ts.map +0 -1
  154. package/dist/checks/surface.js +0 -106
  155. package/dist/checks/surface.js.map +0 -1
  156. package/dist/checks/tls-https.d.ts +0 -13
  157. package/dist/checks/tls-https.d.ts.map +0 -1
  158. package/dist/checks/tls-https.js +0 -163
  159. package/dist/checks/tls-https.js.map +0 -1
  160. package/dist/checks/usage-policy.d.ts +0 -53
  161. package/dist/checks/usage-policy.d.ts.map +0 -1
  162. package/dist/checks/usage-policy.js +0 -339
  163. package/dist/checks/usage-policy.js.map +0 -1
  164. package/dist/checks/utils.d.ts +0 -40
  165. package/dist/checks/utils.d.ts.map +0 -1
  166. package/dist/checks/utils.js +0 -74
  167. package/dist/checks/utils.js.map +0 -1
  168. package/dist/checks/waf.d.ts +0 -75
  169. package/dist/checks/waf.d.ts.map +0 -1
  170. package/dist/checks/waf.js +0 -203
  171. package/dist/checks/waf.js.map +0 -1
  172. package/dist/checks/webmcp.d.ts +0 -55
  173. package/dist/checks/webmcp.d.ts.map +0 -1
  174. package/dist/checks/webmcp.js +0 -209
  175. package/dist/checks/webmcp.js.map +0 -1
  176. package/dist/checks/well-known.d.ts +0 -38
  177. package/dist/checks/well-known.d.ts.map +0 -1
  178. package/dist/checks/well-known.js +0 -202
  179. package/dist/checks/well-known.js.map +0 -1
  180. package/dist/fetcher.d.ts +0 -15
  181. package/dist/fetcher.d.ts.map +0 -1
  182. package/dist/fetcher.js +0 -127
  183. package/dist/fetcher.js.map +0 -1
  184. package/dist/guide-urls.d.ts +0 -2
  185. package/dist/guide-urls.d.ts.map +0 -1
  186. package/dist/guide-urls.js +0 -5
  187. package/dist/guide-urls.js.map +0 -1
  188. package/dist/licensed-checks.d.ts +0 -4
  189. package/dist/licensed-checks.d.ts.map +0 -1
  190. package/dist/licensed-checks.js +0 -11
  191. package/dist/licensed-checks.js.map +0 -1
  192. package/docs/roadmap.md +0 -367
@@ -1,57 +0,0 @@
1
- import type { CheckContext, CheckResult, CheckMeta } from '../types.js';
2
- /**
3
- * "ai-directives" — the page-level controls that actually govern whether your
4
- * content can appear in an AI answer.
5
- *
6
- * robots.txt decides whether a crawler may *fetch* a page. These directives
7
- * decide what may be *done* with it once fetched, and unlike most of the
8
- * machine-readable signals in this audit, the two largest vendors document that
9
- * they honor them:
10
- *
11
- * - Google: `nosnippet` "will also prevent the content from being used as a
12
- * direct input for AI Overviews and AI Mode"; `max-snippet:[n]` limits how
13
- * much may be used; `data-nosnippet` scopes the exclusion to part of a page.
14
- * - Microsoft: `noarchive` excludes a page from Copilot grounding entirely;
15
- * `nocache` limits Copilot to the URL, title and snippet.
16
- *
17
- * The most common misconception this check corrects is about `Google-Extended`.
18
- * Operators disallow it expecting to leave AI Overviews. It does not do that:
19
- * it governs Gemini training and grounding in Gemini Apps and Vertex AI, while
20
- * AI Overviews and AI Mode follow Googlebot and the snippet directives. A site
21
- * that disallows `Google-Extended` and sets no snippet directive has opted out
22
- * of the thing it probably did not mind and stayed in the thing it did.
23
- *
24
- * `noai` and `noimageai` are reported as declared preferences. No major AI
25
- * operator has committed to honoring them, and saying otherwise would be
26
- * misleading.
27
- */
28
- export declare const meta: CheckMeta;
29
- /** Directive tokens parsed out of a robots meta tag or `X-Robots-Tag` header. */
30
- export interface ParsedDirectives {
31
- /** Bare tokens, lowercased. */
32
- tokens: Set<string>;
33
- /** `max-snippet` value, or `null` when absent. `-1` means unlimited. */
34
- maxSnippet: number | null;
35
- /** Which sources declared directives, for the report. */
36
- sources: string[];
37
- }
38
- /**
39
- * Parse robots directives from meta tags and `X-Robots-Tag`.
40
- *
41
- * `X-Robots-Tag` may name a user agent before its directives
42
- * (`X-Robots-Tag: googlebot: noindex`) and may repeat, which HTTP folds into
43
- * one comma-separated value. Both forms are handled; a UA-scoped rule counts
44
- * because the audit reports the strictest thing any major crawler is told.
45
- */
46
- export declare function parseDirectives(html: string, headers: Record<string, string>): ParsedDirectives;
47
- /**
48
- * Count `data-nosnippet` regions and judge whether they cover the page's
49
- * substance. Scoping a cookie banner is housekeeping; wrapping `<main>` removes
50
- * the page from AI answers.
51
- */
52
- export declare function analyseDataNosnippet(html: string): {
53
- count: number;
54
- coversMain: boolean;
55
- };
56
- export default function check(ctx: CheckContext): Promise<CheckResult>;
57
- //# sourceMappingURL=ai-directives.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"ai-directives.d.ts","sourceRoot":"","sources":["../../src/checks/ai-directives.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAIjF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;AAEF,iFAAiF;AACjF,MAAM,WAAW,gBAAgB;IAC/B,+BAA+B;IAC/B,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACpB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,yDAAyD;IACzD,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,gBAAgB,CAyC/F;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAA;CAAE,CAIzF;AAED,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAgK3E"}
@@ -1,263 +0,0 @@
1
- import { guideUrl } from '../guide-urls.js';
2
- import { buildResult } from './utils.js';
3
- import { getMetaContent } from './html-utils.js';
4
- /**
5
- * "ai-directives" — the page-level controls that actually govern whether your
6
- * content can appear in an AI answer.
7
- *
8
- * robots.txt decides whether a crawler may *fetch* a page. These directives
9
- * decide what may be *done* with it once fetched, and unlike most of the
10
- * machine-readable signals in this audit, the two largest vendors document that
11
- * they honor them:
12
- *
13
- * - Google: `nosnippet` "will also prevent the content from being used as a
14
- * direct input for AI Overviews and AI Mode"; `max-snippet:[n]` limits how
15
- * much may be used; `data-nosnippet` scopes the exclusion to part of a page.
16
- * - Microsoft: `noarchive` excludes a page from Copilot grounding entirely;
17
- * `nocache` limits Copilot to the URL, title and snippet.
18
- *
19
- * The most common misconception this check corrects is about `Google-Extended`.
20
- * Operators disallow it expecting to leave AI Overviews. It does not do that:
21
- * it governs Gemini training and grounding in Gemini Apps and Vertex AI, while
22
- * AI Overviews and AI Mode follow Googlebot and the snippet directives. A site
23
- * that disallows `Google-Extended` and sets no snippet directive has opted out
24
- * of the thing it probably did not mind and stayed in the thing it did.
25
- *
26
- * `noai` and `noimageai` are reported as declared preferences. No major AI
27
- * operator has committed to honoring them, and saying otherwise would be
28
- * misleading.
29
- */
30
- export const meta = {
31
- id: 'ai-directives',
32
- name: 'AI Directives',
33
- description: 'Checks page-level robots directives that govern AI answers and summaries',
34
- category: 'access',
35
- };
36
- /**
37
- * Parse robots directives from meta tags and `X-Robots-Tag`.
38
- *
39
- * `X-Robots-Tag` may name a user agent before its directives
40
- * (`X-Robots-Tag: googlebot: noindex`) and may repeat, which HTTP folds into
41
- * one comma-separated value. Both forms are handled; a UA-scoped rule counts
42
- * because the audit reports the strictest thing any major crawler is told.
43
- */
44
- export function parseDirectives(html, headers) {
45
- const tokens = new Set();
46
- const sources = [];
47
- let maxSnippet = null;
48
- const consume = (value, source) => {
49
- let sawAny = false;
50
- for (const raw of value.split(',')) {
51
- // Strip a leading `<user-agent>:` scope if present.
52
- const segment = raw.includes(':') && !/^\s*max-(snippet|image-preview|video-preview)\s*:/i.test(raw)
53
- ? raw.slice(raw.indexOf(':') + 1)
54
- : raw;
55
- const token = segment.trim().toLowerCase();
56
- if (!token)
57
- continue;
58
- const snippet = token.match(/^max-snippet\s*:\s*(-?\d+)$/);
59
- if (snippet) {
60
- // `-1` means "no limit", so it must never win a minimum against a real
61
- // cap. Normalise it to Infinity while comparing and restore it after.
62
- const n = Number(snippet[1]) === -1 ? Infinity : Number(snippet[1]);
63
- maxSnippet = maxSnippet === null ? n : Math.min(maxSnippet, n);
64
- tokens.add('max-snippet');
65
- sawAny = true;
66
- continue;
67
- }
68
- tokens.add(token);
69
- sawAny = true;
70
- }
71
- if (sawAny)
72
- sources.push(source);
73
- };
74
- for (const name of ['robots', 'googlebot', 'bingbot', 'google', 'msnbot']) {
75
- const content = getMetaContent(html, name);
76
- if (content !== null)
77
- consume(content, `<meta name="${name}">`);
78
- }
79
- if (headers['x-robots-tag'] !== undefined)
80
- consume(headers['x-robots-tag'], 'X-Robots-Tag header');
81
- return { tokens, maxSnippet: maxSnippet === Infinity ? -1 : maxSnippet, sources };
82
- }
83
- /**
84
- * Count `data-nosnippet` regions and judge whether they cover the page's
85
- * substance. Scoping a cookie banner is housekeeping; wrapping `<main>` removes
86
- * the page from AI answers.
87
- */
88
- export function analyseDataNosnippet(html) {
89
- const matches = [...html.matchAll(/<([a-z][a-z0-9-]*)\b([^>]*\bdata-nosnippet\b[^>]*)>/gi)];
90
- const coversMain = matches.some((m) => /^(main|article|body)$/i.test(m[1]));
91
- return { count: matches.length, coversMain };
92
- }
93
- export default async function check(ctx) {
94
- const start = performance.now();
95
- const findings = [];
96
- let score = 100;
97
- if (!ctx.html || ctx.html.trim().length === 0) {
98
- findings.push({
99
- status: 'fail',
100
- message: 'Homepage HTML unavailable — cannot read AI directives',
101
- learnMoreUrl: guideUrl(meta.id, 'no-html'),
102
- });
103
- return buildResult(meta, 0, findings, start);
104
- }
105
- const { tokens, maxSnippet, sources } = parseDirectives(ctx.html, ctx.headers ?? {});
106
- const has = (t) => tokens.has(t);
107
- /* ── Indexability: the precondition for everything else ─────────────── */
108
- if (has('noindex') || has('none')) {
109
- findings.push({
110
- status: 'fail',
111
- message: 'Homepage is marked noindex',
112
- detail: sources.join(', '),
113
- hint: 'A noindex page is invisible to every search-grounded assistant, because they all cite from a search index. ' +
114
- 'If this is deliberate, nothing else in this check matters. If it is not, remove the directive.',
115
- learnMoreUrl: guideUrl(meta.id, 'noindex'),
116
- });
117
- return buildResult(meta, 0, findings, start);
118
- }
119
- findings.push({ status: 'pass', message: 'Homepage is indexable' });
120
- /* ── Snippet controls: Google AI Overviews and AI Mode ───────────────── */
121
- if (has('nosnippet')) {
122
- findings.push({
123
- status: 'warn',
124
- message: 'nosnippet excludes this page from Google AI Overviews and AI Mode',
125
- detail: sources.join(', '),
126
- hint: 'Google documents that nosnippet prevents content being used as a direct input for AI Overviews and AI Mode, ' +
127
- 'as well as suppressing the search snippet. Remove it if you want to be quoted; keep it if you do not.',
128
- learnMoreUrl: guideUrl(meta.id, 'nosnippet'),
129
- });
130
- score -= 30;
131
- }
132
- else if (maxSnippet !== null && maxSnippet !== -1) {
133
- if (maxSnippet === 0) {
134
- findings.push({
135
- status: 'warn',
136
- message: 'max-snippet:0 has the same effect as nosnippet',
137
- detail: sources.join(', '),
138
- hint: 'A zero-character snippet limit excludes the page from AI Overviews and AI Mode as an input.',
139
- learnMoreUrl: guideUrl(meta.id, 'nosnippet'),
140
- });
141
- score -= 30;
142
- }
143
- else {
144
- findings.push({
145
- status: maxSnippet < 160 ? 'warn' : 'pass',
146
- message: `max-snippet:${maxSnippet} limits how much of this page an AI answer may use`,
147
- detail: sources.join(', '),
148
- ...(maxSnippet < 160
149
- ? {
150
- hint: 'Below roughly 160 characters there is little room for a useful quotation. Use max-snippet:-1 for no limit.',
151
- learnMoreUrl: guideUrl(meta.id, 'max-snippet'),
152
- }
153
- : {}),
154
- });
155
- }
156
- }
157
- else if (maxSnippet === -1) {
158
- findings.push({ status: 'pass', message: 'max-snippet:-1 — no limit on snippet length' });
159
- }
160
- /* ── Copilot controls ────────────────────────────────────────────────── */
161
- if (has('noarchive')) {
162
- findings.push({
163
- status: 'warn',
164
- message: 'noarchive excludes this page from Microsoft Copilot grounding',
165
- detail: sources.join(', '),
166
- hint: 'Microsoft documents that noarchive means a page is not included in Copilot answers and not linked from them. ' +
167
- 'nocache is the lighter option: Copilot may use the URL, title and snippet but not the body.',
168
- learnMoreUrl: guideUrl(meta.id, 'noarchive'),
169
- });
170
- score -= 30;
171
- }
172
- else if (has('nocache')) {
173
- findings.push({
174
- status: 'warn',
175
- message: 'nocache limits Copilot to the URL, title and snippet of this page',
176
- detail: sources.join(', '),
177
- hint: 'Copilot may cite the page but not use its body. Remove nocache to allow full grounding.',
178
- learnMoreUrl: guideUrl(meta.id, 'nocache'),
179
- });
180
- score -= 10;
181
- }
182
- /* ── Scoped exclusions ───────────────────────────────────────────────── */
183
- const dataNosnippet = analyseDataNosnippet(ctx.html);
184
- if (dataNosnippet.coversMain) {
185
- findings.push({
186
- status: 'warn',
187
- message: 'data-nosnippet wraps the page’s main content',
188
- detail: `${dataNosnippet.count} data-nosnippet region(s) found`,
189
- hint: 'Scoping a cookie banner or a byline is housekeeping. Wrapping <main>, <article> or <body> removes the page from AI answers as surely as nosnippet does.',
190
- learnMoreUrl: guideUrl(meta.id, 'data-nosnippet-main'),
191
- });
192
- score -= 20;
193
- }
194
- else if (dataNosnippet.count > 0) {
195
- findings.push({
196
- status: 'pass',
197
- message: `${dataNosnippet.count} data-nosnippet region(s) — scoped exclusions, main content still quotable`,
198
- });
199
- }
200
- if (has('noimageindex')) {
201
- findings.push({
202
- status: 'warn',
203
- message: 'noimageindex keeps this page’s images out of image results and multimodal answers',
204
- detail: sources.join(', '),
205
- hint: 'Remove it if you want images cited alongside your text.',
206
- learnMoreUrl: guideUrl(meta.id, 'noimageindex'),
207
- });
208
- score -= 5;
209
- }
210
- /* ── Declared preferences with no committed honorer ──────────────────── */
211
- const declared = ['noai', 'noimageai'].filter(has);
212
- if (declared.length > 0) {
213
- findings.push({
214
- status: 'pass',
215
- message: `${declared.join(' and ')} declared`,
216
- detail: 'A stated preference against AI use. No major AI operator documents honoring these tokens, so treat them as a ' +
217
- 'legal or ethical signal rather than an enforcement mechanism.',
218
- });
219
- }
220
- if (has('notranslate')) {
221
- findings.push({
222
- status: 'pass',
223
- message: 'notranslate declared — assistants should not offer a translated version',
224
- });
225
- }
226
- /* ── The Google-Extended misconception ───────────────────────────────── */
227
- await reportGoogleExtendedMismatch(ctx, tokens, findings);
228
- if (findings.every((f) => f.status === 'pass')) {
229
- findings.push({
230
- status: 'pass',
231
- message: 'No directive restricts how AI assistants may use this page',
232
- });
233
- }
234
- return buildResult(meta, score, findings, start);
235
- }
236
- /**
237
- * Flag the most common mistaken belief about AI opt-outs: that disallowing
238
- * `Google-Extended` removes a site from AI Overviews. It does not, and a site
239
- * that made that assumption is opted out of Gemini training while remaining
240
- * fully quotable in the surface it was trying to leave.
241
- */
242
- async function reportGoogleExtendedMismatch(ctx, tokens, findings) {
243
- const robots = await ctx.fetch(`${ctx.url}/robots.txt`);
244
- if (!robots.ok)
245
- return;
246
- const { parseRobotsTxt, toBotEntries, intentBlocked } = await import('./robots-parser.js');
247
- const blocked = intentBlocked(toBotEntries(parseRobotsTxt(robots.body)), 'Google-Extended');
248
- if (!blocked)
249
- return;
250
- const restricted = tokens.has('nosnippet') || tokens.has('noindex');
251
- if (restricted)
252
- return;
253
- findings.push({
254
- status: 'warn',
255
- message: 'Google-Extended is disallowed, but this page is still eligible for AI Overviews',
256
- detail: 'robots.txt disallows Google-Extended; no nosnippet or max-snippet directive is set on the page.',
257
- hint: 'Google-Extended governs Gemini training and grounding in Gemini Apps and Vertex AI. AI Overviews and AI Mode ' +
258
- 'follow Googlebot and the snippet directives instead. If the intent was to stay out of AI Overviews, use ' +
259
- 'nosnippet or max-snippet. If the intent was to opt out of training, this is already correct.',
260
- learnMoreUrl: guideUrl(meta.id, 'google-extended-mismatch'),
261
- });
262
- }
263
- //# sourceMappingURL=ai-directives.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"ai-directives.js","sourceRoot":"","sources":["../../src/checks/ai-directives.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,eAAe;IACnB,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,0EAA0E;IACvF,QAAQ,EAAE,QAAQ;CACnB,CAAC;AAYF;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,OAA+B;IAC3E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAkB,IAAI,CAAC;IAErC,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,MAAc,EAAQ,EAAE;QACtD,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,oDAAoD;YACpD,MAAM,OAAO,GACX,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,oDAAoD,CAAC,IAAI,CAAC,GAAG,CAAC;gBAClF,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACjC,CAAC,CAAC,GAAG,CAAC;YACV,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC3D,IAAI,OAAO,EAAE,CAAC;gBACZ,uEAAuE;gBACvE,sEAAsE;gBACtE,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpE,UAAU,GAAG,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBAC/D,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;gBAC1B,MAAM,GAAG,IAAI,CAAC;gBACd,SAAS;YACX,CAAC;YAED,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAClB,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,CAAC,OAAO,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,OAAO,CAAC,cAAc,CAAC,KAAK,SAAS;QAAE,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAEnG,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC;AACpF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,uDAAuD,CAAC,CAAC,CAAC;IAC5F,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAG,GAAG,CAAC;IAEhB,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uDAAuD;YAChE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC;SAC3C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IACrF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAElD,2EAA2E;IAE3E,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAClC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4BAA4B;YACrC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1B,IAAI,EACF,6GAA6G;gBAC7G,gGAAgG;YAClG,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC;SAC3C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;IAEpE,4EAA4E;IAE5E,IAAI,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;QACrB,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,mEAAmE;YAC5E,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1B,IAAI,EACF,8GAA8G;gBAC9G,uGAAuG;YACzG,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC;SAC7C,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;SAAM,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACpD,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;YACrB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,gDAAgD;gBACzD,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC1B,IAAI,EAAE,6FAA6F;gBACnG,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC;aAC7C,CAAC,CAAC;YACH,KAAK,IAAI,EAAE,CAAC;QACd,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBAC1C,OAAO,EAAE,eAAe,UAAU,oDAAoD;gBACtF,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC1B,GAAG,CAAC,UAAU,GAAG,GAAG;oBAClB,CAAC,CAAC;wBACE,IAAI,EAAE,4GAA4G;wBAClH,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC;qBAC/C;oBACH,CAAC,CAAC,EAAE,CAAC;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;SAAM,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,6CAA6C,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,4EAA4E;IAE5E,IAAI,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;QACrB,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,+DAA+D;YACxE,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1B,IAAI,EACF,+GAA+G;gBAC/G,6FAA6F;YAC/F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC;SAC7C,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;SAAM,IAAI,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,mEAAmE;YAC5E,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1B,IAAI,EAAE,yFAAyF;YAC/F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC;SAC3C,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;IAED,4EAA4E;IAE5E,MAAM,aAAa,GAAG,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrD,IAAI,aAAa,CAAC,UAAU,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,8CAA8C;YACvD,MAAM,EAAE,GAAG,aAAa,CAAC,KAAK,iCAAiC;YAC/D,IAAI,EAAE,yJAAyJ;YAC/J,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAqB,CAAC;SACvD,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;SAAM,IAAI,aAAa,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,aAAa,CAAC,KAAK,4EAA4E;SAC5G,CAAC,CAAC;IACL,CAAC;IAED,IAAI,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,mFAAmF;YAC5F,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1B,IAAI,EAAE,yDAAyD;YAC/D,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;SAChD,CAAC,CAAC;QACH,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,4EAA4E;IAE5E,MAAM,QAAQ,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW;YAC7C,MAAM,EACJ,+GAA+G;gBAC/G,+DAA+D;SAClE,CAAC,CAAC;IACL,CAAC;IAED,IAAI,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,yEAAyE;SACnF,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAE5E,MAAM,4BAA4B,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAE1D,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4DAA4D;SACtE,CAAC,CAAC;IACL,CAAC;IAED,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,4BAA4B,CACzC,GAAiB,EACjB,MAAmB,EACnB,QAAmB;IAEnB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;IACxD,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO;IAEvB,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC3F,MAAM,OAAO,GAAG,aAAa,CAAC,YAAY,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAC5F,IAAI,CAAC,OAAO;QAAE,OAAO;IAErB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACpE,IAAI,UAAU;QAAE,OAAO;IAEvB,QAAQ,CAAC,IAAI,CAAC;QACZ,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,iFAAiF;QAC1F,MAAM,EAAE,iGAAiG;QACzG,IAAI,EACF,+GAA+G;YAC/G,0GAA0G;YAC1G,8FAA8F;QAChG,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,0BAA0B,CAAC;KAC5D,CAAC,CAAC;AACL,CAAC"}
@@ -1,26 +0,0 @@
1
- import type { CheckContext, CheckResult, CheckMeta } from '../types.js';
2
- /**
3
- * "api-discovery" — can an agent find, and read, this site's API description?
4
- *
5
- * The check used to probe exactly one path, `/.well-known/openapi.json`. That
6
- * path is a folk convention: it is not in the IANA well-known registry, and the
7
- * OpenAPI specification itself recommends the file be named `openapi.json` or
8
- * `openapi.yaml` without prescribing a location. So a site doing everything
9
- * right at `/openapi.json` scored zero.
10
- *
11
- * There *is* a registered mechanism: RFC 9727 defines `/.well-known/api-catalog`,
12
- * a linkset of the APIs a publisher offers, each entry pointing at a
13
- * machine-readable description via `service-desc` and human documentation via
14
- * `service-doc` (RFC 8631). Adoption is tiny — a handful of publishers — but it
15
- * is the only standards-track answer, and checking it costs one request.
16
- *
17
- * The check therefore walks discovery in order of authority: the registered
18
- * catalog, then RFC 8631 link relations in headers and HTML, then the
19
- * conventional file locations. Whatever it finds, it validates the same way, so
20
- * broadening discovery can only raise a score, never lower one.
21
- */
22
- export declare const meta: CheckMeta;
23
- /** Pull `service-desc` hrefs out of an RFC 9264 linkset document. */
24
- export declare function serviceDescsFromLinkset(body: string): string[];
25
- export default function check(ctx: CheckContext): Promise<CheckResult>;
26
- //# sourceMappingURL=api-discovery.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"api-discovery.d.ts","sourceRoot":"","sources":["../../src/checks/api-discovery.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAA0B,MAAM,aAAa,CAAC;AAOhG;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,IAAI,EAAE,SAMlB,CAAC;AAyEF,qEAAqE;AACrE,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAW9D;AAWD,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAuD3E"}