ax-audit 2.4.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +39 -17
  3. package/dist/checks/agent-json.d.ts +10 -0
  4. package/dist/checks/agent-json.d.ts.map +1 -1
  5. package/dist/checks/agent-json.js +66 -2
  6. package/dist/checks/agent-json.js.map +1 -1
  7. package/dist/checks/html-rendering.d.ts +21 -0
  8. package/dist/checks/html-rendering.d.ts.map +1 -0
  9. package/dist/checks/html-rendering.js +204 -0
  10. package/dist/checks/html-rendering.js.map +1 -0
  11. package/dist/checks/html-utils.d.ts +37 -0
  12. package/dist/checks/html-utils.d.ts.map +1 -0
  13. package/dist/checks/html-utils.js +93 -0
  14. package/dist/checks/html-utils.js.map +1 -0
  15. package/dist/checks/http-headers.js +1 -1
  16. package/dist/checks/http-headers.js.map +1 -1
  17. package/dist/checks/index.d.ts.map +1 -1
  18. package/dist/checks/index.js +10 -0
  19. package/dist/checks/index.js.map +1 -1
  20. package/dist/checks/llms-txt.d.ts.map +1 -1
  21. package/dist/checks/llms-txt.js +17 -2
  22. package/dist/checks/llms-txt.js.map +1 -1
  23. package/dist/checks/mcp.d.ts.map +1 -1
  24. package/dist/checks/mcp.js +11 -2
  25. package/dist/checks/mcp.js.map +1 -1
  26. package/dist/checks/meta-tags.d.ts +13 -0
  27. package/dist/checks/meta-tags.d.ts.map +1 -1
  28. package/dist/checks/meta-tags.js +104 -24
  29. package/dist/checks/meta-tags.js.map +1 -1
  30. package/dist/checks/openapi.d.ts.map +1 -1
  31. package/dist/checks/openapi.js +11 -2
  32. package/dist/checks/openapi.js.map +1 -1
  33. package/dist/checks/robots-txt.js +1 -1
  34. package/dist/checks/security-txt.js +1 -1
  35. package/dist/checks/seo-basics.d.ts +13 -0
  36. package/dist/checks/seo-basics.d.ts.map +1 -0
  37. package/dist/checks/seo-basics.js +222 -0
  38. package/dist/checks/seo-basics.js.map +1 -0
  39. package/dist/checks/sitemap.d.ts +12 -0
  40. package/dist/checks/sitemap.d.ts.map +1 -0
  41. package/dist/checks/sitemap.js +241 -0
  42. package/dist/checks/sitemap.js.map +1 -0
  43. package/dist/checks/structured-data.js +1 -1
  44. package/dist/checks/structured-data.js.map +1 -1
  45. package/dist/checks/tls-https.d.ts +13 -0
  46. package/dist/checks/tls-https.d.ts.map +1 -0
  47. package/dist/checks/tls-https.js +164 -0
  48. package/dist/checks/tls-https.js.map +1 -0
  49. package/dist/checks/utils.d.ts +13 -1
  50. package/dist/checks/utils.d.ts.map +1 -1
  51. package/dist/checks/utils.js +28 -0
  52. package/dist/checks/utils.js.map +1 -1
  53. package/dist/checks/well-known-ai.d.ts +17 -0
  54. package/dist/checks/well-known-ai.d.ts.map +1 -0
  55. package/dist/checks/well-known-ai.js +123 -0
  56. package/dist/checks/well-known-ai.js.map +1 -0
  57. package/dist/constants.d.ts +19 -0
  58. package/dist/constants.d.ts.map +1 -1
  59. package/dist/constants.js +47 -10
  60. package/dist/constants.js.map +1 -1
  61. package/package.json +1 -1
@@ -0,0 +1,164 @@
1
+ import { guideUrl } from '../guide-urls.js';
2
+ import { buildResult } from './utils.js';
3
+ /**
4
+ * "tls-https" — verify the site is served over HTTPS, that bare HTTP redirects to HTTPS,
5
+ * and that HSTS is properly configured (with `preload` + `includeSubDomains` for full
6
+ * coverage). Many AI agents refuse to fetch plaintext HTTP and downgrade trust on
7
+ * misconfigured TLS.
8
+ *
9
+ * The check intentionally avoids verifying the certificate chain — Node's `fetch` already
10
+ * fails on invalid certificates, so a successful HTTPS fetch is itself a positive signal.
11
+ */
12
+ export const meta = {
13
+ id: 'tls-https',
14
+ name: 'TLS / HTTPS',
15
+ description: 'Checks HTTPS, HTTP→HTTPS redirect, and HSTS configuration',
16
+ weight: 5,
17
+ };
18
+ const HSTS_MIN_MAX_AGE = 15_768_000;
19
+ const HSTS_PRELOAD_MIN_MAX_AGE = 31_536_000;
20
+ export default async function check(ctx) {
21
+ const start = performance.now();
22
+ const findings = [];
23
+ let score = 100;
24
+ let parsed;
25
+ try {
26
+ parsed = new URL(ctx.url);
27
+ }
28
+ catch {
29
+ findings.push({
30
+ status: 'fail',
31
+ message: `Invalid URL: ${ctx.url}`,
32
+ hint: 'Provide a fully qualified URL including scheme (https://...).',
33
+ learnMoreUrl: guideUrl(meta.id, 'invalid-url'),
34
+ });
35
+ return buildResult(meta, 0, findings, start);
36
+ }
37
+ if (parsed.protocol === 'https:') {
38
+ findings.push({ status: 'pass', message: 'Site is served over HTTPS' });
39
+ }
40
+ else {
41
+ findings.push({
42
+ status: 'fail',
43
+ message: `Site is served over ${parsed.protocol.replace(':', '').toUpperCase()} (not HTTPS)`,
44
+ hint: 'Serve the site over HTTPS. Most AI crawlers refuse plaintext HTTP and downgrade trust on insecure origins.',
45
+ learnMoreUrl: guideUrl(meta.id, 'no-https'),
46
+ });
47
+ score -= 50;
48
+ }
49
+ if (parsed.protocol === 'https:') {
50
+ const httpUrl = `http://${parsed.host}${parsed.pathname || ''}`;
51
+ const httpRes = await ctx.fetch(httpUrl);
52
+ const finalUrl = httpRes.url || '';
53
+ if (httpRes.ok && /^https:\/\//i.test(finalUrl)) {
54
+ findings.push({ status: 'pass', message: 'HTTP requests redirect to HTTPS' });
55
+ }
56
+ else if (!httpRes.ok && httpRes.status === 0) {
57
+ findings.push({
58
+ status: 'pass',
59
+ message: 'HTTP endpoint not reachable (HTTPS-only — acceptable)',
60
+ });
61
+ }
62
+ else if (httpRes.ok && /^http:\/\//i.test(finalUrl)) {
63
+ findings.push({
64
+ status: 'fail',
65
+ message: 'HTTP request did not redirect to HTTPS',
66
+ detail: `Final URL: ${finalUrl}`,
67
+ hint: 'Configure your server to 301-redirect every http:// request to https://. Otherwise agents may cache the insecure variant.',
68
+ learnMoreUrl: guideUrl(meta.id, 'no-redirect'),
69
+ });
70
+ score -= 15;
71
+ }
72
+ else {
73
+ findings.push({
74
+ status: 'warn',
75
+ message: 'Could not verify HTTP→HTTPS redirect',
76
+ detail: `HTTP ${httpRes.status} on ${httpUrl}`,
77
+ hint: 'Test manually: a request to http://your-site.com should respond with 301 → https://your-site.com.',
78
+ learnMoreUrl: guideUrl(meta.id, 'redirect-unknown'),
79
+ });
80
+ score -= 5;
81
+ }
82
+ }
83
+ const hsts = ctx.headers['strict-transport-security'];
84
+ if (!hsts) {
85
+ findings.push({
86
+ status: 'warn',
87
+ message: 'No Strict-Transport-Security header',
88
+ hint: 'Add: Strict-Transport-Security: max-age=31536000; includeSubDomains; preload. This locks browsers and many agents to HTTPS for 1 year.',
89
+ learnMoreUrl: guideUrl(meta.id, 'no-hsts'),
90
+ });
91
+ score -= 15;
92
+ }
93
+ else {
94
+ const maxAge = parseHstsMaxAge(hsts);
95
+ const includesSubDomains = /includeSubDomains/i.test(hsts);
96
+ const preload = /preload/i.test(hsts);
97
+ if (maxAge === null) {
98
+ findings.push({
99
+ status: 'warn',
100
+ message: 'HSTS header has no max-age directive',
101
+ detail: hsts,
102
+ hint: 'Set a numeric max-age, e.g., max-age=31536000.',
103
+ learnMoreUrl: guideUrl(meta.id, 'hsts-no-maxage'),
104
+ });
105
+ score -= 10;
106
+ }
107
+ else if (maxAge < HSTS_MIN_MAX_AGE) {
108
+ findings.push({
109
+ status: 'warn',
110
+ message: `HSTS max-age is short (${maxAge}s = ~${Math.round(maxAge / 86400)} days)`,
111
+ hint: `Use at least max-age=${HSTS_MIN_MAX_AGE} (~6 months); preload list submission requires max-age=${HSTS_PRELOAD_MIN_MAX_AGE} (1 year).`,
112
+ learnMoreUrl: guideUrl(meta.id, 'hsts-short'),
113
+ });
114
+ score -= 5;
115
+ }
116
+ else {
117
+ findings.push({ status: 'pass', message: `HSTS max-age=${maxAge}` });
118
+ }
119
+ if (includesSubDomains) {
120
+ findings.push({ status: 'pass', message: 'HSTS includes subdomains' });
121
+ }
122
+ else {
123
+ findings.push({
124
+ status: 'warn',
125
+ message: 'HSTS does not include subdomains',
126
+ hint: 'Add includeSubDomains to apply HSTS across api., docs., etc. Required for preload list submission.',
127
+ learnMoreUrl: guideUrl(meta.id, 'hsts-no-subdomains'),
128
+ });
129
+ score -= 5;
130
+ }
131
+ if (preload) {
132
+ if (maxAge !== null && maxAge >= HSTS_PRELOAD_MIN_MAX_AGE && includesSubDomains) {
133
+ findings.push({ status: 'pass', message: 'HSTS preload-eligible' });
134
+ }
135
+ else {
136
+ findings.push({
137
+ status: 'warn',
138
+ message: 'HSTS has preload directive but does not satisfy preload-list requirements',
139
+ hint: `Preload requires max-age >= ${HSTS_PRELOAD_MIN_MAX_AGE} and includeSubDomains. See https://hstspreload.org.`,
140
+ learnMoreUrl: guideUrl(meta.id, 'hsts-preload-invalid'),
141
+ });
142
+ score -= 5;
143
+ }
144
+ }
145
+ else {
146
+ findings.push({
147
+ status: 'warn',
148
+ message: 'HSTS lacks the preload directive',
149
+ hint: 'Add preload (and ensure max-age >= 31536000 + includeSubDomains) and submit the domain at https://hstspreload.org for browser-built-in HTTPS enforcement.',
150
+ learnMoreUrl: guideUrl(meta.id, 'hsts-no-preload'),
151
+ });
152
+ score -= 3;
153
+ }
154
+ }
155
+ return buildResult(meta, score, findings, start);
156
+ }
157
+ function parseHstsMaxAge(value) {
158
+ const m = value.match(/max-age\s*=\s*"?(\d+)"?/i);
159
+ if (!m)
160
+ return null;
161
+ const n = Number(m[1]);
162
+ return Number.isFinite(n) ? n : null;
163
+ }
164
+ //# sourceMappingURL=tls-https.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tls-https.js","sourceRoot":"","sources":["../../src/checks/tls-https.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,WAAW;IACf,IAAI,EAAE,aAAa;IACnB,WAAW,EAAE,2DAA2D;IACxE,MAAM,EAAE,CAAC;CACV,CAAC;AAEF,MAAM,gBAAgB,GAAG,UAAU,CAAC;AACpC,MAAM,wBAAwB,GAAG,UAAU,CAAC;AAE5C,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAG,GAAG,CAAC;IAEhB,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,gBAAgB,GAAG,CAAC,GAAG,EAAE;YAClC,IAAI,EAAE,+DAA+D;YACrE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC;SAC/C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC,CAAC;IAC1E,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uBAAuB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,cAAc;YAC5F,IAAI,EAAE,4GAA4G;YAClH,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC;SAC5C,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,UAAU,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QAChE,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;QACnC,IAAI,OAAO,CAAC,EAAE,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/C,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,uDAAuD;aACjE,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,OAAO,CAAC,EAAE,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtD,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,wCAAwC;gBACjD,MAAM,EAAE,cAAc,QAAQ,EAAE;gBAChC,IAAI,EAAE,2HAA2H;gBACjI,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC;aAC/C,CAAC,CAAC;YACH,KAAK,IAAI,EAAE,CAAC;QACd,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,sCAAsC;gBAC/C,MAAM,EAAE,QAAQ,OAAO,CAAC,MAAM,OAAO,OAAO,EAAE;gBAC9C,IAAI,EAAE,mGAAmG;gBACzG,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,kBAAkB,CAAC;aACpD,CAAC,CAAC;YACH,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;IACtD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,qCAAqC;YAC9C,IAAI,EAAE,wIAAwI;YAC9I,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC;SAC3C,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEtC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACpB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,sCAAsC;gBAC/C,MAAM,EAAE,IAAI;gBACZ,IAAI,EAAE,gDAAgD;gBACtD,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC;aAClD,CAAC,CAAC;YACH,KAAK,IAAI,EAAE,CAAC;QACd,CAAC;aAAM,IAAI,MAAM,GAAG,gBAAgB,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,0BAA0B,MAAM,QAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ;gBACnF,IAAI,EAAE,wBAAwB,gBAAgB,0DAA0D,wBAAwB,YAAY;gBAC5I,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC;aAC9C,CAAC,CAAC;YACH,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,MAAM,EAAE,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,IAAI,kBAAkB,EAAE,CAAC;YACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,0BAA0B,EAAE,CAAC,CAAC;QACzE,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,kCAAkC;gBAC3C,IAAI,EAAE,oGAAoG;gBAC1G,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,CAAC;aACtD,CAAC,CAAC;YACH,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,IAAI,wBAAwB,IAAI,kBAAkB,EAAE,CAAC;gBAChF,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;YACtE,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC;oBACZ,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,2EAA2E;oBACpF,IAAI,EAAE,+BAA+B,wBAAwB,sDAAsD;oBACnH,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,sBAAsB,CAAC;iBACxD,CAAC,CAAC;gBACH,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;QACH,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,kCAAkC;gBAC3C,IAAI,EAAE,2JAA2J;gBACjK,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,iBAAiB,CAAC;aACnD,CAAC,CAAC;YACH,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAClD,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC"}
@@ -1,4 +1,16 @@
1
- import type { CheckMeta, CheckResult, Finding } from '../types.js';
1
+ import type { CheckMeta, CheckResult, FetchResponse, Finding } from '../types.js';
2
2
  export declare function clampScore(score: number): number;
3
3
  export declare function buildResult(meta: CheckMeta, score: number, findings: Finding[], start: number): CheckResult;
4
+ /**
5
+ * Validate the `Content-Type` of a fetched resource against a list of acceptable MIME types.
6
+ *
7
+ * Returns:
8
+ * - `null` when the content type is acceptable (no finding to add)
9
+ * - a `Finding` describing the mismatch otherwise (caller decides whether to apply a score penalty)
10
+ */
11
+ export declare function checkContentType(res: FetchResponse, expected: string[], context: {
12
+ checkId: string;
13
+ resourceLabel: string;
14
+ anchor: string;
15
+ }): Finding | null;
4
16
  //# sourceMappingURL=utils.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/checks/utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEnE,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,CAS3G"}
1
+ {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/checks/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAElF,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,CAS3G;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,aAAa,EAClB,QAAQ,EAAE,MAAM,EAAE,EAClB,OAAO,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAClE,OAAO,GAAG,IAAI,CAkBhB"}
@@ -1,3 +1,4 @@
1
+ import { guideUrl } from '../guide-urls.js';
1
2
  export function clampScore(score) {
2
3
  return Math.max(0, Math.min(100, score));
3
4
  }
@@ -11,4 +12,31 @@ export function buildResult(meta, score, findings, start) {
11
12
  duration: Math.round(performance.now() - start),
12
13
  };
13
14
  }
15
+ /**
16
+ * Validate the `Content-Type` of a fetched resource against a list of acceptable MIME types.
17
+ *
18
+ * Returns:
19
+ * - `null` when the content type is acceptable (no finding to add)
20
+ * - a `Finding` describing the mismatch otherwise (caller decides whether to apply a score penalty)
21
+ */
22
+ export function checkContentType(res, expected, context) {
23
+ const ct = (res.headers['content-type'] ?? '').toLowerCase();
24
+ if (!ct) {
25
+ return {
26
+ status: 'warn',
27
+ message: `${context.resourceLabel} has no Content-Type header`,
28
+ hint: `Serve ${context.resourceLabel} with one of: ${expected.join(', ')}.`,
29
+ learnMoreUrl: guideUrl(context.checkId, context.anchor),
30
+ };
31
+ }
32
+ if (expected.some((mime) => ct.includes(mime)))
33
+ return null;
34
+ return {
35
+ status: 'warn',
36
+ message: `${context.resourceLabel} Content-Type is "${ct.split(';')[0]}"`,
37
+ detail: `Expected one of: ${expected.join(', ')}`,
38
+ hint: `Configure your server to serve ${context.resourceLabel} as ${expected[0]} so AI agents parse it correctly.`,
39
+ learnMoreUrl: guideUrl(context.checkId, context.anchor),
40
+ };
41
+ }
14
42
  //# sourceMappingURL=utils.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/checks/utils.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAe,EAAE,KAAa,EAAE,QAAmB,EAAE,KAAa;IAC5F,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC;QACxB,QAAQ;QACR,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;KAChD,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/checks/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAG5C,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAe,EAAE,KAAa,EAAE,QAAmB,EAAE,KAAa;IAC5F,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC;QACxB,QAAQ;QACR,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;KAChD,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,GAAkB,EAClB,QAAkB,EAClB,OAAmE;IAEnE,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7D,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,OAAO;YACL,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,OAAO,CAAC,aAAa,6BAA6B;YAC9D,IAAI,EAAE,SAAS,OAAO,CAAC,aAAa,iBAAiB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YAC3E,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;SACxD,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5D,OAAO;QACL,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,GAAG,OAAO,CAAC,aAAa,qBAAqB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;QACzE,MAAM,EAAE,oBAAoB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACjD,IAAI,EAAE,kCAAkC,OAAO,CAAC,aAAa,OAAO,QAAQ,CAAC,CAAC,CAAC,mCAAmC;QAClH,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;KACxD,CAAC;AACJ,CAAC"}
@@ -0,0 +1,17 @@
1
+ import type { CheckContext, CheckResult, CheckMeta } from '../types.js';
2
+ /**
3
+ * "well-known-ai" — emerging AI-specific discovery files that don't have a settled spec yet
4
+ * but are already published by some sites and read by some agents. Each file present is a
5
+ * positive signal (bonus); none of them are individually required, so a site that publishes
6
+ * just one or two should not be punished. The score reflects coverage of this small bundle.
7
+ *
8
+ * Files probed:
9
+ * - `/.well-known/ai.txt` (Spawning AI — opt-in/opt-out for AI training)
10
+ * - `/.well-known/genai.txt` (proposed — generative AI policy)
11
+ * - `/ai-plugin.json` and `/.well-known/ai-plugin.json` (legacy ChatGPT plugin manifest)
12
+ * - `/agents.json` (Wildcard / OpenAgents — emerging agent capability manifest)
13
+ * - `/.well-known/nlweb.json` (Microsoft NLWeb — natural-language site interface)
14
+ */
15
+ export declare const meta: CheckMeta;
16
+ export default function check(ctx: CheckContext): Promise<CheckResult>;
17
+ //# sourceMappingURL=well-known-ai.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"well-known-ai.d.ts","sourceRoot":"","sources":["../../src/checks/well-known-ai.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAA0B,MAAM,aAAa,CAAC;AAGhG;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;AAmEF,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA+C3E"}
@@ -0,0 +1,123 @@
1
+ import { guideUrl } from '../guide-urls.js';
2
+ import { buildResult } from './utils.js';
3
+ /**
4
+ * "well-known-ai" — emerging AI-specific discovery files that don't have a settled spec yet
5
+ * but are already published by some sites and read by some agents. Each file present is a
6
+ * positive signal (bonus); none of them are individually required, so a site that publishes
7
+ * just one or two should not be punished. The score reflects coverage of this small bundle.
8
+ *
9
+ * Files probed:
10
+ * - `/.well-known/ai.txt` (Spawning AI — opt-in/opt-out for AI training)
11
+ * - `/.well-known/genai.txt` (proposed — generative AI policy)
12
+ * - `/ai-plugin.json` and `/.well-known/ai-plugin.json` (legacy ChatGPT plugin manifest)
13
+ * - `/agents.json` (Wildcard / OpenAgents — emerging agent capability manifest)
14
+ * - `/.well-known/nlweb.json` (Microsoft NLWeb — natural-language site interface)
15
+ */
16
+ export const meta = {
17
+ id: 'well-known-ai',
18
+ name: 'AI Well-Known',
19
+ description: 'Checks emerging AI-specific discovery files (ai.txt, agents.json, ai-plugin.json, nlweb)',
20
+ weight: 3,
21
+ };
22
+ const PROBES = [
23
+ {
24
+ paths: ['/.well-known/ai.txt'],
25
+ label: 'ai.txt',
26
+ hint: 'Publish /.well-known/ai.txt declaring opt-in/opt-out signals for AI training. See https://site.spawning.ai/spawning-ai-txt for the format.',
27
+ guideAnchor: 'ai-txt',
28
+ },
29
+ {
30
+ paths: ['/.well-known/genai.txt'],
31
+ label: 'genai.txt',
32
+ hint: 'Publish /.well-known/genai.txt declaring your generative-AI usage policy.',
33
+ guideAnchor: 'genai-txt',
34
+ },
35
+ {
36
+ paths: ['/.well-known/ai-plugin.json', '/ai-plugin.json'],
37
+ label: 'ai-plugin.json',
38
+ hint: 'Publish /ai-plugin.json (legacy ChatGPT plugin manifest). Schema: name_for_model, description_for_model, api.url. Still consumed by some agents.',
39
+ guideAnchor: 'ai-plugin',
40
+ parser: (body) => {
41
+ try {
42
+ const data = JSON.parse(body);
43
+ return Boolean(data.name_for_model || data.name_for_human || data.schema_version);
44
+ }
45
+ catch {
46
+ return false;
47
+ }
48
+ },
49
+ },
50
+ {
51
+ paths: ['/agents.json', '/.well-known/agents.json'],
52
+ label: 'agents.json',
53
+ hint: 'Publish /agents.json describing your site as a callable agent (OpenAgents / Wildcard emerging spec). Includes name, description, and operations[].',
54
+ guideAnchor: 'agents-json',
55
+ parser: (body) => {
56
+ try {
57
+ const data = JSON.parse(body);
58
+ return Boolean(data.name || data.operations || data.agents);
59
+ }
60
+ catch {
61
+ return false;
62
+ }
63
+ },
64
+ },
65
+ {
66
+ paths: ['/.well-known/nlweb.json', '/nlweb.json'],
67
+ label: 'nlweb.json',
68
+ hint: 'Publish /.well-known/nlweb.json (Microsoft NLWeb) so agents can interact with the site through a natural-language interface.',
69
+ guideAnchor: 'nlweb',
70
+ parser: (body) => {
71
+ try {
72
+ JSON.parse(body);
73
+ return true;
74
+ }
75
+ catch {
76
+ return false;
77
+ }
78
+ },
79
+ },
80
+ ];
81
+ export default async function check(ctx) {
82
+ const start = performance.now();
83
+ const findings = [];
84
+ let presentCount = 0;
85
+ for (const probe of PROBES) {
86
+ let hit = null;
87
+ for (const path of probe.paths) {
88
+ const res = await ctx.fetch(`${ctx.url}${path}`);
89
+ if (res.ok && res.body.trim().length > 0) {
90
+ hit = { path, res };
91
+ break;
92
+ }
93
+ }
94
+ if (!hit) {
95
+ findings.push({
96
+ status: 'warn',
97
+ message: `${probe.label} not found`,
98
+ detail: `Tried: ${probe.paths.join(', ')}`,
99
+ hint: probe.hint,
100
+ learnMoreUrl: guideUrl(meta.id, probe.guideAnchor),
101
+ });
102
+ continue;
103
+ }
104
+ if (probe.parser && !probe.parser(hit.res.body)) {
105
+ findings.push({
106
+ status: 'warn',
107
+ message: `${probe.label} present at ${hit.path} but does not look valid`,
108
+ hint: probe.hint,
109
+ learnMoreUrl: guideUrl(meta.id, `${probe.guideAnchor}-invalid`),
110
+ });
111
+ continue;
112
+ }
113
+ presentCount++;
114
+ findings.push({ status: 'pass', message: `${probe.label} present at ${hit.path}` });
115
+ }
116
+ const score = Math.round((presentCount / PROBES.length) * 100);
117
+ findings.unshift({
118
+ status: presentCount > 0 ? 'pass' : 'warn',
119
+ message: `${presentCount}/${PROBES.length} emerging AI discovery files published`,
120
+ });
121
+ return buildResult(meta, score, findings, start);
122
+ }
123
+ //# sourceMappingURL=well-known-ai.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"well-known-ai.js","sourceRoot":"","sources":["../../src/checks/well-known-ai.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,eAAe;IACnB,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,0FAA0F;IACvG,MAAM,EAAE,CAAC;CACV,CAAC;AAUF,MAAM,MAAM,GAAY;IACtB;QACE,KAAK,EAAE,CAAC,qBAAqB,CAAC;QAC9B,KAAK,EAAE,QAAQ;QACf,IAAI,EAAE,4IAA4I;QAClJ,WAAW,EAAE,QAAQ;KACtB;IACD;QACE,KAAK,EAAE,CAAC,wBAAwB,CAAC;QACjC,KAAK,EAAE,WAAW;QAClB,IAAI,EAAE,2EAA2E;QACjF,WAAW,EAAE,WAAW;KACzB;IACD;QACE,KAAK,EAAE,CAAC,6BAA6B,EAAE,iBAAiB,CAAC;QACzD,KAAK,EAAE,gBAAgB;QACvB,IAAI,EAAE,kJAAkJ;QACxJ,WAAW,EAAE,WAAW;QACxB,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACf,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;gBACzD,OAAO,OAAO,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC,CAAC;YACpF,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;KACF;IACD;QACE,KAAK,EAAE,CAAC,cAAc,EAAE,0BAA0B,CAAC;QACnD,KAAK,EAAE,aAAa;QACpB,IAAI,EAAE,oJAAoJ;QAC1J,WAAW,EAAE,aAAa;QAC1B,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACf,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;gBACzD,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9D,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;KACF;IACD;QACE,KAAK,EAAE,CAAC,yBAAyB,EAAE,aAAa,CAAC;QACjD,KAAK,EAAE,YAAY;QACnB,IAAI,EAAE,8HAA8H;QACpI,WAAW,EAAE,OAAO;QACpB,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACf,IAAI,CAAC;gBACH,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjB,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;KACF;CACF,CAAC;AAEF,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAgD,IAAI,CAAC;QAC5D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;YACjD,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzC,GAAG,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;gBACpB,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,YAAY;gBACnC,MAAM,EAAE,UAAU,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC1C,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,WAAW,CAAC;aACnD,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,eAAe,GAAG,CAAC,IAAI,0BAA0B;gBACxE,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,WAAW,UAAU,CAAC;aAChE,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,YAAY,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,eAAe,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;IAC/D,QAAQ,CAAC,OAAO,CAAC;QACf,MAAM,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QAC1C,OAAO,EAAE,GAAG,YAAY,IAAI,MAAM,CAAC,MAAM,wCAAwC;KAClF,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC"}
@@ -1,9 +1,28 @@
1
1
  import type { Grade, SecurityHeader } from './types.js';
2
2
  export declare const VERSION: string;
3
3
  export declare const USER_AGENT: string;
4
+ /**
5
+ * Known AI / LLM crawlers grouped by primary purpose. The audit recommends explicit
6
+ * `User-agent:` rules for every entry so site operators can see exactly which agents
7
+ * have access. Buckets:
8
+ *
9
+ * - `training` — bots that scrape content for model training corpora
10
+ * - `search` — bots that fetch on behalf of a live answer engine / search UI
11
+ * - `fetching` — generic on-demand fetchers (browsing, summarization, automation)
12
+ */
4
13
  export declare const AI_CRAWLERS: Record<string, string[]>;
5
14
  export declare const ALL_AI_CRAWLERS: string[];
15
+ /**
16
+ * The "must-configure" subset — these are the agents a typical operator should grant
17
+ * (or knowingly deny) explicit access to. ax-audit grades the robots.txt section heavily
18
+ * on coverage of this short list, while still rewarding broader explicit rules.
19
+ */
6
20
  export declare const CORE_AI_CRAWLERS: string[];
21
+ /**
22
+ * Default weight per check (sum: 100). Individual `CheckMeta.weight` overrides this map.
23
+ * Keep weights aligned with real-world impact — discovery + content-rendering are the
24
+ * highest-leverage signals for AI agents.
25
+ */
7
26
  export declare const CHECK_WEIGHTS: Record<string, number>;
8
27
  export declare const GRADES: Grade[];
9
28
  export declare const AGENT_JSON_REQUIRED_FIELDS: string[];
@@ -1 +1 @@
1
- {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAKxD,eAAO,MAAM,OAAO,EAAE,MAAoB,CAAC;AAC3C,eAAO,MAAM,UAAU,QAAqE,CAAC;AAE7F,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAmChD,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAAM,EAA8E,CAAC;AAEnH,eAAO,MAAM,gBAAgB,EAAE,MAAM,EAOpC,CAAC;AAEF,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAUhD,CAAC;AAEF,eAAO,MAAM,MAAM,EAAE,KAAK,EAKzB,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,MAAM,EAA6C,CAAC;AAE7F,eAAO,MAAM,4BAA4B,EAAE,MAAM,EAA2B,CAAC;AAE7E,eAAO,MAAM,gBAAgB,EAAE,cAAc,EAQ5C,CAAC"}
1
+ {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAKxD,eAAO,MAAM,OAAO,EAAE,MAAoB,CAAC;AAC3C,eAAO,MAAM,UAAU,QAAqE,CAAC;AAE7F;;;;;;;;GAQG;AACH,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CA8ChD,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAAM,EAA8E,CAAC;AAEnH;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,EAAE,MAAM,EASpC,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAehD,CAAC;AAEF,eAAO,MAAM,MAAM,EAAE,KAAK,EAKzB,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,MAAM,EAA6C,CAAC;AAE7F,eAAO,MAAM,4BAA4B,EAAE,MAAM,EAA2B,CAAC;AAE7E,eAAO,MAAM,gBAAgB,EAAE,cAAc,EAQ5C,CAAC"}
package/dist/constants.js CHANGED
@@ -5,6 +5,15 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
5
5
  const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf-8'));
6
6
  export const VERSION = pkg.version;
7
7
  export const USER_AGENT = `ax-audit/${pkg.version} (https://github.com/lucioduran/ax-audit)`;
8
+ /**
9
+ * Known AI / LLM crawlers grouped by primary purpose. The audit recommends explicit
10
+ * `User-agent:` rules for every entry so site operators can see exactly which agents
11
+ * have access. Buckets:
12
+ *
13
+ * - `training` — bots that scrape content for model training corpora
14
+ * - `search` — bots that fetch on behalf of a live answer engine / search UI
15
+ * - `fetching` — generic on-demand fetchers (browsing, summarization, automation)
16
+ */
8
17
  export const AI_CRAWLERS = {
9
18
  training: [
10
19
  'GPTBot',
@@ -25,6 +34,13 @@ export const AI_CRAWLERS = {
25
34
  'DeepSeek-AI',
26
35
  'PanguBot',
27
36
  'Diffbot',
37
+ 'MistralAI-User',
38
+ 'Kangaroo Bot',
39
+ 'Timpibot',
40
+ 'omgili',
41
+ 'omgilibot',
42
+ 'ImagesiftBot',
43
+ 'Webzio-Extended',
28
44
  ],
29
45
  search: [
30
46
  'OAI-SearchBot',
@@ -38,10 +54,19 @@ export const AI_CRAWLERS = {
38
54
  'Petalbot',
39
55
  'Google-CloudVertexBot',
40
56
  'Gemini',
57
+ 'GeminiBot',
58
+ 'KagiBot',
59
+ 'NeevaBot',
60
+ 'PhindBot',
41
61
  ],
42
- fetching: ['FirecrawlAgent', 'Facebookbot'],
62
+ fetching: ['FirecrawlAgent', 'Facebookbot', 'Bingbot', 'Goose', 'AwarioBot', 'AwarioRssBot', 'AwarioSmartBot'],
43
63
  };
44
64
  export const ALL_AI_CRAWLERS = [...AI_CRAWLERS.training, ...AI_CRAWLERS.search, ...AI_CRAWLERS.fetching];
65
+ /**
66
+ * The "must-configure" subset — these are the agents a typical operator should grant
67
+ * (or knowingly deny) explicit access to. ax-audit grades the robots.txt section heavily
68
+ * on coverage of this short list, while still rewarding broader explicit rules.
69
+ */
45
70
  export const CORE_AI_CRAWLERS = [
46
71
  'GPTBot',
47
72
  'ClaudeBot',
@@ -49,17 +74,29 @@ export const CORE_AI_CRAWLERS = [
49
74
  'Claude-SearchBot',
50
75
  'Google-Extended',
51
76
  'PerplexityBot',
77
+ 'OAI-SearchBot',
78
+ 'CCBot',
52
79
  ];
80
+ /**
81
+ * Default weight per check (sum: 100). Individual `CheckMeta.weight` overrides this map.
82
+ * Keep weights aligned with real-world impact — discovery + content-rendering are the
83
+ * highest-leverage signals for AI agents.
84
+ */
53
85
  export const CHECK_WEIGHTS = {
54
- 'llms-txt': 15,
55
- 'robots-txt': 15,
56
- 'structured-data': 13,
57
- 'http-headers': 13,
58
- 'agent-json': 10,
59
- mcp: 10,
60
- 'security-txt': 8,
61
- 'meta-tags': 8,
62
- openapi: 8,
86
+ 'llms-txt': 11,
87
+ 'robots-txt': 11,
88
+ 'html-rendering': 9,
89
+ 'structured-data': 9,
90
+ 'http-headers': 9,
91
+ 'agent-json': 7,
92
+ mcp: 7,
93
+ 'seo-basics': 7,
94
+ 'security-txt': 6,
95
+ 'meta-tags': 6,
96
+ openapi: 6,
97
+ 'tls-https': 5,
98
+ sitemap: 4,
99
+ 'well-known-ai': 3,
63
100
  };
64
101
  export const GRADES = [
65
102
  { min: 90, label: 'Excellent', color: 'green' },
@@ -1 +1 @@
1
- {"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AAErF,MAAM,CAAC,MAAM,OAAO,GAAW,GAAG,CAAC,OAAO,CAAC;AAC3C,MAAM,CAAC,MAAM,UAAU,GAAG,YAAY,GAAG,CAAC,OAAO,2CAA2C,CAAC;AAE7F,MAAM,CAAC,MAAM,WAAW,GAA6B;IACnD,QAAQ,EAAE;QACR,QAAQ;QACR,WAAW;QACX,YAAY;QACZ,cAAc;QACd,iBAAiB;QACjB,OAAO;QACP,YAAY;QACZ,oBAAoB;QACpB,sBAAsB;QACtB,WAAW;QACX,8BAA8B;QAC9B,mBAAmB;QACnB,WAAW;QACX,QAAQ;QACR,cAAc;QACd,aAAa;QACb,UAAU;QACV,SAAS;KACV;IACD,MAAM,EAAE;QACN,eAAe;QACf,cAAc;QACd,kBAAkB;QAClB,aAAa;QACb,eAAe;QACf,iBAAiB;QACjB,eAAe;QACf,QAAQ;QACR,UAAU;QACV,uBAAuB;QACvB,QAAQ;KACT;IACD,QAAQ,EAAE,CAAC,gBAAgB,EAAE,aAAa,CAAC;CAC5C,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAa,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAEnH,MAAM,CAAC,MAAM,gBAAgB,GAAa;IACxC,QAAQ;IACR,WAAW;IACX,cAAc;IACd,kBAAkB;IAClB,iBAAiB;IACjB,eAAe;CAChB,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAA2B;IACnD,UAAU,EAAE,EAAE;IACd,YAAY,EAAE,EAAE;IAChB,iBAAiB,EAAE,EAAE;IACrB,cAAc,EAAE,EAAE;IAClB,YAAY,EAAE,EAAE;IAChB,GAAG,EAAE,EAAE;IACP,cAAc,EAAE,CAAC;IACjB,WAAW,EAAE,CAAC;IACd,OAAO,EAAE,CAAC;CACX,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAY;IAC7B,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE;IAC/C,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;IAC3C,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;IAC3C,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE;CACxC,CAAC;AAEF,MAAM,CAAC,MAAM,0BAA0B,GAAa,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;AAE7F,MAAM,CAAC,MAAM,4BAA4B,GAAa,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,gBAAgB,GAAqB;IAChD,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,2BAA2B,EAAE,QAAQ,EAAE,IAAI,EAAE;IACzF,EAAE,IAAI,EAAE,wBAAwB,EAAE,KAAK,EAAE,wBAAwB,EAAE,QAAQ,EAAE,IAAI,EAAE;IACnF,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,EAAE;IACtE,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,kBAAkB,EAAE,QAAQ,EAAE,KAAK,EAAE;IACxE,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,EAAE;IACtE,EAAE,IAAI,EAAE,oBAAoB,EAAE,KAAK,EAAE,oBAAoB,EAAE,QAAQ,EAAE,KAAK,EAAE;IAC5E,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,yBAAyB,EAAE,QAAQ,EAAE,KAAK,EAAE;CACvF,CAAC"}
1
+ {"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AAErF,MAAM,CAAC,MAAM,OAAO,GAAW,GAAG,CAAC,OAAO,CAAC;AAC3C,MAAM,CAAC,MAAM,UAAU,GAAG,YAAY,GAAG,CAAC,OAAO,2CAA2C,CAAC;AAE7F;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,WAAW,GAA6B;IACnD,QAAQ,EAAE;QACR,QAAQ;QACR,WAAW;QACX,YAAY;QACZ,cAAc;QACd,iBAAiB;QACjB,OAAO;QACP,YAAY;QACZ,oBAAoB;QACpB,sBAAsB;QACtB,WAAW;QACX,8BAA8B;QAC9B,mBAAmB;QACnB,WAAW;QACX,QAAQ;QACR,cAAc;QACd,aAAa;QACb,UAAU;QACV,SAAS;QACT,gBAAgB;QAChB,cAAc;QACd,UAAU;QACV,QAAQ;QACR,WAAW;QACX,cAAc;QACd,iBAAiB;KAClB;IACD,MAAM,EAAE;QACN,eAAe;QACf,cAAc;QACd,kBAAkB;QAClB,aAAa;QACb,eAAe;QACf,iBAAiB;QACjB,eAAe;QACf,QAAQ;QACR,UAAU;QACV,uBAAuB;QACvB,QAAQ;QACR,WAAW;QACX,SAAS;QACT,UAAU;QACV,UAAU;KACX;IACD,QAAQ,EAAE,CAAC,gBAAgB,EAAE,aAAa,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,gBAAgB,CAAC;CAC/G,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAa,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAEnH;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAa;IACxC,QAAQ;IACR,WAAW;IACX,cAAc;IACd,kBAAkB;IAClB,iBAAiB;IACjB,eAAe;IACf,eAAe;IACf,OAAO;CACR,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAA2B;IACnD,UAAU,EAAE,EAAE;IACd,YAAY,EAAE,EAAE;IAChB,gBAAgB,EAAE,CAAC;IACnB,iBAAiB,EAAE,CAAC;IACpB,cAAc,EAAE,CAAC;IACjB,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,CAAC;IACN,YAAY,EAAE,CAAC;IACf,cAAc,EAAE,CAAC;IACjB,WAAW,EAAE,CAAC;IACd,OAAO,EAAE,CAAC;IACV,WAAW,EAAE,CAAC;IACd,OAAO,EAAE,CAAC;IACV,eAAe,EAAE,CAAC;CACnB,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAY;IAC7B,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE;IAC/C,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;IAC3C,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;IAC3C,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE;CACxC,CAAC;AAEF,MAAM,CAAC,MAAM,0BAA0B,GAAa,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;AAE7F,MAAM,CAAC,MAAM,4BAA4B,GAAa,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,gBAAgB,GAAqB;IAChD,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,2BAA2B,EAAE,QAAQ,EAAE,IAAI,EAAE;IACzF,EAAE,IAAI,EAAE,wBAAwB,EAAE,KAAK,EAAE,wBAAwB,EAAE,QAAQ,EAAE,IAAI,EAAE;IACnF,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,EAAE;IACtE,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,kBAAkB,EAAE,QAAQ,EAAE,KAAK,EAAE;IACxE,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,EAAE;IACtE,EAAE,IAAI,EAAE,oBAAoB,EAAE,KAAK,EAAE,oBAAoB,EAAE,QAAQ,EAAE,KAAK,EAAE;IAC5E,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,yBAAyB,EAAE,QAAQ,EAAE,KAAK,EAAE;CACvF,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ax-audit",
3
- "version": "2.4.0",
3
+ "version": "3.0.0",
4
4
  "description": "Audit websites for AI Agent Experience (AX) readiness. Lighthouse for AI Agents.",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",