@nebulacomponents/citable 0.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +8 -2
- package/dist/universal/.agents/skills/citable/VERSION +1 -1
- package/dist/universal/.agents/skills/citable/manifest.json +5 -5
- package/dist/universal/.agents/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.claude/skills/citable/VERSION +1 -1
- package/dist/universal/.claude/skills/citable/manifest.json +5 -5
- package/dist/universal/.claude/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.cursor/skills/citable/VERSION +1 -1
- package/dist/universal/.cursor/skills/citable/manifest.json +5 -5
- package/dist/universal/.cursor/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.gemini/skills/citable/VERSION +1 -1
- package/dist/universal/.gemini/skills/citable/manifest.json +5 -5
- package/dist/universal/.gemini/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.github/skills/citable/VERSION +1 -1
- package/dist/universal/.github/skills/citable/manifest.json +5 -5
- package/dist/universal/.github/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.kiro/skills/citable/VERSION +1 -1
- package/dist/universal/.kiro/skills/citable/manifest.json +5 -5
- package/dist/universal/.kiro/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.opencode/skills/citable/VERSION +1 -1
- package/dist/universal/.opencode/skills/citable/manifest.json +5 -5
- package/dist/universal/.opencode/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.pi/agent/skills/citable/VERSION +1 -1
- package/dist/universal/.pi/agent/skills/citable/manifest.json +5 -5
- package/dist/universal/.pi/agent/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.qoder/skills/citable/VERSION +1 -1
- package/dist/universal/.qoder/skills/citable/manifest.json +5 -5
- package/dist/universal/.qoder/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.rovodev/skills/citable/VERSION +1 -1
- package/dist/universal/.rovodev/skills/citable/manifest.json +5 -5
- package/dist/universal/.rovodev/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.trae/skills/citable/VERSION +1 -1
- package/dist/universal/.trae/skills/citable/manifest.json +5 -5
- package/dist/universal/.trae/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/.trae-cn/skills/citable/VERSION +1 -1
- package/dist/universal/.trae-cn/skills/citable/manifest.json +5 -5
- package/dist/universal/.trae-cn/skills/citable/schemas/finding.schema.json +253 -50
- package/dist/universal/manifest.json +51 -51
- package/package.json +4 -1
- package/schemas/finding.schema.json +253 -50
- package/src/cli/index.js +7 -0
- package/src/commands/selfUpgrade.js +146 -0
- package/src/crawler/fetch.js +42 -9
- package/src/detectors/agent.js +495 -0
- package/src/detectors/cwv.js +171 -0
- package/src/detectors/framework.js +1 -1
- package/src/detectors/hreflang.js +164 -0
- package/src/detectors/index.js +4 -1
- package/src/installer/index.js +7 -2
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AGENT namespace — Agent-readiness detectors
|
|
3
|
+
*
|
|
4
|
+
* Checks whether a site is discoverable, accessible, and interoperable
|
|
5
|
+
* with AI agents and autonomous systems. Based on the checks at
|
|
6
|
+
* https://isitagentready.com/ covering:
|
|
7
|
+
* - Discoverability (robots.txt AI rules, sitemaps, Link headers)
|
|
8
|
+
* - Content Accessibility (Markdown negotiation, llms.txt)
|
|
9
|
+
* - Bot Access Control (Web Bot Auth, Content Signals)
|
|
10
|
+
* - Protocol Discovery (MCP Server Card, A2A Agent Card, Auth.md)
|
|
11
|
+
* - Agentic Commerce (x402, MPP, UCP, ACP)
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { defineDetector } from './framework.js';
|
|
15
|
+
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
// Internal helpers
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
|
|
20
|
+
function pageHeaders(page) {
|
|
21
|
+
return page?.responseHeaders || page?.headers || {};
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function headerValue(headers, name) {
|
|
25
|
+
if (!headers) return null;
|
|
26
|
+
const lower = name.toLowerCase();
|
|
27
|
+
for (const [k, v] of Object.entries(headers)) {
|
|
28
|
+
if (k.toLowerCase() === lower) return v;
|
|
29
|
+
}
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function homepageMeta(ctx) {
|
|
34
|
+
if (!ctx.site?.pages?.length) return null;
|
|
35
|
+
const base = (ctx.config?.site?.base_url || '').replace(/\/$/, '');
|
|
36
|
+
return (
|
|
37
|
+
ctx.site.pages.find((p) => p.url === base || p.url === base + '/' || p.path === '/' || p.path === '') ||
|
|
38
|
+
ctx.site.pages[0]
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function siteUrl(ctx) {
|
|
43
|
+
return ctx.config?.site?.base_url || 'site';
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// AGENT-001: AI bot rules in robots.txt
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
export const AGENT_001 = defineDetector({
|
|
51
|
+
id: 'AGENT-001',
|
|
52
|
+
name: 'AI bot rules absent from robots.txt',
|
|
53
|
+
namespace: 'AGENT',
|
|
54
|
+
discipline: ['agent-readiness'],
|
|
55
|
+
severity: 'medium',
|
|
56
|
+
deterministic: true,
|
|
57
|
+
description:
|
|
58
|
+
'robots.txt contains no rules for known AI crawlers (GPTBot, ClaudeBot, PerplexityBot, ' +
|
|
59
|
+
'anthropic-ai, Googlebot-Extended, cohere-ai, meta-externalagent). ' +
|
|
60
|
+
'AI agents cannot determine whether they have explicit crawl permission.',
|
|
61
|
+
applicable_requirement: 'AEO §2 required crawler access; isitagentready.com Discoverability',
|
|
62
|
+
remediation:
|
|
63
|
+
'Add User-agent rules for AI crawlers. To allow all: `User-agent: GPTBot\\nAllow: /`. ' +
|
|
64
|
+
'To deny: `User-agent: GPTBot\\nDisallow: /`. Explicit rules signal intentional policy.',
|
|
65
|
+
verification: 'Fetch /robots.txt and confirm at least one AI crawler user-agent rule is present.',
|
|
66
|
+
check(ctx) {
|
|
67
|
+
const robots = ctx.site?.robots;
|
|
68
|
+
if (!robots?.raw) return [];
|
|
69
|
+
|
|
70
|
+
const AI_BOTS = [
|
|
71
|
+
'gptbot', 'claudebot', 'perplexitybot', 'googlebot-extended',
|
|
72
|
+
'anthropic-ai', 'cohere-ai', 'meta-externalagent', 'bytespider',
|
|
73
|
+
'applebot-extended', 'diffbot', 'youbot', 'img2dataset', 'omgili',
|
|
74
|
+
];
|
|
75
|
+
|
|
76
|
+
const raw = robots.raw.toLowerCase();
|
|
77
|
+
const hasAiRule = AI_BOTS.some((bot) => raw.includes(`user-agent: ${bot}`));
|
|
78
|
+
if (hasAiRule) return [];
|
|
79
|
+
|
|
80
|
+
return [{
|
|
81
|
+
subject: { type: 'file', identifier: '/robots.txt' },
|
|
82
|
+
summary: 'robots.txt contains no AI crawler rules',
|
|
83
|
+
evidence: ['No User-agent entry found for GPTBot, ClaudeBot, PerplexityBot, or similar AI crawlers'],
|
|
84
|
+
captured: 'no AI bot rules',
|
|
85
|
+
expected: 'User-agent entries for at least one AI crawler (GPTBot, ClaudeBot, etc.)',
|
|
86
|
+
}];
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
// ---------------------------------------------------------------------------
|
|
91
|
+
// AGENT-002: Link response headers for discovery
|
|
92
|
+
// ---------------------------------------------------------------------------
|
|
93
|
+
|
|
94
|
+
export const AGENT_002 = defineDetector({
|
|
95
|
+
id: 'AGENT-002',
|
|
96
|
+
name: 'Link response headers absent',
|
|
97
|
+
namespace: 'AGENT',
|
|
98
|
+
discipline: ['agent-readiness'],
|
|
99
|
+
severity: 'low',
|
|
100
|
+
deterministic: false,
|
|
101
|
+
description:
|
|
102
|
+
'HTTP Link headers on the homepage allow agents to discover structured resources ' +
|
|
103
|
+
'(sitemaps, feeds, MCP endpoints, API catalogs) without parsing HTML. ' +
|
|
104
|
+
'Absence means agents relying on header-based discovery cannot find these resources.',
|
|
105
|
+
applicable_requirement: 'RFC 8288 Web Linking; isitagentready.com Discoverability',
|
|
106
|
+
remediation:
|
|
107
|
+
'Add Link headers to your CDN responses. Example: ' +
|
|
108
|
+
'`Link: </sitemap.xml>; rel="sitemap", </.well-known/mcp>; rel="mcp"`. ' +
|
|
109
|
+
'Cloudflare Workers or nginx `add_header` can inject these.',
|
|
110
|
+
verification: 'Run `curl -I <homepage>` and inspect Link headers in the response.',
|
|
111
|
+
check(ctx) {
|
|
112
|
+
const page = homepageMeta(ctx);
|
|
113
|
+
if (!page) return [];
|
|
114
|
+
const headers = pageHeaders(page);
|
|
115
|
+
const link = headerValue(headers, 'link');
|
|
116
|
+
if (link && link.trim().length > 0) return [];
|
|
117
|
+
return [{
|
|
118
|
+
subject: { type: 'page', identifier: siteUrl(ctx) },
|
|
119
|
+
summary: 'Homepage has no Link response headers',
|
|
120
|
+
evidence: ['HTTP Link header absent from homepage response'],
|
|
121
|
+
captured: 'no Link header',
|
|
122
|
+
expected: 'Link headers pointing to sitemap, MCP endpoint, or API catalog',
|
|
123
|
+
}];
|
|
124
|
+
},
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
// AGENT-003: llms.txt presence
|
|
129
|
+
// ---------------------------------------------------------------------------
|
|
130
|
+
|
|
131
|
+
export const AGENT_003 = defineDetector({
|
|
132
|
+
id: 'AGENT-003',
|
|
133
|
+
name: 'llms.txt missing',
|
|
134
|
+
namespace: 'AGENT',
|
|
135
|
+
discipline: ['agent-readiness', 'geo'],
|
|
136
|
+
severity: 'low',
|
|
137
|
+
deterministic: false,
|
|
138
|
+
description:
|
|
139
|
+
'llms.txt (https://llmstxt.org/) is a convention for sites to provide a structured ' +
|
|
140
|
+
'Markdown summary of their content for LLM consumption. ' +
|
|
141
|
+
'Its absence means language models cannot quickly orient themselves about your site.',
|
|
142
|
+
applicable_requirement: 'GEO §3 discoverability; isitagentready.com Content Accessibility',
|
|
143
|
+
remediation:
|
|
144
|
+
'Create /llms.txt at your site root with: site purpose, key pages list with descriptions, ' +
|
|
145
|
+
'and optionally /llms-full.txt with complete content. See https://llmstxt.org/ for spec.',
|
|
146
|
+
verification: 'Fetch /llms.txt and confirm it returns 200 with Markdown content.',
|
|
147
|
+
check(ctx) {
|
|
148
|
+
// Check for llms.txt in crawled pages or site metadata
|
|
149
|
+
const llmsTxt = ctx.site?.llmsTxt || ctx.site?.meta?.llmsTxt;
|
|
150
|
+
if (llmsTxt?.found || llmsTxt?.status === 200) return [];
|
|
151
|
+
const hasLlmsTxt = ctx.site?.pages?.some(
|
|
152
|
+
(p) => p.path === '/llms.txt' || p.url?.endsWith('/llms.txt'),
|
|
153
|
+
);
|
|
154
|
+
if (hasLlmsTxt) return [];
|
|
155
|
+
return [{
|
|
156
|
+
subject: { type: 'file', identifier: '/llms.txt' },
|
|
157
|
+
summary: 'llms.txt not discovered during crawl',
|
|
158
|
+
evidence: ['No /llms.txt found in crawl results or site metadata'],
|
|
159
|
+
captured: 'llms.txt absent',
|
|
160
|
+
expected: '/llms.txt returning 200 with structured Markdown content',
|
|
161
|
+
}];
|
|
162
|
+
},
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
// ---------------------------------------------------------------------------
|
|
166
|
+
// AGENT-004: MCP Server Card discovery
|
|
167
|
+
// ---------------------------------------------------------------------------
|
|
168
|
+
|
|
169
|
+
export const AGENT_004 = defineDetector({
|
|
170
|
+
id: 'AGENT-004',
|
|
171
|
+
name: 'MCP Server Card not discoverable',
|
|
172
|
+
namespace: 'AGENT',
|
|
173
|
+
discipline: ['agent-readiness'],
|
|
174
|
+
severity: 'low',
|
|
175
|
+
deterministic: false,
|
|
176
|
+
description:
|
|
177
|
+
'A Model Context Protocol (MCP) Server Card at /.well-known/mcp enables AI agents ' +
|
|
178
|
+
'to discover and connect to your MCP server programmatically. ' +
|
|
179
|
+
'Without it, agents cannot auto-discover tool/resource APIs you publish.',
|
|
180
|
+
applicable_requirement: 'MCP spec §discovery; isitagentready.com Protocol Discovery',
|
|
181
|
+
remediation:
|
|
182
|
+
'If you operate an MCP server, publish /.well-known/mcp as a JSON document. ' +
|
|
183
|
+
'See https://modelcontextprotocol.io/ for the spec. ' +
|
|
184
|
+
'If not applicable, suppress via .citable/config.yaml.',
|
|
185
|
+
verification: 'Fetch /.well-known/mcp and confirm 200 with valid MCP server descriptor JSON.',
|
|
186
|
+
check(ctx) {
|
|
187
|
+
const mcpCard = ctx.site?.meta?.mcpCard || ctx.site?.wellKnown?.mcp;
|
|
188
|
+
if (mcpCard?.found || mcpCard?.status === 200) return [];
|
|
189
|
+
const hasMcpPage = ctx.site?.pages?.some(
|
|
190
|
+
(p) => p.path === '/.well-known/mcp' || p.url?.includes('/.well-known/mcp'),
|
|
191
|
+
);
|
|
192
|
+
if (hasMcpPage) return [];
|
|
193
|
+
// Only flag if site has API-like patterns (avoid noise for pure content sites)
|
|
194
|
+
const hasApiSignals = ctx.site?.pages?.some(
|
|
195
|
+
(p) => p.path?.includes('/api/') || p.path?.includes('/v1/') || p.path?.includes('/graphql'),
|
|
196
|
+
);
|
|
197
|
+
if (!hasApiSignals) return [];
|
|
198
|
+
return [{
|
|
199
|
+
subject: { type: 'file', identifier: '/.well-known/mcp' },
|
|
200
|
+
summary: 'MCP Server Card not found at /.well-known/mcp (site has API endpoints)',
|
|
201
|
+
evidence: [
|
|
202
|
+
'/.well-known/mcp not discovered during crawl',
|
|
203
|
+
'Site has API-like paths suggesting a programmable interface',
|
|
204
|
+
],
|
|
205
|
+
captured: 'mcp card absent',
|
|
206
|
+
expected: '/.well-known/mcp JSON descriptor for MCP server auto-discovery',
|
|
207
|
+
}];
|
|
208
|
+
},
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
// ---------------------------------------------------------------------------
|
|
212
|
+
// AGENT-005: A2A Agent Card discovery
|
|
213
|
+
// ---------------------------------------------------------------------------
|
|
214
|
+
|
|
215
|
+
export const AGENT_005 = defineDetector({
|
|
216
|
+
id: 'AGENT-005',
|
|
217
|
+
name: 'A2A Agent Card not discoverable',
|
|
218
|
+
namespace: 'AGENT',
|
|
219
|
+
discipline: ['agent-readiness'],
|
|
220
|
+
severity: 'low',
|
|
221
|
+
deterministic: false,
|
|
222
|
+
description:
|
|
223
|
+
'An Agent-to-Agent (A2A) Agent Card at /.well-known/agent.json describes agent ' +
|
|
224
|
+
'capabilities for inter-agent communication (Google A2A protocol). ' +
|
|
225
|
+
'Without it, other AI agents cannot discover your agent\'s skills or endpoints.',
|
|
226
|
+
applicable_requirement: 'Google A2A spec; isitagentready.com Protocol Discovery',
|
|
227
|
+
remediation:
|
|
228
|
+
'Publish /.well-known/agent.json describing your agent\'s capabilities and authentication. ' +
|
|
229
|
+
'See https://google.github.io/A2A/ for the specification. ' +
|
|
230
|
+
'If not running an agent endpoint, suppress this finding.',
|
|
231
|
+
verification: 'Fetch /.well-known/agent.json and confirm 200 with valid A2A agent descriptor.',
|
|
232
|
+
check(ctx) {
|
|
233
|
+
const a2aCard = ctx.site?.meta?.a2aCard || ctx.site?.wellKnown?.agent;
|
|
234
|
+
if (a2aCard?.found || a2aCard?.status === 200) return [];
|
|
235
|
+
const hasA2aPage = ctx.site?.pages?.some(
|
|
236
|
+
(p) => p.path === '/.well-known/agent.json' || p.url?.includes('/.well-known/agent.json'),
|
|
237
|
+
);
|
|
238
|
+
if (hasA2aPage) return [];
|
|
239
|
+
// Only flag for sites that appear to be agent platforms
|
|
240
|
+
const hasAgentSignals = ctx.site?.pages?.some(
|
|
241
|
+
(p) => p.path?.includes('/agent') || p.path?.includes('/api/'),
|
|
242
|
+
);
|
|
243
|
+
if (!hasAgentSignals) return [];
|
|
244
|
+
return [{
|
|
245
|
+
subject: { type: 'file', identifier: '/.well-known/agent.json' },
|
|
246
|
+
summary: 'A2A Agent Card not found at /.well-known/agent.json',
|
|
247
|
+
evidence: ['/.well-known/agent.json not discovered during crawl'],
|
|
248
|
+
captured: 'a2a agent card absent',
|
|
249
|
+
expected: '/.well-known/agent.json JSON descriptor for A2A agent discovery',
|
|
250
|
+
}];
|
|
251
|
+
},
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
// ---------------------------------------------------------------------------
|
|
255
|
+
// AGENT-006: Markdown content negotiation
|
|
256
|
+
// ---------------------------------------------------------------------------
|
|
257
|
+
|
|
258
|
+
export const AGENT_006 = defineDetector({
|
|
259
|
+
id: 'AGENT-006',
|
|
260
|
+
name: 'Markdown content negotiation not supported',
|
|
261
|
+
namespace: 'AGENT',
|
|
262
|
+
discipline: ['agent-readiness'],
|
|
263
|
+
severity: 'low',
|
|
264
|
+
deterministic: false,
|
|
265
|
+
description:
|
|
266
|
+
'Markdown content negotiation (Accept: text/markdown) lets AI agents retrieve ' +
|
|
267
|
+
'pages as clean Markdown rather than HTML, improving token efficiency. ' +
|
|
268
|
+
'Cloudflare supports this natively. Without it, agents must parse raw HTML.',
|
|
269
|
+
applicable_requirement: 'isitagentready.com Content Accessibility; Cloudflare Markdown for Agents',
|
|
270
|
+
remediation:
|
|
271
|
+
'Enable via Cloudflare (automatic with Cloudflare proxying) or serve .md variants of key pages. ' +
|
|
272
|
+
'See https://developers.cloudflare.com/fundamentals/reference/markdown-for-agents/.',
|
|
273
|
+
verification: 'Run `curl -H "Accept: text/markdown" <page-url>` and confirm Markdown response.',
|
|
274
|
+
check(ctx) {
|
|
275
|
+
const page = homepageMeta(ctx);
|
|
276
|
+
if (!page) return [];
|
|
277
|
+
const headers = pageHeaders(page);
|
|
278
|
+
const contentType = headerValue(headers, 'content-type') || '';
|
|
279
|
+
const vary = headerValue(headers, 'vary') || '';
|
|
280
|
+
if (contentType.includes('text/markdown')) return [];
|
|
281
|
+
if (vary.toLowerCase().includes('accept')) return []; // Vary: Accept signals negotiation
|
|
282
|
+
const hasMdPages = ctx.site?.pages?.some((p) => p.path?.endsWith('.md'));
|
|
283
|
+
if (hasMdPages) return [];
|
|
284
|
+
return [{
|
|
285
|
+
subject: { type: 'page', identifier: siteUrl(ctx) },
|
|
286
|
+
summary: 'No Markdown content negotiation support detected',
|
|
287
|
+
evidence: [
|
|
288
|
+
'Content-Type does not include text/markdown',
|
|
289
|
+
vary ? `Vary header: ${vary} (does not indicate Accept negotiation)` : 'No Vary header present',
|
|
290
|
+
'No .md pages found in crawl',
|
|
291
|
+
],
|
|
292
|
+
captured: 'markdown negotiation absent',
|
|
293
|
+
expected: 'Vary: Accept header or text/markdown Content-Type support',
|
|
294
|
+
}];
|
|
295
|
+
},
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
// ---------------------------------------------------------------------------
|
|
299
|
+
// AGENT-007: Web Bot Auth
|
|
300
|
+
// ---------------------------------------------------------------------------
|
|
301
|
+
|
|
302
|
+
export const AGENT_007 = defineDetector({
|
|
303
|
+
id: 'AGENT-007',
|
|
304
|
+
name: 'Web Bot Auth not declared',
|
|
305
|
+
namespace: 'AGENT',
|
|
306
|
+
discipline: ['agent-readiness'],
|
|
307
|
+
severity: 'low',
|
|
308
|
+
deterministic: false,
|
|
309
|
+
description:
|
|
310
|
+
'Web Bot Auth is a Cloudflare proposal for authenticated bot access, allowing ' +
|
|
311
|
+
'sites to distinguish legitimate AI agents from scrapers via HTTP auth headers. ' +
|
|
312
|
+
'See https://blog.cloudflare.com/web-bot-auth/.',
|
|
313
|
+
applicable_requirement: 'isitagentready.com Bot Access Control; Cloudflare Web Bot Auth',
|
|
314
|
+
remediation:
|
|
315
|
+
'Implement Web Bot Auth via Cloudflare Bot Management or return appropriate ' +
|
|
316
|
+
'WWW-Authenticate headers for bot clients. This is an emerging standard.',
|
|
317
|
+
verification: 'Check for Web-Bot-Auth or WWW-Authenticate headers in HTTP responses.',
|
|
318
|
+
check(ctx) {
|
|
319
|
+
const page = homepageMeta(ctx);
|
|
320
|
+
if (!page) return [];
|
|
321
|
+
const headers = pageHeaders(page);
|
|
322
|
+
const webBotAuth = headerValue(headers, 'web-bot-auth') || headerValue(headers, 'x-web-bot-auth');
|
|
323
|
+
if (webBotAuth) return [];
|
|
324
|
+
// Only flag if site has AI/bot-relevant signals (not all sites need this)
|
|
325
|
+
const hasAiRules = ctx.site?.robots?.raw?.toLowerCase()?.includes('gptbot') ||
|
|
326
|
+
ctx.site?.robots?.raw?.toLowerCase()?.includes('claudebot');
|
|
327
|
+
if (!hasAiRules) return []; // Only relevant if already engaging with AI bots
|
|
328
|
+
return [{
|
|
329
|
+
subject: { type: 'page', identifier: siteUrl(ctx) },
|
|
330
|
+
summary: 'Web Bot Auth headers not present (site has AI bot rules in robots.txt)',
|
|
331
|
+
evidence: [
|
|
332
|
+
'No Web-Bot-Auth header found in HTTP response',
|
|
333
|
+
'Site has AI crawler rules in robots.txt, suggesting bot-access awareness',
|
|
334
|
+
],
|
|
335
|
+
captured: 'web bot auth absent',
|
|
336
|
+
expected: 'Web-Bot-Auth header for permissioned bot access control',
|
|
337
|
+
}];
|
|
338
|
+
},
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
// ---------------------------------------------------------------------------
|
|
342
|
+
// AGENT-008: Content Signals header
|
|
343
|
+
// ---------------------------------------------------------------------------
|
|
344
|
+
|
|
345
|
+
export const AGENT_008 = defineDetector({
|
|
346
|
+
id: 'AGENT-008',
|
|
347
|
+
name: 'Content-Signals header absent',
|
|
348
|
+
namespace: 'AGENT',
|
|
349
|
+
discipline: ['agent-readiness'],
|
|
350
|
+
severity: 'low',
|
|
351
|
+
deterministic: false,
|
|
352
|
+
description:
|
|
353
|
+
'Content Signals is a Cloudflare proposal for sites to declare content type and ' +
|
|
354
|
+
'AI usage permissions via HTTP headers, eliminating the need to scrape ToS pages. ' +
|
|
355
|
+
'See https://blog.cloudflare.com/content-signals/.',
|
|
356
|
+
applicable_requirement: 'isitagentready.com Bot Access Control; Cloudflare Content Signals',
|
|
357
|
+
remediation:
|
|
358
|
+
'Add Content-Signals headers to your CDN responses. ' +
|
|
359
|
+
'Example: `Content-Signals: type=editorial; ai-training=disallowed; ai-inference=allowed`. ' +
|
|
360
|
+
'See https://blog.cloudflare.com/content-signals/ for the current proposal.',
|
|
361
|
+
verification: 'Check for Content-Signals header in HTTP responses via `curl -I <url>`.',
|
|
362
|
+
check(ctx) {
|
|
363
|
+
const page = homepageMeta(ctx);
|
|
364
|
+
if (!page) return [];
|
|
365
|
+
const headers = pageHeaders(page);
|
|
366
|
+
if (headerValue(headers, 'content-signals')) return [];
|
|
367
|
+
// Only flag if site has content worth protecting (has structured content)
|
|
368
|
+
const hasStructuredContent = ctx.site?.pages?.some(
|
|
369
|
+
(p) => p.structuredData?.length > 0 || p.schema?.length > 0,
|
|
370
|
+
);
|
|
371
|
+
if (!hasStructuredContent) return [];
|
|
372
|
+
return [{
|
|
373
|
+
subject: { type: 'page', identifier: siteUrl(ctx) },
|
|
374
|
+
summary: 'Content-Signals header not present',
|
|
375
|
+
evidence: ['No Content-Signals header found in HTTP response headers'],
|
|
376
|
+
captured: 'content signals absent',
|
|
377
|
+
expected: 'Content-Signals header declaring AI usage permissions for your content',
|
|
378
|
+
}];
|
|
379
|
+
},
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
// ---------------------------------------------------------------------------
|
|
383
|
+
// AGENT-009: Auth.md discovery
|
|
384
|
+
// ---------------------------------------------------------------------------
|
|
385
|
+
|
|
386
|
+
export const AGENT_009 = defineDetector({
|
|
387
|
+
id: 'AGENT-009',
|
|
388
|
+
name: 'auth.md not present',
|
|
389
|
+
namespace: 'AGENT',
|
|
390
|
+
discipline: ['agent-readiness'],
|
|
391
|
+
severity: 'low',
|
|
392
|
+
deterministic: false,
|
|
393
|
+
description:
|
|
394
|
+
'auth.md is a convention for documenting authentication requirements in a ' +
|
|
395
|
+
'machine-readable Markdown file. AI agents can read /auth.md to understand ' +
|
|
396
|
+
'how to authenticate before making API calls.',
|
|
397
|
+
applicable_requirement: 'isitagentready.com Protocol Discovery; Auth.md convention',
|
|
398
|
+
remediation:
|
|
399
|
+
'Create /auth.md documenting: supported auth methods (API key, OAuth, JWT), ' +
|
|
400
|
+
'how to obtain credentials, rate limits, and scope requirements.',
|
|
401
|
+
verification: 'Fetch /auth.md and confirm it returns 200 with authentication documentation.',
|
|
402
|
+
check(ctx) {
|
|
403
|
+
const authMd = ctx.site?.meta?.authMd;
|
|
404
|
+
if (authMd?.found || authMd?.status === 200) return [];
|
|
405
|
+
const hasAuthMd = ctx.site?.pages?.some(
|
|
406
|
+
(p) => p.path === '/auth.md' || p.url?.endsWith('/auth.md'),
|
|
407
|
+
);
|
|
408
|
+
if (hasAuthMd) return [];
|
|
409
|
+
// Only flag if site has auth or API paths
|
|
410
|
+
const hasApiPaths = ctx.site?.pages?.some(
|
|
411
|
+
(p) => p.path?.includes('/api/') || p.path?.includes('/login') || p.path?.includes('/oauth'),
|
|
412
|
+
);
|
|
413
|
+
if (!hasApiPaths) return [];
|
|
414
|
+
return [{
|
|
415
|
+
subject: { type: 'file', identifier: '/auth.md' },
|
|
416
|
+
summary: 'auth.md absent (site has authentication or API paths)',
|
|
417
|
+
evidence: [
|
|
418
|
+
'/auth.md not discovered during crawl',
|
|
419
|
+
'Site has login, OAuth, or /api/ paths',
|
|
420
|
+
],
|
|
421
|
+
captured: 'auth.md absent',
|
|
422
|
+
expected: '/auth.md documenting authentication requirements for agent consumers',
|
|
423
|
+
}];
|
|
424
|
+
},
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
// ---------------------------------------------------------------------------
|
|
428
|
+
// AGENT-010: Agentic commerce signals (x402 / MPP / UCP / ACP)
|
|
429
|
+
// ---------------------------------------------------------------------------
|
|
430
|
+
|
|
431
|
+
export const AGENT_010 = defineDetector({
|
|
432
|
+
id: 'AGENT-010',
|
|
433
|
+
name: 'Agentic commerce protocols not declared',
|
|
434
|
+
namespace: 'AGENT',
|
|
435
|
+
discipline: ['agent-readiness'],
|
|
436
|
+
severity: 'low',
|
|
437
|
+
deterministic: false,
|
|
438
|
+
description:
|
|
439
|
+
'Agentic commerce protocols (x402, MPP, UCP, ACP) enable AI agents to make ' +
|
|
440
|
+
'micropayments or access paid resources programmatically. ' +
|
|
441
|
+
'x402 uses HTTP 402 with payment headers; MPP/UCP/ACP provide higher-level primitives. ' +
|
|
442
|
+
'See https://www.x402.org/, https://mpp.dev/, https://ucp.dev/, https://agenticcommerce.dev/.',
|
|
443
|
+
applicable_requirement: 'isitagentready.com Commerce; x402/MPP/UCP/ACP protocols',
|
|
444
|
+
remediation:
|
|
445
|
+
'If selling API access or content: implement x402 payment flows (return HTTP 402 with ' +
|
|
446
|
+
'X-Payment header on gated resources) or integrate with MPP/UCP/ACP. ' +
|
|
447
|
+
'If commerce is not applicable, suppress this finding.',
|
|
448
|
+
verification:
|
|
449
|
+
'Fetch a gated resource and confirm 402 with payment instructions, ' +
|
|
450
|
+
'or check for X-Payment/X-MPP/X-UCP headers.',
|
|
451
|
+
check(ctx) {
|
|
452
|
+
const page = homepageMeta(ctx);
|
|
453
|
+
if (!page) return [];
|
|
454
|
+
const headers = pageHeaders(page);
|
|
455
|
+
if (
|
|
456
|
+
headerValue(headers, 'x-payment') ||
|
|
457
|
+
headerValue(headers, 'x-402') ||
|
|
458
|
+
headerValue(headers, 'x-mpp') ||
|
|
459
|
+
headerValue(headers, 'x-ucp') ||
|
|
460
|
+
headerValue(headers, 'x-acp')
|
|
461
|
+
) return [];
|
|
462
|
+
// Only flag for sites with visible commerce (pricing, checkout, API access)
|
|
463
|
+
const hasCommerce = ctx.site?.pages?.some(
|
|
464
|
+
(p) =>
|
|
465
|
+
p.path?.includes('/pricing') ||
|
|
466
|
+
p.path?.includes('/checkout') ||
|
|
467
|
+
p.path?.includes('/buy') ||
|
|
468
|
+
p.path?.includes('/subscribe'),
|
|
469
|
+
);
|
|
470
|
+
if (!hasCommerce) return [];
|
|
471
|
+
return [{
|
|
472
|
+
subject: { type: 'page', identifier: siteUrl(ctx) },
|
|
473
|
+
summary: 'No agentic commerce protocol headers detected (site has commerce pages)',
|
|
474
|
+
evidence: [
|
|
475
|
+
'No x402, MPP, UCP, or ACP protocol headers found',
|
|
476
|
+
'Site has pricing or checkout paths',
|
|
477
|
+
],
|
|
478
|
+
captured: 'no agentic commerce protocol',
|
|
479
|
+
expected: 'x402 X-Payment header or MPP/UCP/ACP declarations for agent-accessible transactions',
|
|
480
|
+
}];
|
|
481
|
+
},
|
|
482
|
+
});
|
|
483
|
+
|
|
484
|
+
export const AGENT_DETECTORS = [
|
|
485
|
+
AGENT_001,
|
|
486
|
+
AGENT_002,
|
|
487
|
+
AGENT_003,
|
|
488
|
+
AGENT_004,
|
|
489
|
+
AGENT_005,
|
|
490
|
+
AGENT_006,
|
|
491
|
+
AGENT_007,
|
|
492
|
+
AGENT_008,
|
|
493
|
+
AGENT_009,
|
|
494
|
+
AGENT_010,
|
|
495
|
+
];
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Core Web Vitals detectors
|
|
3
|
+
*
|
|
4
|
+
* NOTE: These are infrastructure/readiness checks only. No live CWV measurement.
|
|
5
|
+
* Real CWV data requires browser instrumentation or Search Console API access.
|
|
6
|
+
*
|
|
7
|
+
* Vitals:
|
|
8
|
+
* - LCP (Largest Contentful Paint): < 2.5s good, < 4s needs improvement
|
|
9
|
+
* - FID (First Input Delay): < 100ms good, < 300ms needs improvement (replaced by INP)
|
|
10
|
+
* - INP (Interaction to Next Paint): < 200ms good, < 500ms needs improvement
|
|
11
|
+
* - CLS (Cumulative Layout Shift): < 0.1 good, < 0.25 needs improvement
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { defineDetector } from './framework.js';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* CWV-001: LCP element identified and optimized
|
|
18
|
+
*
|
|
19
|
+
* Checks for common LCP blockers. Cannot measure actual LCP.
|
|
20
|
+
*/
|
|
21
|
+
export const CWV_001 = defineDetector({
|
|
22
|
+
id: 'CWV-001',
|
|
23
|
+
name: 'LCP potential blockers',
|
|
24
|
+
namespace: 'CWV',
|
|
25
|
+
discipline: ['seo'],
|
|
26
|
+
severity: 'medium',
|
|
27
|
+
deterministic: true,
|
|
28
|
+
determinismNote: 'Checks infrastructure only; cannot measure actual LCP',
|
|
29
|
+
description: 'Page has potential LCP blockers (render-blocking resources, unoptimized images)',
|
|
30
|
+
remediation: 'Eliminate render-blocking resources, preload LCP image, use srcset for responsive images',
|
|
31
|
+
verification: 'Manual Lighthouse run or Search Console CWV report',
|
|
32
|
+
check: (ctx) => {
|
|
33
|
+
const hits = [];
|
|
34
|
+
const page = ctx.page;
|
|
35
|
+
if (!page || !page.head) return hits;
|
|
36
|
+
|
|
37
|
+
// Check for render-blocking scripts
|
|
38
|
+
const scripts = page.head.querySelectorAll('script:not([async]):not([defer])');
|
|
39
|
+
for (const script of scripts) {
|
|
40
|
+
const src = script.getAttribute('src');
|
|
41
|
+
if (src && !src.includes('analytics') && !src.includes('tracking')) {
|
|
42
|
+
hits.push({
|
|
43
|
+
subject: ctx.url,
|
|
44
|
+
evidence: ['Render-blocking script: ' + src],
|
|
45
|
+
remediation: 'Add async or defer attribute, or move script to end of body',
|
|
46
|
+
});
|
|
47
|
+
break; // One hit enough
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Check for render-blocking stylesheets
|
|
52
|
+
const styles = page.head.querySelectorAll('link[rel="stylesheet"]');
|
|
53
|
+
const headStyleCount = styles.length;
|
|
54
|
+
if (headStyleCount > 5) {
|
|
55
|
+
hits.push({
|
|
56
|
+
subject: ctx.url,
|
|
57
|
+
evidence: [headStyleCount + ' stylesheets in <head> may delay rendering'],
|
|
58
|
+
remediation: 'Combine critical CSS inline, defer non-critical stylesheets',
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Check for images without dimensions (causes CLS)
|
|
63
|
+
const bodyImages = (page.body || page).querySelectorAll?.('img:not([width]):not([height])') || [];
|
|
64
|
+
if (bodyImages.length > 0) {
|
|
65
|
+
hits.push({
|
|
66
|
+
subject: ctx.url,
|
|
67
|
+
evidence: [bodyImages.length + ' images without explicit width/height attributes'],
|
|
68
|
+
remediation: 'Add width and height attributes to all images to prevent layout shift',
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return hits;
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* CWV-002: Preconnect hints for critical origins
|
|
78
|
+
*/
|
|
79
|
+
export const CWV_002 = defineDetector({
|
|
80
|
+
id: 'CWV-002',
|
|
81
|
+
name: 'preconnect hints',
|
|
82
|
+
namespace: 'CWV',
|
|
83
|
+
discipline: ['seo'],
|
|
84
|
+
severity: 'low',
|
|
85
|
+
deterministic: true,
|
|
86
|
+
description: 'Page uses preconnect for critical third-party origins',
|
|
87
|
+
remediation: 'Add <link rel="preconnect"> for fonts, analytics, CDN origins',
|
|
88
|
+
verification: 'Check network waterfall for connection timing',
|
|
89
|
+
check: (ctx) => {
|
|
90
|
+
const page = ctx.page;
|
|
91
|
+
if (!page || !page.head) return [];
|
|
92
|
+
|
|
93
|
+
const preconnects = page.head.querySelectorAll('link[rel="preconnect"]');
|
|
94
|
+
|
|
95
|
+
// Check for external fonts without preconnect
|
|
96
|
+
const fontLinks = page.head.querySelectorAll('link[href*="fonts.googleapis.com"], link[href*="fonts.gstatic.com"], link[href*="typekit.net"]');
|
|
97
|
+
|
|
98
|
+
if (fontLinks.length > 0 && preconnects.length === 0) {
|
|
99
|
+
return [{
|
|
100
|
+
subject: ctx.url,
|
|
101
|
+
evidence: ['Page loads external fonts but has no preconnect hints'],
|
|
102
|
+
remediation: 'Add <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>',
|
|
103
|
+
}];
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return [];
|
|
107
|
+
},
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* CWV-003: Image optimization for LCP
|
|
112
|
+
*/
|
|
113
|
+
export const CWV_003 = defineDetector({
|
|
114
|
+
id: 'CWV-003',
|
|
115
|
+
name: 'image optimization',
|
|
116
|
+
namespace: 'CWV',
|
|
117
|
+
discipline: ['seo'],
|
|
118
|
+
severity: 'medium',
|
|
119
|
+
deterministic: true,
|
|
120
|
+
description: 'Hero/LCP candidate images use modern formats and responsive sizing',
|
|
121
|
+
remediation: 'Use WebP/AVIF with fallback, implement srcset for responsive images',
|
|
122
|
+
verification: 'Check Network panel for image format and size',
|
|
123
|
+
check: (ctx) => {
|
|
124
|
+
const page = ctx.page;
|
|
125
|
+
if (!page) return [];
|
|
126
|
+
|
|
127
|
+
// Find large images in the first viewport (approximate LCP candidates)
|
|
128
|
+
const doc = page.body || page.documentElement || page;
|
|
129
|
+
const heroImages = doc.querySelectorAll?.('img') || [];
|
|
130
|
+
const hits = [];
|
|
131
|
+
|
|
132
|
+
for (const img of heroImages) {
|
|
133
|
+
const src = img.getAttribute('src') || '';
|
|
134
|
+
const srcset = img.getAttribute('srcset');
|
|
135
|
+
const loading = img.getAttribute('loading');
|
|
136
|
+
|
|
137
|
+
// Check for modern format
|
|
138
|
+
if (src && !src.includes('.webp') && !src.includes('.avif') && !srcset) {
|
|
139
|
+
// Could be a hit - but we can't measure actual size
|
|
140
|
+
// Just flag that srcset is missing for responsive sizing
|
|
141
|
+
if (heroImages.length === 1) {
|
|
142
|
+
hits.push({
|
|
143
|
+
subject: src,
|
|
144
|
+
evidence: ['Single hero image without srcset'],
|
|
145
|
+
remediation: 'Add srcset for responsive image sizing across devices',
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
break; // One hit enough
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Check for eager loading on above-fold images
|
|
152
|
+
const width = parseInt(img.getAttribute('width') || '0', 10);
|
|
153
|
+
if (width > 300 && loading === 'lazy') {
|
|
154
|
+
hits.push({
|
|
155
|
+
subject: src,
|
|
156
|
+
evidence: ['Large image uses lazy loading - may delay LCP'],
|
|
157
|
+
remediation: 'Remove loading="lazy" from above-fold images',
|
|
158
|
+
});
|
|
159
|
+
break;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
return hits;
|
|
164
|
+
},
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
export const cwvDetectors = [
|
|
168
|
+
CWV_001,
|
|
169
|
+
CWV_002,
|
|
170
|
+
CWV_003,
|
|
171
|
+
];
|
|
@@ -3,7 +3,7 @@ import { sha256 } from '../shared/io.js';
|
|
|
3
3
|
const REQUIRED = ['id', 'name', 'namespace', 'description', 'discipline', 'severity', 'deterministic', 'remediation', 'verification', 'check'];
|
|
4
4
|
// RENDER (browser-rendered truth / source-render divergence) is reserved for the
|
|
5
5
|
// rendered-truth phase; no RENDER detectors ship until a real renderer backs them.
|
|
6
|
-
const NAMESPACES = ['TECH', 'CRAWL', 'ARCH', 'PAGE', 'ANS', 'ENTITY', 'CLAIM', 'EVD', 'SCHEMA', 'LINK', 'EXT', 'GEO', 'RECO', 'LIFE', 'MEAS', 'RENDER'];
|
|
6
|
+
const NAMESPACES = ['TECH', 'CRAWL', 'ARCH', 'PAGE', 'ANS', 'ENTITY', 'CLAIM', 'EVD', 'SCHEMA', 'LINK', 'EXT', 'GEO', 'RECO', 'LIFE', 'MEAS', 'RENDER', 'HREFLANG', 'CWV', 'AGENT'];
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Detector definition contract. `check(ctx)` returns raw hits:
|