aeo-ready 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.aeo-ready/dashboard.html +339 -0
- package/README.md +116 -0
- package/bin/cli.js +106 -0
- package/package.json +35 -0
- package/skills/agent-web/SKILL.md +135 -0
- package/skills/agent-web/best-practices.md +303 -0
- package/src/benchmark/agentic-seo.js +40 -0
- package/src/checks/agent-readiness/actionable.js +165 -0
- package/src/checks/agent-readiness/capability.js +209 -0
- package/src/checks/agent-readiness/content-structure.js +242 -0
- package/src/checks/agent-readiness/discovery.js +231 -0
- package/src/checks/ai-visibility/authority.js +195 -0
- package/src/checks/ai-visibility/citation-readiness.js +228 -0
- package/src/checks/ai-visibility/freshness.js +182 -0
- package/src/checks/ai-visibility/structured-data.js +180 -0
- package/src/dashboard/generate.js +156 -0
- package/src/dashboard/sections/agent-readiness.js +71 -0
- package/src/dashboard/sections/ai-visibility.js +67 -0
- package/src/dashboard/sections/history-table.js +36 -0
- package/src/dashboard/sections/overall-score.js +128 -0
- package/src/dashboard/sections/recommendations.js +49 -0
- package/src/dashboard/sections/trend-chart.js +78 -0
- package/src/fix/generators/agents-json.js +73 -0
- package/src/fix/generators/agents-md.js +85 -0
- package/src/fix/generators/llms-txt.js +166 -0
- package/src/fix/generators/robots-txt.js +64 -0
- package/src/fix/index.js +177 -0
- package/src/history/index.js +47 -0
- package/src/index.js +2 -0
- package/src/scan.js +358 -0
- package/src/track/index.js +167 -0
- package/src/utils/detect-type.js +99 -0
- package/src/utils/fetch.js +42 -0
- package/src/utils/tokens.js +18 -0
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: agent-web
|
|
3
|
+
description: "Audit and fix agent-readiness for any website. Scores three layers, generates missing files by site type."
|
|
4
|
+
when_to_use: "make site agent-ready, llms.txt, agents.json, agent discovery, structured data for AI, robot.txt AI crawlers, agent experience audit"
|
|
5
|
+
disable-model-invocation: true
|
|
6
|
+
user-invocable: true
|
|
7
|
+
allowed-tools: WebFetch Read(*) Write(*) Bash(find *) Bash(grep *) Bash(ls *) Bash(cat *)
|
|
8
|
+
argument-hint: "[url-or-nothing]"
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
# agent-web — Make Your Site Agent-Ready
|
|
12
|
+
|
|
13
|
+
You audit websites for agent-readiness and generate the missing files.
|
|
14
|
+
|
|
15
|
+
## Philosophy
|
|
16
|
+
|
|
17
|
+
Three layers define agent-readiness:
|
|
18
|
+
1. **Discoverable** — Agents can find you (llms.txt, robots.txt, sitemap)
|
|
19
|
+
2. **Parseable** — Agents can understand you without scraping HTML (structured data, JSON endpoints, manifests)
|
|
20
|
+
3. **Actionable** — Agents can DO something with you (contact, API spec, pricing they can compare, booking links)
|
|
21
|
+
|
|
22
|
+
Most sites score 0/10. You fix that.
|
|
23
|
+
|
|
24
|
+
## Detect Mode
|
|
25
|
+
|
|
26
|
+
If `$ARGUMENTS` contains a URL (starts with http:// or https://):
|
|
27
|
+
→ **URL mode**: Fetch the live site, audit what's publicly visible, then check the local repo for gaps between source and live.
|
|
28
|
+
|
|
29
|
+
If `$ARGUMENTS` is empty or not a URL:
|
|
30
|
+
→ **Repo mode**: Audit the current working directory. Ask site type if unclear.
|
|
31
|
+
|
|
32
|
+
## Step 1: Determine Site Type
|
|
33
|
+
|
|
34
|
+
Infer from signals, then confirm with the user:
|
|
35
|
+
- Has pricing page + signup/login → SaaS product
|
|
36
|
+
- Single person, portfolio content, "about me" → Personal/portfolio
|
|
37
|
+
- Has /docs, /api, SDK references, developer-focused → API/developer tool
|
|
38
|
+
- Blog posts, articles, publication cadence → Content/blog
|
|
39
|
+
|
|
40
|
+
Four types:
|
|
41
|
+
|
|
42
|
+
- **SaaS product** — Has pricing, signup, dashboard. Needs machine-readable pricing, OpenAPI, capability manifest.
|
|
43
|
+
- **Personal/portfolio** — Person or small team. Needs expertise manifest, contact structured data, project listings.
|
|
44
|
+
- **API/developer tool** — Developer audience. Needs OpenAPI spec, integration docs as llms.txt, SDK manifest.
|
|
45
|
+
- **Content/blog** — Publishing-first. Needs topic map, article structured data, citation format.
|
|
46
|
+
|
|
47
|
+
## Step 2: Audit
|
|
48
|
+
|
|
49
|
+
### URL Mode
|
|
50
|
+
Fetch and check for:
|
|
51
|
+
- `GET /llms.txt` — exists? Quality?
|
|
52
|
+
- `GET /robots.txt` — allows AI crawlers?
|
|
53
|
+
- `GET /.well-known/agent.json` — agent manifest?
|
|
54
|
+
- `GET /sitemap.xml` — present?
|
|
55
|
+
- Check HTML `<head>` for JSON-LD structured data
|
|
56
|
+
- Check for `openapi.json` or `/api/docs`
|
|
57
|
+
- Check `<meta>` tags (og:, description)
|
|
58
|
+
|
|
59
|
+
### Repo Mode
|
|
60
|
+
Check local files:
|
|
61
|
+
- `llms.txt` or `llms-full.txt`
|
|
62
|
+
- `robots.txt`
|
|
63
|
+
- `agents.json` or `.well-known/agent.json`
|
|
64
|
+
- `sitemap.xml`
|
|
65
|
+
- Any `*.json` with schema.org or OpenAPI structure
|
|
66
|
+
- HTML files for JSON-LD in `<head>`
|
|
67
|
+
- `openapi.json` or `openapi.yaml`
|
|
68
|
+
|
|
69
|
+
## Step 3: Score
|
|
70
|
+
|
|
71
|
+
Rate each layer 0-10. Be harsh. Most sites are 1-3.
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
## Agent-Readiness Audit: [site]
|
|
75
|
+
Type: [inferred type]
|
|
76
|
+
Overall: X/10
|
|
77
|
+
|
|
78
|
+
### Discoverable (X/3)
|
|
79
|
+
- [ ] llms.txt — [MISSING / PRESENT / WEAK: reason]
|
|
80
|
+
- [ ] robots.txt AI crawlers — [MISSING / BLOCKS AI / ALLOWS]
|
|
81
|
+
- [ ] sitemap.xml — [MISSING / PRESENT]
|
|
82
|
+
|
|
83
|
+
### Parseable (X/4)
|
|
84
|
+
- [ ] Structured data (JSON-LD) — [MISSING / PRESENT / WEAK: reason]
|
|
85
|
+
- [ ] Agent manifest (agents.json) — [MISSING / PRESENT]
|
|
86
|
+
- [ ] OpenAPI spec — [MISSING / PRESENT / N/A]
|
|
87
|
+
- [ ] Content negotiation (markdown) — [MISSING / PRESENT / N/A]
|
|
88
|
+
|
|
89
|
+
### Actionable (X/3)
|
|
90
|
+
- [ ] Machine-readable contact/booking — [MISSING / PRESENT]
|
|
91
|
+
- [ ] Programmatic pricing — [MISSING / PRESENT / N/A]
|
|
92
|
+
- [ ] API endpoint / action available — [MISSING / PRESENT / N/A]
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Step 4: Advise
|
|
96
|
+
|
|
97
|
+
For each gap, explain:
|
|
98
|
+
- **What's missing** — one line
|
|
99
|
+
- **Why it matters** — what agents can't do without it
|
|
100
|
+
- **What good looks like** — reference a real company doing it well
|
|
101
|
+
|
|
102
|
+
See [best-practices.md](best-practices.md) for the full opinionated framework per site type.
|
|
103
|
+
|
|
104
|
+
## Step 5: Generate
|
|
105
|
+
|
|
106
|
+
In repo mode, generate missing files directly. In URL mode, output the files as copyable blocks.
|
|
107
|
+
|
|
108
|
+
Before writing any file that already exists, show the user what you'd change and confirm.
|
|
109
|
+
|
|
110
|
+
### Generation order (dependencies):
|
|
111
|
+
1. `robots.txt` (foundation — tells crawlers what's allowed)
|
|
112
|
+
2. `llms.txt` (discovery — what the site IS)
|
|
113
|
+
3. `llms-full.txt` (if API/dev tool type — full docs inlined for context loading)
|
|
114
|
+
4. `AGENTS.md` (if GitHub repo — project structure for coding agents)
|
|
115
|
+
5. `agents.json` or `.well-known/agent.json` (manifest — structured capabilities)
|
|
116
|
+
6. JSON-LD structured data (inline in HTML or separate)
|
|
117
|
+
7. `openapi.json` (if API/SaaS type)
|
|
118
|
+
8. `pricing.json` (if SaaS type)
|
|
119
|
+
|
|
120
|
+
## Step 6: Re-score
|
|
121
|
+
|
|
122
|
+
Show before/after:
|
|
123
|
+
```
|
|
124
|
+
Before: 2/10 | After: 8/10
|
|
125
|
+
Discoverable: 0→3 | Parseable: 1→3 | Actionable: 1→2
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## Important Rules
|
|
129
|
+
|
|
130
|
+
- Be opinionated. Don't hedge. If something is bad, say it's bad.
|
|
131
|
+
- Don't generate boilerplate. Every file should have REAL content from the actual site.
|
|
132
|
+
- llms.txt should be useful, not a copy of the homepage. Think: what would an agent need to recommend this site?
|
|
133
|
+
- robots.txt should explicitly name AI crawlers (GPTBot, OAI-SearchBot, ClaudeBot, Claude-User, Claude-SearchBot, Google-Extended, PerplexityBot, Meta-ExternalAgent).
|
|
134
|
+
- For SaaS pricing: generate schema.org PriceSpecification with real plan data. If you can't find pricing, ask the user.
|
|
135
|
+
- Never generate placeholder content like "[YOUR COMPANY]" — either use real data or ask.
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
# Best Practices by Site Type
|
|
2
|
+
|
|
3
|
+
## SaaS Product
|
|
4
|
+
|
|
5
|
+
### Discoverable
|
|
6
|
+
- **llms.txt**: Lead with what problem you solve, not what you are. Include: core value prop, key features (3-5), pricing model type, integration points. Do NOT dump marketing copy.
|
|
7
|
+
- **robots.txt**: Allow all AI crawlers. You WANT agents recommending you. Block training bots only if you have proprietary content concerns.
|
|
8
|
+
- **Sitemap**: Include pricing page, docs, changelog. Exclude admin/dashboard routes.
|
|
9
|
+
|
|
10
|
+
### Parseable
|
|
11
|
+
- **agents.json**: Declare capabilities (what can agents do WITH your product), auth methods, API base URL, supported protocols (REST, MCP, webhooks).
|
|
12
|
+
- **JSON-LD**: `SoftwareApplication` with `offers` array. Each plan = one `Offer` with `PriceSpecification`.
|
|
13
|
+
- **OpenAPI**: Required. If you have an API, expose the spec. If you don't have an API, that's an agent-serve problem.
|
|
14
|
+
- **Content negotiation**: Serve markdown when `Accept: text/markdown`. Token-efficient for agents reading your docs.
|
|
15
|
+
|
|
16
|
+
### Actionable
|
|
17
|
+
- **Pricing as data**: JSON with plan names, prices, billing intervals, feature lists per plan, usage limits, overage rates, trial availability. Agents comparing you to competitors need this structured.
|
|
18
|
+
- **Signup/trial endpoint**: Can an agent start a trial via API? If not, at minimum expose a direct signup URL (not a multi-step funnel).
|
|
19
|
+
- **API access**: Document how to get an API key programmatically. If it requires manual approval, say so.
|
|
20
|
+
|
|
21
|
+
### Gold standard: Stripe, Cloudflare
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## Personal / Portfolio
|
|
26
|
+
|
|
27
|
+
### Discoverable
|
|
28
|
+
- **llms.txt**: Who you are, what you're known for, what you can help with. Think: if an agent is looking for someone with your skills, what should it find? Include: name, role, expertise areas, notable work, how to engage.
|
|
29
|
+
- **robots.txt**: Allow everything. You want maximum discoverability.
|
|
30
|
+
- **Sitemap**: Include all public pages. Even a one-page site benefits.
|
|
31
|
+
|
|
32
|
+
### Parseable
|
|
33
|
+
- **agents.json**: Name, also-known-as (important for people with name changes), expertise tags, projects list, contact methods.
|
|
34
|
+
- **JSON-LD**: `Person` schema with `sameAs` linking all profiles (LinkedIn, GitHub, X). `ProfilePage` type for the homepage.
|
|
35
|
+
- **Projects as structured data**: Each project with name, URL, description, role, tech stack.
|
|
36
|
+
|
|
37
|
+
### Actionable
|
|
38
|
+
- **Contact**: Structured email, cal.com link, LinkedIn URL. Not "reach out" — a machine-parseable endpoint.
|
|
39
|
+
- **Hire/book**: If you're available for work, make it explicit and structured. Rate, availability, engagement types.
|
|
40
|
+
- **Content**: If you write/speak, include topics you cover. Agents recommending speakers/writers need this.
|
|
41
|
+
|
|
42
|
+
### Gold standard: brianjackson.io, cassidoo.co
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## API / Developer Tool
|
|
47
|
+
|
|
48
|
+
### Discoverable
|
|
49
|
+
- **llms.txt**: What the tool does, who it's for, quickstart (3 steps max), key concepts. This IS your agent-facing documentation. IDE agents (Cursor, Claude Code) read this directly.
|
|
50
|
+
- **robots.txt**: Allow all. Developer tools need maximum discoverability.
|
|
51
|
+
- **AGENTS.md**: If you have a GitHub repo, add AGENTS.md (OpenAI/Anthropic convention) with project structure and contribution patterns.
|
|
52
|
+
|
|
53
|
+
### Parseable
|
|
54
|
+
- **OpenAPI spec**: Non-negotiable. Complete, versioned, with examples. This is how agents integrate with you.
|
|
55
|
+
- **llms-full.txt**: Full documentation inlined. Coding agents load this into context to write integrations.
|
|
56
|
+
- **SDK manifest**: What languages, what package managers, install commands.
|
|
57
|
+
|
|
58
|
+
### Actionable
|
|
59
|
+
- **Zero-config quickstart**: Can an agent go from nothing to "hello world" in one API call? Document that path.
|
|
60
|
+
- **API key generation**: Programmatic or one-click. No "apply for access" gates.
|
|
61
|
+
- **MCP server**: If you can ship one, do it. This is the emerging standard for agent-product integration. Stripe, Supabase, Cloudflare all have one.
|
|
62
|
+
|
|
63
|
+
### Gold standard: Stripe docs, Supabase
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## Content / Blog
|
|
68
|
+
|
|
69
|
+
### Discoverable
|
|
70
|
+
- **llms.txt**: Topic map. What do you cover? What's your angle/expertise? Include: primary topics, publication cadence, who writes, editorial stance.
|
|
71
|
+
- **robots.txt**: Allow retrieval bots (Claude-User, OAI-SearchBot). Consider blocking training bots if you want citation credit without content extraction.
|
|
72
|
+
- **RSS**: Still the best structured feed for content. Include full content, not just excerpts.
|
|
73
|
+
|
|
74
|
+
### Parseable
|
|
75
|
+
- **Article structured data**: JSON-LD `Article` with author, datePublished, dateModified, headline, description, wordCount. On EVERY article page.
|
|
76
|
+
- **Topic taxonomy**: Structured category/tag data. Agents need to know what you cover to recommend specific articles.
|
|
77
|
+
- **Author schema**: `Person` with expertise, credentials. E-E-A-T signals matter for AI citation.
|
|
78
|
+
|
|
79
|
+
### Actionable
|
|
80
|
+
- **Citation format**: Tell agents how to cite you. Preferred link format, attribution requirements.
|
|
81
|
+
- **Subscribe endpoint**: API or direct URL for newsletter signup. Not a popup.
|
|
82
|
+
- **Syndication permissions**: Can agents quote you? Republish? State it explicitly.
|
|
83
|
+
|
|
84
|
+
### Gold standard: Anthropic docs, Vercel blog
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## Robots.txt — AI Crawler Reference
|
|
89
|
+
|
|
90
|
+
Allow all AI crawlers (recommended for maximum agent discoverability):
|
|
91
|
+
```
|
|
92
|
+
User-agent: GPTBot
|
|
93
|
+
Allow: /
|
|
94
|
+
|
|
95
|
+
User-agent: OAI-SearchBot
|
|
96
|
+
Allow: /
|
|
97
|
+
|
|
98
|
+
User-agent: ClaudeBot
|
|
99
|
+
Allow: /
|
|
100
|
+
|
|
101
|
+
User-agent: Claude-User
|
|
102
|
+
Allow: /
|
|
103
|
+
|
|
104
|
+
User-agent: Claude-SearchBot
|
|
105
|
+
Allow: /
|
|
106
|
+
|
|
107
|
+
User-agent: Google-Extended
|
|
108
|
+
Allow: /
|
|
109
|
+
|
|
110
|
+
User-agent: PerplexityBot
|
|
111
|
+
Allow: /
|
|
112
|
+
|
|
113
|
+
User-agent: Meta-ExternalAgent
|
|
114
|
+
Allow: /
|
|
115
|
+
|
|
116
|
+
User-agent: CCBot
|
|
117
|
+
Allow: /
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Allow retrieval only (block training, stay in AI search):
|
|
121
|
+
```
|
|
122
|
+
User-agent: GPTBot
|
|
123
|
+
Disallow: /
|
|
124
|
+
|
|
125
|
+
User-agent: OAI-SearchBot
|
|
126
|
+
Allow: /
|
|
127
|
+
|
|
128
|
+
User-agent: ClaudeBot
|
|
129
|
+
Disallow: /
|
|
130
|
+
|
|
131
|
+
User-agent: Claude-User
|
|
132
|
+
Allow: /
|
|
133
|
+
|
|
134
|
+
User-agent: Claude-SearchBot
|
|
135
|
+
Allow: /
|
|
136
|
+
|
|
137
|
+
User-agent: Google-Extended
|
|
138
|
+
Disallow: /
|
|
139
|
+
|
|
140
|
+
User-agent: CCBot
|
|
141
|
+
Disallow: /
|
|
142
|
+
|
|
143
|
+
User-agent: PerplexityBot
|
|
144
|
+
Allow: /
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
---
|
|
148
|
+
|
|
149
|
+
## agents.json — Minimum Viable Structure
|
|
150
|
+
|
|
151
|
+
```json
|
|
152
|
+
{
|
|
153
|
+
"schema_version": "1.0",
|
|
154
|
+
"name": "",
|
|
155
|
+
"description": "",
|
|
156
|
+
"interfaces": {
|
|
157
|
+
"human": "/",
|
|
158
|
+
"llm": "/llms.txt",
|
|
159
|
+
"structured": "/agents.json"
|
|
160
|
+
},
|
|
161
|
+
"capabilities": [],
|
|
162
|
+
"contact": {},
|
|
163
|
+
"auth": {}
|
|
164
|
+
}
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
For SaaS, add:
|
|
168
|
+
```json
|
|
169
|
+
{
|
|
170
|
+
"protocols": ["rest", "mcp"],
|
|
171
|
+
"api_base": "https://api.example.com/v1",
|
|
172
|
+
"openapi": "/openapi.json",
|
|
173
|
+
"pricing": "/pricing.json"
|
|
174
|
+
}
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
---
|
|
178
|
+
|
|
179
|
+
## The AEO Stack (Addy Osmani, Google)
|
|
180
|
+
|
|
181
|
+
Agentic Engine Optimization. Six layers, in implementation order:
|
|
182
|
+
|
|
183
|
+
| Layer | What | Time |
|
|
184
|
+
|-------|------|------|
|
|
185
|
+
| 1. Access control | robots.txt configured for AI crawlers | 10 min |
|
|
186
|
+
| 2. Discovery | llms.txt with task-organized links | Few hours |
|
|
187
|
+
| 3. Capability signaling | skill.md or agents.json — what can agents DO with you | Half day |
|
|
188
|
+
| 4. Content formatting | Serve markdown, kill nav noise, front-load first 500 tokens | Day |
|
|
189
|
+
| 5. Token surfacing | Expose token counts (meta tag or HTTP header) | Weekend |
|
|
190
|
+
| 6. "Copy for AI" | Button that copies clean Markdown to clipboard | Hours |
|
|
191
|
+
|
|
192
|
+
Key insight: agents compress multi-page navigation into 1-2 HTTP requests. All client-side analytics (scroll depth, time-on-page, clicks) become invisible.
|
|
193
|
+
|
|
194
|
+
### Token targets per doc type
|
|
195
|
+
- Quick start / getting started: < 15,000 tokens
|
|
196
|
+
- Individual API reference: < 25,000 tokens
|
|
197
|
+
- Full API reference: chunk by resource/endpoint, not by product
|
|
198
|
+
- Conceptual guides: < 20,000 tokens; link to detail rather than embed
|
|
199
|
+
- llms.txt itself: < 5,000 tokens
|
|
200
|
+
|
|
201
|
+
### Front-loading rule
|
|
202
|
+
First 500 tokens of any page should answer: what is this, what does it do, what do you need. Agents often truncate after initial assessment.
|
|
203
|
+
|
|
204
|
+
---
|
|
205
|
+
|
|
206
|
+
## AGENTS.md — The New README for Agents
|
|
207
|
+
|
|
208
|
+
60K+ AGENTS.md files on GitHub. Measured impact: 28% runtime reduction, 16% token reduction across 124 PRs (Gloaguen et al. 2026).
|
|
209
|
+
|
|
210
|
+
Put in repo root. Include:
|
|
211
|
+
- Project structure (key directories and their purpose)
|
|
212
|
+
- Documentation links
|
|
213
|
+
- Development sandboxes / environments
|
|
214
|
+
- Rate limits and constraints
|
|
215
|
+
- Patterns and conventions
|
|
216
|
+
- MCP server links (if applicable)
|
|
217
|
+
|
|
218
|
+
Warning: LLM-generated AGENTS.md files hurt agent success rates. Developer-written, minimal, precise files help (+4%). Write it yourself.
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
## Content-Signal Directives
|
|
223
|
+
|
|
224
|
+
Emerging robots.txt extension (contentsignals.org):
|
|
225
|
+
```
|
|
226
|
+
Content-Signal: search=yes, ai-input=yes, ai-train=no
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Granular control beyond allow/disallow:
|
|
230
|
+
- `search=yes` — allow inclusion in AI search results
|
|
231
|
+
- `ai-input=yes` — allow as real-time context for agent queries
|
|
232
|
+
- `ai-train=no` — block use for model training
|
|
233
|
+
|
|
234
|
+
Can also be served as HTTP headers or meta tags.
|
|
235
|
+
|
|
236
|
+
---
|
|
237
|
+
|
|
238
|
+
## Effort Tiers (Implementation Timeline)
|
|
239
|
+
|
|
240
|
+
### Tier 1: Afternoon (zero code)
|
|
241
|
+
- Audit robots.txt, add AI crawler rules
|
|
242
|
+
- Content-Signal directives
|
|
243
|
+
- Write llms.txt (by hand, not generated)
|
|
244
|
+
- Add Link headers pointing to llms.txt
|
|
245
|
+
- Run scanners (agent-ready.dev, isitagentready.com, agentic-seo CLI)
|
|
246
|
+
|
|
247
|
+
### Tier 2: Day or two
|
|
248
|
+
- Write llms-full.txt (concatenated key docs)
|
|
249
|
+
- "Copy as Markdown" button on doc pages
|
|
250
|
+
- JSON-LD structured data on key pages
|
|
251
|
+
- OpenAPI spec at known endpoint
|
|
252
|
+
|
|
253
|
+
### Tier 3: Sprint
|
|
254
|
+
- Content negotiation (Accept: text/markdown returns markdown)
|
|
255
|
+
- Markdown sitemap
|
|
256
|
+
- Auto-generation pipeline (Mintlify, Fern, Context7)
|
|
257
|
+
- AGENTS.md in all repos
|
|
258
|
+
|
|
259
|
+
### Tier 4: Ongoing
|
|
260
|
+
- "Single-shot" doc quality test: can an agent complete the feature from the markdown alone? (Netlify's framework)
|
|
261
|
+
- Agent traffic monitoring (track AI referral sources)
|
|
262
|
+
- Token budget optimization per page
|
|
263
|
+
- Agent skills index (`.well-known/agent-skills/index.json`)
|
|
264
|
+
|
|
265
|
+
---
|
|
266
|
+
|
|
267
|
+
## Validation Tools
|
|
268
|
+
|
|
269
|
+
| Tool | What it checks | URL |
|
|
270
|
+
|------|---------------|-----|
|
|
271
|
+
| agent-ready.dev | Vercel's Agent Readability Spec (15 site-wide + 23 per-page checks) | agent-ready.dev |
|
|
272
|
+
| isitagentready.com | Cloudflare's Agent Readiness Score | isitagentready.com |
|
|
273
|
+
| agentic-seo CLI | Lightweight audit: llms.txt, robots.txt blocking, token counts, markdown availability | github.com/addyosmani/agentic-seo |
|
|
274
|
+
|
|
275
|
+
---
|
|
276
|
+
|
|
277
|
+
## DX vs AX: Key Divergences
|
|
278
|
+
|
|
279
|
+
What works for human developers breaks for agent users:
|
|
280
|
+
|
|
281
|
+
| Dimension | DX (humans) | AX (agents) |
|
|
282
|
+
|-----------|-------------|-------------|
|
|
283
|
+
| Onboarding | "Wow moment" gated by signup | Millisecond access, deploy-then-claim |
|
|
284
|
+
| Documentation | Scannable HTML with progressive disclosure | llms.txt, pure Markdown, zero nav noise |
|
|
285
|
+
| SDKs | Language-native libraries | Strict REST + OpenAPI (agents write own SDKs) |
|
|
286
|
+
| Defaults | Execute immediately | Dry-run first (opt in to mutation, not to safety) |
|
|
287
|
+
| Errors | Helpful prose messages | Structured codes with recovery instructions |
|
|
288
|
+
| Navigation | Progressive disclosure, breadcrumbs | Single context load, everything front-loaded |
|
|
289
|
+
|
|
290
|
+
---
|
|
291
|
+
|
|
292
|
+
## Common Mistakes
|
|
293
|
+
|
|
294
|
+
1. **llms.txt is just the homepage copy pasted** — Useless. Write for an agent that needs to decide whether to recommend you.
|
|
295
|
+
2. **robots.txt blocks all bots by default** — Many hosting platforms do this. Check.
|
|
296
|
+
3. **JSON-LD is only on the homepage** — Put it on EVERY page. Each page has its own type.
|
|
297
|
+
4. **Pricing exists only as HTML** — If an agent can't parse your pricing without rendering a browser, you're invisible to comparison.
|
|
298
|
+
5. **"Contact us" instead of structured contact** — An agent can't fill out a contact form. Give it an email, a cal link, or an API endpoint.
|
|
299
|
+
6. **OpenAPI spec is stale** — If your spec doesn't match your actual API, agents will generate broken integrations.
|
|
300
|
+
7. **No content negotiation** — Agents requesting `text/markdown` get HTML. Wastes tokens, degrades quality.
|
|
301
|
+
8. **LLM-generated AGENTS.md** — Hurts success rates. Write it yourself, keep it minimal and precise.
|
|
302
|
+
9. **No token awareness** — Pages exceeding 25K tokens get truncated or skipped. Surface counts, chunk appropriately.
|
|
303
|
+
10. **Visual hierarchy without text hierarchy** — Agents can't see your CSS. Heading structure and front-loaded content matter more than layout.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { execSync } from "child_process";
|
|
2
|
+
|
|
3
|
+
export async function runBenchmark(target) {
|
|
4
|
+
try {
|
|
5
|
+
const args =
|
|
6
|
+
target && target.startsWith("http")
|
|
7
|
+
? `--url ${target} --json`
|
|
8
|
+
: `${target || "."} --json`;
|
|
9
|
+
|
|
10
|
+
const output = execSync(`npx agentic-seo ${args}`, {
|
|
11
|
+
timeout: 30000,
|
|
12
|
+
encoding: "utf8",
|
|
13
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
const result = JSON.parse(output);
|
|
17
|
+
return {
|
|
18
|
+
score: result.score ?? result.percentage ?? 0,
|
|
19
|
+
maxScore: 100,
|
|
20
|
+
grade: result.grade || null,
|
|
21
|
+
categories: result.categories || null,
|
|
22
|
+
available: true,
|
|
23
|
+
};
|
|
24
|
+
} catch (err) {
|
|
25
|
+
if (err.message?.includes("not found") || err.message?.includes("ENOENT")) {
|
|
26
|
+
return {
|
|
27
|
+
score: null,
|
|
28
|
+
maxScore: 100,
|
|
29
|
+
available: false,
|
|
30
|
+
reason: "agentic-seo not installed",
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
score: null,
|
|
35
|
+
maxScore: 100,
|
|
36
|
+
available: false,
|
|
37
|
+
reason: err.message?.slice(0, 100),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
export async function runActionableChecks(context) {
|
|
2
|
+
const checks = [
|
|
3
|
+
checkContact(context),
|
|
4
|
+
checkPricing(context),
|
|
5
|
+
checkApiEndpoint(context),
|
|
6
|
+
checkSdkManifest(context),
|
|
7
|
+
];
|
|
8
|
+
|
|
9
|
+
const score = checks.reduce((sum, c) => sum + (c.passed ? c.points : 0), 0);
|
|
10
|
+
return { score, maxScore: 10, checks };
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function checkContact(context) {
|
|
14
|
+
const html = context.html || "";
|
|
15
|
+
const allContent = getAllContent(context);
|
|
16
|
+
|
|
17
|
+
const hasEmail = /[\w.+-]+@[\w-]+\.[\w.]+/.test(allContent);
|
|
18
|
+
const hasCalLink = /cal\.com|calendly\.com|hubspot\.com\/meetings/.test(
|
|
19
|
+
allContent,
|
|
20
|
+
);
|
|
21
|
+
const hasStructuredContact =
|
|
22
|
+
allContent.includes("ContactPoint") || allContent.includes('"email"');
|
|
23
|
+
|
|
24
|
+
let points = 0;
|
|
25
|
+
if (hasEmail) points += 1;
|
|
26
|
+
if (hasCalLink) points += 1;
|
|
27
|
+
if (hasStructuredContact) points += 1;
|
|
28
|
+
|
|
29
|
+
if (points === 3) return pass("Machine-readable contact", 3);
|
|
30
|
+
if (points > 0) {
|
|
31
|
+
const missing = [];
|
|
32
|
+
if (!hasEmail) missing.push("email");
|
|
33
|
+
if (!hasCalLink) missing.push("booking link");
|
|
34
|
+
if (!hasStructuredContact) missing.push("structured ContactPoint");
|
|
35
|
+
return partial(
|
|
36
|
+
"Machine-readable contact",
|
|
37
|
+
points,
|
|
38
|
+
3,
|
|
39
|
+
`Add: ${missing.join(", ")}.`,
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
return fail(
|
|
43
|
+
"Machine-readable contact",
|
|
44
|
+
3,
|
|
45
|
+
"No machine-readable contact info. Add email, cal.com link, or ContactPoint schema.",
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function checkPricing(context) {
|
|
50
|
+
if (context.siteType === "personal" || context.siteType === "content") {
|
|
51
|
+
return {
|
|
52
|
+
name: "Programmatic pricing",
|
|
53
|
+
passed: true,
|
|
54
|
+
points: 3,
|
|
55
|
+
maxPoints: 3,
|
|
56
|
+
note: "N/A for site type",
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const allContent = getAllContent(context);
|
|
61
|
+
const files = context.files || [];
|
|
62
|
+
|
|
63
|
+
const hasPricingJson = files.some((f) => f.includes("pricing.json"));
|
|
64
|
+
const hasPriceSchema =
|
|
65
|
+
allContent.includes("PriceSpecification") || allContent.includes("offers");
|
|
66
|
+
const hasPricingPage =
|
|
67
|
+
allContent.includes("/pricing") || allContent.includes("pricing");
|
|
68
|
+
|
|
69
|
+
if (hasPricingJson || hasPriceSchema) return pass("Programmatic pricing", 3);
|
|
70
|
+
if (hasPricingPage)
|
|
71
|
+
return partial(
|
|
72
|
+
"Programmatic pricing",
|
|
73
|
+
1,
|
|
74
|
+
3,
|
|
75
|
+
"Pricing page exists but not in structured format. Add schema.org PriceSpecification or pricing.json.",
|
|
76
|
+
);
|
|
77
|
+
return fail(
|
|
78
|
+
"Programmatic pricing",
|
|
79
|
+
3,
|
|
80
|
+
"No programmatic pricing. Agents comparing you to competitors need structured plan data.",
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function checkApiEndpoint(context) {
|
|
85
|
+
if (context.siteType === "personal") {
|
|
86
|
+
return {
|
|
87
|
+
name: "API endpoint",
|
|
88
|
+
passed: true,
|
|
89
|
+
points: 2,
|
|
90
|
+
maxPoints: 2,
|
|
91
|
+
note: "N/A for site type",
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const allContent = getAllContent(context);
|
|
96
|
+
const hasApiRef = /\/api\/|\/v[12]\/|api-key|apikey/i.test(allContent);
|
|
97
|
+
const hasOpenApi =
|
|
98
|
+
context.files?.some((f) => f.includes("openapi")) ||
|
|
99
|
+
context.pages?.openapi?.status === 200;
|
|
100
|
+
|
|
101
|
+
if (hasApiRef && hasOpenApi) return pass("API endpoint", 2);
|
|
102
|
+
if (hasApiRef)
|
|
103
|
+
return partial(
|
|
104
|
+
"API endpoint",
|
|
105
|
+
1,
|
|
106
|
+
2,
|
|
107
|
+
"API references found but no formal spec. Add OpenAPI spec.",
|
|
108
|
+
);
|
|
109
|
+
return fail("API endpoint", 2, "No API endpoint detected.");
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function checkSdkManifest(context) {
|
|
113
|
+
if (context.siteType === "personal" || context.siteType === "content") {
|
|
114
|
+
return {
|
|
115
|
+
name: "SDK / integration manifest",
|
|
116
|
+
passed: true,
|
|
117
|
+
points: 2,
|
|
118
|
+
maxPoints: 2,
|
|
119
|
+
note: "N/A for site type",
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const allContent = getAllContent(context);
|
|
124
|
+
const hasSdk = /npm install|pip install|go get|cargo add|maven|gradle/i.test(
|
|
125
|
+
allContent,
|
|
126
|
+
);
|
|
127
|
+
const hasIntegrations = /integrat|connect|plugin|extension|mcp/i.test(
|
|
128
|
+
allContent,
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
if (hasSdk) return pass("SDK / integration manifest", 2);
|
|
132
|
+
if (hasIntegrations)
|
|
133
|
+
return partial(
|
|
134
|
+
"SDK / integration manifest",
|
|
135
|
+
1,
|
|
136
|
+
2,
|
|
137
|
+
"Integration mentions found but no install commands or SDK manifest.",
|
|
138
|
+
);
|
|
139
|
+
return fail(
|
|
140
|
+
"SDK / integration manifest",
|
|
141
|
+
2,
|
|
142
|
+
"No SDK or integration manifest detected.",
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function getAllContent(context) {
|
|
147
|
+
if (context.mode === "url") {
|
|
148
|
+
return Object.values(context.pages || {})
|
|
149
|
+
.map((p) => p?.text || "")
|
|
150
|
+
.join("\n");
|
|
151
|
+
}
|
|
152
|
+
return Object.values(context.fileContents || {}).join("\n");
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function pass(name, points) {
|
|
156
|
+
return { name, passed: true, points, maxPoints: points };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function partial(name, points, maxPoints, fix) {
|
|
160
|
+
return { name, passed: false, points, maxPoints, fix };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function fail(name, maxPoints, fix) {
|
|
164
|
+
return { name, passed: false, points: 0, maxPoints, fix };
|
|
165
|
+
}
|