@automatelab/ai-seo-mcp 0.7.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -22
- package/dist/index.js +177 -175
- package/dist/index.js.map +1 -1
- package/dist/lib/crawlers.json +1 -1
- package/dist/lib/robots.d.ts.map +1 -1
- package/dist/lib/robots.js +3 -1
- package/dist/lib/robots.js.map +1 -1
- package/dist/output-schemas.d.ts +283 -80
- package/dist/output-schemas.d.ts.map +1 -1
- package/dist/output-schemas.js +61 -84
- package/dist/output-schemas.js.map +1 -1
- package/dist/tools/check-robots.d.ts.map +1 -1
- package/dist/tools/check-robots.js +11 -22
- package/dist/tools/check-robots.js.map +1 -1
- package/dist/tools/generate-pricing-md.js +1 -1
- package/dist/tools/save-audit-report.js +1 -1
- package/package.json +74 -74
- package/smithery.yaml +13 -13
- package/src/lib/crawlers.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync as readMcpPackage } from "node:fs";
|
|
3
|
+
const MCP_PACKAGE_VERSION = JSON.parse(readMcpPackage(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
2
4
|
// AI-SEO MCP Server - entrypoint.
|
|
3
5
|
// All logging goes to stderr. stdout is reserved for JSON-RPC transport.
|
|
4
6
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
@@ -28,7 +30,7 @@ import { auditPageOutputShape, auditSchemaOutputShape, auditCanonicalOutputShape
|
|
|
28
30
|
import { ToolFetchError } from "./lib/fetch.js";
|
|
29
31
|
const server = new McpServer({
|
|
30
32
|
name: "@automatelab/ai-seo-mcp",
|
|
31
|
-
version:
|
|
33
|
+
version: MCP_PACKAGE_VERSION,
|
|
32
34
|
});
|
|
33
35
|
/** Serialize a ToolError to MCP error content. */
|
|
34
36
|
function toolError(err) {
|
|
@@ -70,15 +72,15 @@ function wrapHandler(handler) {
|
|
|
70
72
|
// report.* - report rendering and storage
|
|
71
73
|
// Renamed in v0.4.0 from the previous flat snake_case names.
|
|
72
74
|
// ---------------------------------------------------------------------------
|
|
73
|
-
// ---
|
|
74
|
-
server.registerTool("
|
|
75
|
+
// --- audit_page ---
|
|
76
|
+
server.registerTool("audit_page", {
|
|
75
77
|
title: "Audit page (full)",
|
|
76
78
|
description: [
|
|
77
79
|
"Full AI-SEO audit of a single URL: returns categorized findings (info/warning/error) with severity, fix instructions, and a 0-100 composite score plus per-dimension subscores.",
|
|
78
80
|
"Read-only. Fetches the URL once and runs every sub-audit (schema, robots, technical, sitemap, AI-Overview eligibility) against the response. No writes, no third-party APIs, no auth required, no rate limits beyond polite per-host throttling.",
|
|
79
81
|
"Deterministic, rule-based scoring; no LLM calls. Same URL + same input flags returns the same score.",
|
|
80
82
|
"Supports `render: \"static\" | \"headless\"`. Default `static` (fast, raw HTML only). Use `headless` for React/Vue/Angular SPAs — adds 3-10s and requires the optional `playwright-core` peer dep plus a one-time `npx playwright install chromium`.",
|
|
81
|
-
"When to use: the default entry point for `audit any page`. Use this instead of calling
|
|
83
|
+
"When to use: the default entry point for `audit any page`. Use this instead of calling check_technical / audit_schema / check_robots / check_sitemap / score_ai_overview_eligibility individually unless you specifically need only one dimension - this tool composes all of them.",
|
|
82
84
|
].join("\n\n"),
|
|
83
85
|
inputSchema: auditPageInputSchema.shape,
|
|
84
86
|
outputSchema: auditPageOutputShape,
|
|
@@ -90,14 +92,14 @@ server.registerTool("audit.page", {
|
|
|
90
92
|
openWorldHint: true,
|
|
91
93
|
},
|
|
92
94
|
}, async (input) => wrapHandler(() => auditPage(input)));
|
|
93
|
-
// ---
|
|
94
|
-
server.registerTool("
|
|
95
|
+
// --- audit_schema ---
|
|
96
|
+
server.registerTool("audit_schema", {
|
|
95
97
|
title: "Audit JSON-LD schema",
|
|
96
98
|
description: [
|
|
97
99
|
"Validate JSON-LD structured data against Schema.org rules and AI-citation best practices. Accepts either a URL (fetched) or a raw JSON string (parsed directly).",
|
|
98
100
|
"Read-only when given `url` (one HTTP GET). Zero network when given `schema_json`. No writes.",
|
|
99
101
|
"Deterministic, rule-based; no LLM. Validates required/recommended properties, @context correctness, sameAs links, and AI-search-friendly patterns.",
|
|
100
|
-
"When to use: focused JSON-LD audits, or to validate a schema block you're about to ship. For a full page audit that includes schema + everything else, use `
|
|
102
|
+
"When to use: focused JSON-LD audits, or to validate a schema block you're about to ship. For a full page audit that includes schema + everything else, use `audit_page` instead.",
|
|
101
103
|
"Either `url` or `schema_json` must be provided (not both). If both are provided, `schema_json` wins and no fetch happens.",
|
|
102
104
|
].join("\n\n"),
|
|
103
105
|
inputSchema: {
|
|
@@ -130,14 +132,14 @@ server.registerTool("audit.schema", {
|
|
|
130
132
|
}
|
|
131
133
|
return wrapHandler(() => auditSchema(input));
|
|
132
134
|
});
|
|
133
|
-
// ---
|
|
134
|
-
server.registerTool("
|
|
135
|
+
// --- audit_canonical ---
|
|
136
|
+
server.registerTool("audit_canonical", {
|
|
135
137
|
title: "Audit canonical link integrity",
|
|
136
138
|
description: [
|
|
137
139
|
"Audit a page's canonical link integrity: presence, self-reference, cross-domain mismatches, trailing-slash hygiene, and og:url consistency.",
|
|
138
140
|
"Read-only. One HTTP GET to fetch the HEAD section.",
|
|
139
141
|
"Deterministic, rule-based; no LLM.",
|
|
140
|
-
"When to use: a focused canonical-only audit (e.g. debugging a duplicate-content issue). For a full HEAD audit including OpenGraph, hreflang, noindex, title, use `
|
|
142
|
+
"When to use: a focused canonical-only audit (e.g. debugging a duplicate-content issue). For a full HEAD audit including OpenGraph, hreflang, noindex, title, use `check_technical`. For everything-on-a-page, use `audit_page`.",
|
|
141
143
|
].join("\n\n"),
|
|
142
144
|
inputSchema: auditCanonicalInputSchema.shape,
|
|
143
145
|
outputSchema: auditCanonicalOutputShape,
|
|
@@ -149,14 +151,14 @@ server.registerTool("audit.canonical", {
|
|
|
149
151
|
openWorldHint: true,
|
|
150
152
|
},
|
|
151
153
|
}, async (input) => wrapHandler(() => auditCanonical(input)));
|
|
152
|
-
// ---
|
|
153
|
-
server.registerTool("
|
|
154
|
+
// --- check_robots ---
|
|
155
|
+
server.registerTool("check_robots", {
|
|
154
156
|
title: "Check robots.txt crawler posture",
|
|
155
157
|
description: [
|
|
156
158
|
"Fetch and parse a domain's robots.txt; report per-crawler allow/disallow posture for every known AI training crawler (GPTBot, CCBot, Anthropic-AI, Google-Extended, etc.), AI search crawlers (ChatGPT-User, PerplexityBot, OAI-SearchBot), and user-triggered fetchers.",
|
|
157
159
|
"Read-only. One HTTP GET to /robots.txt. No auth, no rate limits applied.",
|
|
158
160
|
"Deterministic, rule-based; no LLM. Returns structured findings with per-crawler status.",
|
|
159
|
-
"When to use: figuring out which AI crawlers a site blocks vs allows. Combine with `
|
|
161
|
+
"When to use: figuring out which AI crawlers a site blocks vs allows. Combine with `check_sitemap` for a full pre-crawl audit. Distinct from `audit_page` which evaluates a single URL; this evaluates a whole-domain policy.",
|
|
160
162
|
].join("\n\n"),
|
|
161
163
|
inputSchema: checkRobotsInputSchema.shape,
|
|
162
164
|
outputSchema: checkRobotsOutputShape,
|
|
@@ -168,14 +170,14 @@ server.registerTool("check.robots", {
|
|
|
168
170
|
openWorldHint: true,
|
|
169
171
|
},
|
|
170
172
|
}, async (input) => wrapHandler(() => checkRobots(input)));
|
|
171
|
-
// ---
|
|
172
|
-
server.registerTool("
|
|
173
|
+
// --- check_sitemap ---
|
|
174
|
+
server.registerTool("check_sitemap", {
|
|
173
175
|
title: "Check XML sitemap health",
|
|
174
176
|
description: [
|
|
175
177
|
"Validate a domain's XML sitemap: presence, accessibility, URL count, lastmod freshness, sitemap-index handling, and image/video sitemap extensions.",
|
|
176
178
|
"Read-only. Issues N+1 HTTP GETs: one for robots.txt + sitemap, then up to `max_urls_to_check` HEADs against sampled URLs.",
|
|
177
179
|
"Deterministic, rule-based; no LLM.",
|
|
178
|
-
"When to use: site-wide indexing audits. Pair with `
|
|
180
|
+
"When to use: site-wide indexing audits. Pair with `check_robots` for a full pre-crawl picture. For per-page checks, use `audit_page` or `check_technical` instead.",
|
|
179
181
|
].join("\n\n"),
|
|
180
182
|
inputSchema: checkSitemapInputSchema.shape,
|
|
181
183
|
outputSchema: checkSitemapOutputShape,
|
|
@@ -187,14 +189,14 @@ server.registerTool("check.sitemap", {
|
|
|
187
189
|
openWorldHint: true,
|
|
188
190
|
},
|
|
189
191
|
}, async (input) => wrapHandler(() => checkSitemap(input)));
|
|
190
|
-
// ---
|
|
191
|
-
server.registerTool("
|
|
192
|
+
// --- check_technical ---
|
|
193
|
+
server.registerTool("check_technical", {
|
|
192
194
|
title: "Check technical HEAD signals",
|
|
193
195
|
description: [
|
|
194
196
|
"Audit a page's HEAD section for technical signals relevant to AI crawlers: HTTPS, canonical, OpenGraph, Twitter Card, hreflang, noindex, and title-vs-H1 hygiene.",
|
|
195
197
|
"Read-only. One HTTP GET, inspects HEAD only (body is not parsed).",
|
|
196
198
|
"Deterministic, rule-based; no LLM.",
|
|
197
|
-
"When to use: when you specifically need HEAD-tag audit findings. For the full page including schema and AI-Overview scoring, use `
|
|
199
|
+
"When to use: when you specifically need HEAD-tag audit findings. For the full page including schema and AI-Overview scoring, use `audit_page`. For canonical-only, use `audit_canonical`.",
|
|
198
200
|
].join("\n\n"),
|
|
199
201
|
inputSchema: checkTechnicalInputSchema.shape,
|
|
200
202
|
outputSchema: checkTechnicalOutputShape,
|
|
@@ -206,14 +208,14 @@ server.registerTool("check.technical", {
|
|
|
206
208
|
openWorldHint: true,
|
|
207
209
|
},
|
|
208
210
|
}, async (input) => wrapHandler(() => checkTechnical(input)));
|
|
209
|
-
// ---
|
|
210
|
-
server.registerTool("
|
|
211
|
+
// --- score_ai_overview_eligibility ---
|
|
212
|
+
server.registerTool("score_ai_overview_eligibility", {
|
|
211
213
|
title: "Score AI Overview eligibility",
|
|
212
214
|
description: [
|
|
213
215
|
"Score a page's probability of appearing in Google AI Overviews. Returns an overall 0-100 score plus six factor subscores: semantic completeness, structured data, E-E-A-T signals, entity density, freshness, and technical hygiene.",
|
|
214
216
|
"Read-only. One HTTP GET.",
|
|
215
217
|
"Deterministic, rule-based scoring derived from published 2025-2026 AI-Overview correlation studies. No LLM calls. Same URL returns the same score on repeated runs.",
|
|
216
|
-
"When to use: AI-Overview-specific prioritization. For a multi-dimensional audit that includes this scoring plus everything else, use `
|
|
218
|
+
"When to use: AI-Overview-specific prioritization. For a multi-dimensional audit that includes this scoring plus everything else, use `audit_page`. For citation-worthiness of a specific text passage (rather than a URL ranking probability), use `score_citation_worthiness`.",
|
|
217
219
|
].join("\n\n"),
|
|
218
220
|
inputSchema: scoreAiOverviewEligibilityInputSchema.shape,
|
|
219
221
|
outputSchema: scoreAiOverviewOutputShape,
|
|
@@ -225,14 +227,14 @@ server.registerTool("score.ai_overview_eligibility", {
|
|
|
225
227
|
openWorldHint: true,
|
|
226
228
|
},
|
|
227
229
|
}, async (input) => wrapHandler(() => scoreAiOverviewEligibility(input)));
|
|
228
|
-
// ---
|
|
229
|
-
server.registerTool("
|
|
230
|
+
// --- score_agentic_browsing ---
|
|
231
|
+
server.registerTool("score_agentic_browsing", {
|
|
230
232
|
title: "Score Agentic Browsing readiness",
|
|
231
233
|
description: [
|
|
232
234
|
"Score a page against the four signals Google added to the Lighthouse \"Agentic Browsing\" category in May 2026: presence of an llms.txt, WebMCP integration, accessibility-tree integrity, and layout stability. Returns an overall 0-100 score, a letter grade, and a per-factor breakdown.",
|
|
233
235
|
"Read-only. One HTTP GET for the page plus one for /llms.txt (skip with check_llms_txt=false). Pass `html` instead of `url` to score markup offline (llms.txt is then treated as absent).",
|
|
234
236
|
"Deterministic, rule-based heuristics over the fetched HTML; no LLM and no headless render required. This approximates Lighthouse's runtime signals from static markup - it does not execute Lighthouse.",
|
|
235
|
-
"When to use: checking whether a site is ready for AI agents / agentic browsers, or tracking the new Lighthouse Agentic Browsing category. For citation-eligibility of content, use `
|
|
237
|
+
"When to use: checking whether a site is ready for AI agents / agentic browsers, or tracking the new Lighthouse Agentic Browsing category. For citation-eligibility of content, use `score_citation_worthiness`; for a full page audit, use `audit_page`.",
|
|
236
238
|
].join("\n\n"),
|
|
237
239
|
inputSchema: {
|
|
238
240
|
url: z.string().url().optional().describe("Public URL to fetch and score. Either this OR `html` is required."),
|
|
@@ -255,14 +257,14 @@ server.registerTool("score.agentic_browsing", {
|
|
|
255
257
|
}
|
|
256
258
|
return wrapHandler(() => scoreAgenticBrowsingTool(input));
|
|
257
259
|
});
|
|
258
|
-
// ---
|
|
259
|
-
server.registerTool("
|
|
260
|
+
// --- llms_txt_generate ---
|
|
261
|
+
server.registerTool("llms_txt_generate", {
|
|
260
262
|
title: "Generate llms.txt",
|
|
261
263
|
description: [
|
|
262
264
|
"Generate a spec-compliant llms.txt (and optionally llms-full.txt) for a domain by reading its sitemap, sampling up to `max_pages` pages, and synthesizing a grouped, sectioned summary.",
|
|
263
265
|
"Read-only. Issues one HTTP GET for the sitemap then one per sampled page.",
|
|
264
266
|
"Deterministic; no LLM. Output is the file content as a string - this tool does NOT write to disk or upload anywhere. The caller is responsible for hosting the resulting file at `https://<domain>/llms.txt`.",
|
|
265
|
-
"When to use: bootstrapping llms.txt for a site you own. To check an existing llms.txt, use `
|
|
267
|
+
"When to use: bootstrapping llms.txt for a site you own. To check an existing llms.txt, use `llms_txt_validate` instead.",
|
|
266
268
|
].join("\n\n"),
|
|
267
269
|
inputSchema: generateLlmsTxtInputSchema.shape,
|
|
268
270
|
outputSchema: generateLlmsTxtOutputShape,
|
|
@@ -274,8 +276,8 @@ server.registerTool("llms_txt.generate", {
|
|
|
274
276
|
openWorldHint: true,
|
|
275
277
|
},
|
|
276
278
|
}, async (input) => wrapHandler(() => generateLlmsTxtTool(input)));
|
|
277
|
-
// ---
|
|
278
|
-
server.registerTool("
|
|
279
|
+
// --- pricing_generate ---
|
|
280
|
+
server.registerTool("pricing_generate", {
|
|
279
281
|
title: "Generate pricing.md",
|
|
280
282
|
description: [
|
|
281
283
|
"Generate a machine-readable /pricing.md for AI shopping/agent flows. Finds the site's pricing page (or uses `pricing_url`), extracts named tiers and price lines, and returns the file content as a string.",
|
|
@@ -292,14 +294,14 @@ server.registerTool("pricing.generate", {
|
|
|
292
294
|
openWorldHint: true,
|
|
293
295
|
},
|
|
294
296
|
}, async (input) => wrapHandler(() => generatePricingMdTool(input)));
|
|
295
|
-
// ---
|
|
296
|
-
server.registerTool("
|
|
297
|
+
// --- llms_txt_validate ---
|
|
298
|
+
server.registerTool("llms_txt_validate", {
|
|
297
299
|
title: "Validate llms.txt",
|
|
298
300
|
description: [
|
|
299
301
|
"Validate an existing llms.txt or llms-full.txt against the spec: structure, section ordering, link format, and (optionally) broken-link detection.",
|
|
300
302
|
"Read-only. One HTTP GET when given `url`; zero network when given `content`. Optional link-check issues HEAD requests against each link if `check_links` is true.",
|
|
301
303
|
"Deterministic; no LLM.",
|
|
302
|
-
"When to use: auditing an llms.txt you already have. To generate one from scratch, use `
|
|
304
|
+
"When to use: auditing an llms.txt you already have. To generate one from scratch, use `llms_txt_generate`.",
|
|
303
305
|
"Either `url` or `content` must be provided.",
|
|
304
306
|
].join("\n\n"),
|
|
305
307
|
inputSchema: {
|
|
@@ -332,15 +334,15 @@ server.registerTool("llms_txt.validate", {
|
|
|
332
334
|
}
|
|
333
335
|
return wrapHandler(() => validateLlmsTxt(input));
|
|
334
336
|
});
|
|
335
|
-
// ---
|
|
336
|
-
server.registerTool("
|
|
337
|
+
// --- score_citation_worthiness ---
|
|
338
|
+
server.registerTool("score_citation_worthiness", {
|
|
337
339
|
title: "Score AI citation worthiness",
|
|
338
340
|
description: [
|
|
339
341
|
"Score how citable a page or text block is for AI engines (ChatGPT, Claude, Perplexity, Google AI Overviews). Evaluates BLUF (bottom-line-up-front) opening, FAQ patterns, statistic density, entity clarity, and answer-shape fit for the optional `target_query`.",
|
|
340
342
|
"Also returns `extractability_score` plus per-section `chunk_analysis`: how cleanly an LLM can lift a self-contained answer from each heading-delimited section (length band, lead-sentence directness, anaphora, concrete anchors). This is the GEO mechanic - it pinpoints the exact sections to tighten, with `most_extractable` / `least_extractable` called out.",
|
|
341
343
|
"Read-only when given `url` (one HTTP GET). Zero network when given `text`. No writes.",
|
|
342
344
|
"Deterministic, rule-based; no LLM calls. Returns reproducible scores.",
|
|
343
|
-
"When to use: pre-publish content QA, or to triage which existing pages are worth optimizing for AI citation first. Distinct from `
|
|
345
|
+
"When to use: pre-publish content QA, or to triage which existing pages are worth optimizing for AI citation first. Distinct from `score_ai_overview_eligibility` which scores Google-AI-Overview ranking probability for a URL; this scores the inherent citability of a text passage regardless of host.",
|
|
344
346
|
"Either `url` or `text` must be provided.",
|
|
345
347
|
].join("\n\n"),
|
|
346
348
|
inputSchema: {
|
|
@@ -377,14 +379,14 @@ server.registerTool("score.citation_worthiness", {
|
|
|
377
379
|
}
|
|
378
380
|
return wrapHandler(() => scoreCitationWorthiness(input));
|
|
379
381
|
});
|
|
380
|
-
// ---
|
|
381
|
-
server.registerTool("
|
|
382
|
+
// --- rewrite_aeo ---
|
|
383
|
+
server.registerTool("rewrite_aeo", {
|
|
382
384
|
title: "Rewrite for Answer Engine Optimization",
|
|
383
385
|
description: [
|
|
384
386
|
"Rewrite a content block for Answer Engine Optimization. Adds a BLUF opening, FAQ structure, schema additions, and concise question-shaped headings tuned for ChatGPT / Perplexity / Google AI Overviews.",
|
|
385
387
|
"Read-only when given `url` (one HTTP GET). Zero network when given `text`. The tool does NOT write back to the URL - it only returns the rewritten content as a string. No side effects on the source.",
|
|
386
388
|
"This tool delegates the actual rewrite to the calling LLM via MCP sampling - it does not call any external API itself. The MCP host's model produces the rewrite. Same input may produce different output across runs (model-dependent).",
|
|
387
|
-
"When to use: optimizing content for direct-answer surfaces (definitions, how-tos, FAQs). For Generative Engine Optimization (entity-rich, comparison-ready synthesis), use `
|
|
389
|
+
"When to use: optimizing content for direct-answer surfaces (definitions, how-tos, FAQs). For Generative Engine Optimization (entity-rich, comparison-ready synthesis), use `rewrite_geo` instead.",
|
|
388
390
|
"Either `url` or `text` must be provided. `target_query` is required.",
|
|
389
391
|
].join("\n\n"),
|
|
390
392
|
inputSchema: {
|
|
@@ -433,14 +435,14 @@ server.registerTool("rewrite.aeo", {
|
|
|
433
435
|
}
|
|
434
436
|
return wrapHandler(() => rewriteForAeo(input, undefined, undefined, server));
|
|
435
437
|
});
|
|
436
|
-
// ---
|
|
437
|
-
server.registerTool("
|
|
438
|
+
// --- rewrite_geo ---
|
|
439
|
+
server.registerTool("rewrite_geo", {
|
|
438
440
|
title: "Rewrite for Generative Engine Optimization",
|
|
439
441
|
description: [
|
|
440
442
|
"Rewrite a content block for Generative Engine Optimization: entity-rich, comparison-ready, synthesis-friendly. Tuned for surfaces that summarize across sources (Perplexity, Google AI Mode, Claude search).",
|
|
441
443
|
"Read-only on input. Does NOT write back to the source URL - returns the rewritten content as a string.",
|
|
442
444
|
"This tool delegates the actual rewrite to the calling LLM via MCP sampling - it does not call any external API itself. The MCP host's model produces the rewrite. Output may vary across runs (model-dependent).",
|
|
443
|
-
"When to use: optimizing for synthesis-style answers across multiple sources. For direct-answer (BLUF + FAQ) optimization on a single page, use `
|
|
445
|
+
"When to use: optimizing for synthesis-style answers across multiple sources. For direct-answer (BLUF + FAQ) optimization on a single page, use `rewrite_aeo` instead.",
|
|
444
446
|
"Either `url` or `text` must be provided. `target_query` is required.",
|
|
445
447
|
].join("\n\n"),
|
|
446
448
|
inputSchema: {
|
|
@@ -489,14 +491,14 @@ server.registerTool("rewrite.geo", {
|
|
|
489
491
|
}
|
|
490
492
|
return wrapHandler(() => rewriteForGeo(input, undefined, undefined, server));
|
|
491
493
|
});
|
|
492
|
-
// ---
|
|
493
|
-
server.registerTool("
|
|
494
|
+
// --- extract_entities ---
|
|
495
|
+
server.registerTool("extract_entities", {
|
|
494
496
|
title: "Extract named entities and sameAs links",
|
|
495
497
|
description: [
|
|
496
498
|
"Extract named entities, linked concepts, and sameAs graph nodes from a page's content and structured data. Combines body-text NER with JSON-LD `@type` / `sameAs` walking.",
|
|
497
499
|
"Read-only when given `url` (one HTTP GET). Zero network when given `text`.",
|
|
498
500
|
"Primary path: MCP sampling - the host LLM does the NER and returns typed entities with sameAs URIs. Fallback path: deterministic regex-based extractor when sampling is unavailable. The result includes `mode: \"sampling\" | \"regex_fallback\"` so callers can tell which path ran.",
|
|
499
|
-
"When to use: building an entity map for schema generation, or auditing whether a page's entities match its target topic. To validate the JSON-LD itself, use `
|
|
501
|
+
"When to use: building an entity map for schema generation, or auditing whether a page's entities match its target topic. To validate the JSON-LD itself, use `audit_schema`.",
|
|
500
502
|
"Either `url` or `text` must be provided.",
|
|
501
503
|
].join("\n\n"),
|
|
502
504
|
inputSchema: {
|
|
@@ -535,14 +537,14 @@ server.registerTool("extract.entities", {
|
|
|
535
537
|
}
|
|
536
538
|
return wrapHandler(() => extractEntities(input, undefined, undefined, server));
|
|
537
539
|
});
|
|
538
|
-
// ---
|
|
539
|
-
server.registerTool("
|
|
540
|
+
// --- score_test_citation ---
|
|
541
|
+
server.registerTool("score_test_citation", {
|
|
540
542
|
title: "Test whether an AI engine would cite this page",
|
|
541
543
|
description: [
|
|
542
544
|
"Simulate `would an AI engine cite this page for this query?`. The host LLM role-plays the chosen engine (chatgpt / claude / perplexity / google_ai_overviews / any), reads the page content, and returns a cite/no-cite verdict with the verbatim excerpt it would surface plus ranked improvements.",
|
|
543
545
|
"Read-only when given `url` (one HTTP GET). Zero network when given `text`.",
|
|
544
|
-
"Primary path uses MCP sampling. If the host doesn't support sampling, falls back to a deterministic heuristic derived from `
|
|
545
|
-
"When to use: pre-publish gut-check for a specific query, or auditing whether existing content earns citation surface. Distinct from `
|
|
546
|
+
"Primary path uses MCP sampling. If the host doesn't support sampling, falls back to a deterministic heuristic derived from `score_citation_worthiness` (overall_score + per-engine subscore must both clear thresholds). The result includes `mode: \"sampling\" | \"static_heuristic\"` so callers can tell which path ran.",
|
|
547
|
+
"When to use: pre-publish gut-check for a specific query, or auditing whether existing content earns citation surface. Distinct from `score_citation_worthiness` (deterministic 0-100 score) and `audit_page` (whole-page rubric); this returns a binary cite/no-cite verdict tied to one query.",
|
|
546
548
|
"Either `url` or `text` must be provided. `target_query` is required.",
|
|
547
549
|
].join("\n\n"),
|
|
548
550
|
inputSchema: {
|
|
@@ -585,15 +587,15 @@ server.registerTool("score.test_citation", {
|
|
|
585
587
|
}
|
|
586
588
|
return wrapHandler(() => testCitation(input, undefined, undefined, server));
|
|
587
589
|
});
|
|
588
|
-
// ---
|
|
589
|
-
server.registerTool("
|
|
590
|
+
// --- diff_pages ---
|
|
591
|
+
server.registerTool("diff_pages", {
|
|
590
592
|
title: "Diff two pages for citation-worthiness",
|
|
591
593
|
description: [
|
|
592
594
|
"Compare two URLs for AI citation-worthiness and return a structured breakdown of which page is more likely to be cited and why. Typical use: your page (url_a) vs a competitor's page (url_b).",
|
|
593
|
-
"Read-only. Runs
|
|
595
|
+
"Read-only. Runs audit_page on both URLs in parallel (2 HTTP fetches per URL), then diffs dimension_scores and findings. No new fetch logic beyond what audit_page already does.",
|
|
594
596
|
"Deterministic, rule-based; no LLM calls. Same two URLs return the same comparison on repeated runs.",
|
|
595
|
-
"When to use: competitive gap analysis - understand exactly which dimensions (schema, structure, robots, entity density, freshness, technical, authority, sitemap) put a competitor ahead, and get prioritized fix_recommendations_for_a to close the gap. For a single-URL audit, use
|
|
596
|
-
"Capped at 2 URLs per call. Heuristic verdict - does not claim to know what AI assistants actually cite; verdict matches
|
|
597
|
+
"When to use: competitive gap analysis - understand exactly which dimensions (schema, structure, robots, entity density, freshness, technical, authority, sitemap) put a competitor ahead, and get prioritized fix_recommendations_for_a to close the gap. For a single-URL audit, use audit_page. For overall scoring of one page, use score_citation_worthiness.",
|
|
598
|
+
"Capped at 2 URLs per call. Heuristic verdict - does not claim to know what AI assistants actually cite; verdict matches audit_page's existing rubric.",
|
|
597
599
|
].join("\n\n"),
|
|
598
600
|
inputSchema: diffPagesInputSchema.shape,
|
|
599
601
|
outputSchema: diffPagesOutputShape,
|
|
@@ -605,14 +607,14 @@ server.registerTool("diff.pages", {
|
|
|
605
607
|
openWorldHint: true,
|
|
606
608
|
},
|
|
607
609
|
}, async (input) => wrapHandler(() => diffPages(input)));
|
|
608
|
-
// ---
|
|
609
|
-
server.registerTool("
|
|
610
|
+
// --- audit_site ---
|
|
611
|
+
server.registerTool("audit_site", {
|
|
610
612
|
title: "Audit site (homepage + robots + sitemap + schema)",
|
|
611
613
|
description: [
|
|
612
|
-
"Single-call site sweep: runs
|
|
613
|
-
"Read-only. Issues several HTTP GETs against the domain (homepage fetch, robots.txt, sitemap.xml, and up to 50 sitemap URL HEAD checks); no writes, no auth required, no rate limits beyond polite per-host throttling. The homepage GET is deduplicated across
|
|
614
|
-
"Output: domain, homepage_url, fetched_at, overall_score (0–100), overall_grade, top_5_fixes (Finding[]), and a parts breakdown with individual
|
|
615
|
-
"When to use: quick 'how does this site look overall?' — use when you want a single consolidated score and actionable fix list without calling 4 tools individually. Distinct from
|
|
614
|
+
"Single-call site sweep: runs audit_page (homepage), check_robots, check_sitemap, and audit_schema in parallel and returns an overall grade (A–F) plus top-5 highest-impact fixes.",
|
|
615
|
+
"Read-only. Issues several HTTP GETs against the domain (homepage fetch, robots.txt, sitemap.xml, and up to 50 sitemap URL HEAD checks); no writes, no auth required, no rate limits beyond polite per-host throttling. The homepage GET is deduplicated across audit_page and audit_schema (~2 network fetches for 4 logical checks). Deterministic, rule-based scoring; no LLM calls. Same domain returns the same grade on repeated runs given unchanged content.",
|
|
616
|
+
"Output: domain, homepage_url, fetched_at, overall_score (0–100), overall_grade, top_5_fixes (Finding[]), and a parts breakdown with individual audit_page, check_robots, check_sitemap, and audit_schema results — each may be a full result or { error: string } when that sub-audit fails.",
|
|
617
|
+
"When to use: quick 'how does this site look overall?' — use when you want a single consolidated score and actionable fix list without calling 4 tools individually. Distinct from audit_sitemap (samples N pages from the sitemap, not just the homepage) and audit_page (single-URL deep dive with all findings, not just top-5).",
|
|
616
618
|
].join("\n\n"),
|
|
617
619
|
inputSchema: auditSiteInputSchema.shape,
|
|
618
620
|
outputSchema: auditSiteOutputShape,
|
|
@@ -624,14 +626,14 @@ server.registerTool("audit.site", {
|
|
|
624
626
|
openWorldHint: true,
|
|
625
627
|
},
|
|
626
628
|
}, async (input) => wrapHandler(() => auditSite(input)));
|
|
627
|
-
// ---
|
|
628
|
-
server.registerTool("
|
|
629
|
+
// --- audit_sitemap ---
|
|
630
|
+
server.registerTool("audit_sitemap", {
|
|
629
631
|
title: "Audit a site's content by sampling its sitemap",
|
|
630
632
|
description: [
|
|
631
|
-
"Site-wide content audit: discovers the sitemap, samples N URLs by deterministic uniform stride, runs
|
|
632
|
-
"Read-only. One HTTP GET for sitemap discovery, optionally a few more for sitemap-index children, then `sample_size` ×
|
|
633
|
+
"Site-wide content audit: discovers the sitemap, samples N URLs by deterministic uniform stride, runs audit_page on each, and returns score distribution + worst pages + most-common findings.",
|
|
634
|
+
"Read-only. One HTTP GET for sitemap discovery, optionally a few more for sitemap-index children, then `sample_size` × audit_page calls (each one HTTP GET + parsing). Polite throttling is enforced per host.",
|
|
633
635
|
"Deterministic — same domain + same sample_size returns the same set of URLs (uniform-stride sampling). Per-page scoring is rule-based; no LLM.",
|
|
634
|
-
"When to use: portfolio-level health check across a site (\"how does our content score on average?\"). Distinct from `
|
|
636
|
+
"When to use: portfolio-level health check across a site (\"how does our content score on average?\"). Distinct from `audit_site` (homepage-only composite) and `check_sitemap` (validates sitemap.xml structure, not page content).",
|
|
635
637
|
].join("\n\n"),
|
|
636
638
|
inputSchema: auditSitemapInputSchema.shape,
|
|
637
639
|
outputSchema: auditSitemapOutputShape,
|
|
@@ -643,10 +645,10 @@ server.registerTool("audit.sitemap", {
|
|
|
643
645
|
openWorldHint: true,
|
|
644
646
|
},
|
|
645
647
|
}, async (input) => wrapHandler(() => auditSitemap(input)));
|
|
646
|
-
// ---
|
|
647
|
-
server.registerTool("
|
|
648
|
+
// --- report_save ---
|
|
649
|
+
server.registerTool("report_save", {
|
|
648
650
|
title: "Save audit report to disk",
|
|
649
|
-
description: "Render an
|
|
651
|
+
description: "Render an audit_page or audit_site result as a Markdown report and write it to a file under MCP_WORKSPACE_ROOT (defaults to cwd).",
|
|
650
652
|
inputSchema: saveAuditReportInputSchema.shape,
|
|
651
653
|
outputSchema: saveReportOutputShape,
|
|
652
654
|
annotations: {
|
|
@@ -670,9 +672,9 @@ server.prompt("audit_my_homepage", "Run a full AI-SEO audit of a site's homepage
|
|
|
670
672
|
role: "user",
|
|
671
673
|
content: {
|
|
672
674
|
type: "text",
|
|
673
|
-
text: `Call the \`
|
|
675
|
+
text: `Call the \`audit_site\` tool with domain="${domain}" and respect_robots=true. ` +
|
|
674
676
|
`When the result returns, summarize the overall_grade, the top_5_fixes (highest-impact first), ` +
|
|
675
|
-
`and the content_quality from the underlying
|
|
677
|
+
`and the content_quality from the underlying audit_page. Recommend the single most leveraged fix first.`,
|
|
676
678
|
},
|
|
677
679
|
},
|
|
678
680
|
],
|
|
@@ -683,7 +685,7 @@ server.prompt("find_citation_blockers", "Audit a URL and surface only the critic
|
|
|
683
685
|
role: "user",
|
|
684
686
|
content: {
|
|
685
687
|
type: "text",
|
|
686
|
-
text: `Call \`
|
|
688
|
+
text: `Call \`audit_page\` with url="${url}" and respect_robots=true. ` +
|
|
687
689
|
`From the result, filter findings to severity="critical" only. ` +
|
|
688
690
|
`Group them by category and present a prioritized fix list. ` +
|
|
689
691
|
`Ignore warnings and info-level findings for this pass.`,
|
|
@@ -697,7 +699,7 @@ server.prompt("generate_llms_txt_for_domain", "Generate a valid llms.txt for a d
|
|
|
697
699
|
role: "user",
|
|
698
700
|
content: {
|
|
699
701
|
type: "text",
|
|
700
|
-
text: `Call \`
|
|
702
|
+
text: `Call \`llms_txt_generate\` with domain="${domain}", max_pages=50, include_full=false. ` +
|
|
701
703
|
`Return the generated llms.txt verbatim in a code block, then summarize how many pages were indexed.`,
|
|
702
704
|
},
|
|
703
705
|
},
|
|
@@ -709,7 +711,7 @@ server.prompt("check_ai_crawler_access", "Report which AI training and search cr
|
|
|
709
711
|
role: "user",
|
|
710
712
|
content: {
|
|
711
713
|
type: "text",
|
|
712
|
-
text: `Call \`
|
|
714
|
+
text: `Call \`check_robots\` with domain="${domain}". ` +
|
|
713
715
|
`From the result, build a table grouped by training_crawlers / search_crawlers / user_triggered, ` +
|
|
714
716
|
`with one row per crawler and an allowed/blocked verdict. ` +
|
|
715
717
|
`Highlight any AI search crawlers that are blocked - those silently kill citation surface.`,
|
|
@@ -726,7 +728,7 @@ server.prompt("score_my_citation_worthiness", "Score how citable a URL is for AI
|
|
|
726
728
|
role: "user",
|
|
727
729
|
content: {
|
|
728
730
|
type: "text",
|
|
729
|
-
text: `Call \`
|
|
731
|
+
text: `Call \`score_citation_worthiness\` with url="${url}", target_query="${target_query}", respect_robots=true. ` +
|
|
730
732
|
`Present overall_score and per-engine scores (Perplexity, ChatGPT, Google AI Overviews, Claude). ` +
|
|
731
733
|
`Then suggest the 3 most impactful structural changes (BLUF opening, FAQ blocks, entity density) to lift the lowest score.`,
|
|
732
734
|
},
|
|
@@ -735,108 +737,108 @@ server.prompt("score_my_citation_worthiness", "Score how citable a URL is for AI
|
|
|
735
737
|
}));
|
|
736
738
|
// --- Resources: browsable reference data ---
|
|
737
739
|
// Hosts can read these on demand to ground answers about AI-SEO without a tool call.
|
|
738
|
-
const SIGNALS_DOC = `# AI-citation signals
|
|
739
|
-
|
|
740
|
-
The 13 signals AI assistants weigh when deciding what to cite.
|
|
741
|
-
|
|
742
|
-
## 1. JSON-LD structured data
|
|
743
|
-
Article / FAQPage / HowTo schema. With Person/Organization author nodes carrying \`sameAs\` links.
|
|
744
|
-
Example:
|
|
745
|
-
\`\`\`json
|
|
746
|
-
{"@context":"https://schema.org","@type":"Article","author":{"@type":"Person","name":"Jane","sameAs":["https://linkedin.com/in/jane"]}}
|
|
747
|
-
\`\`\`
|
|
748
|
-
|
|
749
|
-
## 2. FAQ structure
|
|
750
|
-
H3 headings ending in "?" with concise answer paragraphs. FAQPage JSON-LD doubles the signal.
|
|
751
|
-
|
|
752
|
-
## 3. BLUF (bottom line up front)
|
|
753
|
-
First 1-2 sentences answer the query directly. No throat-clearing.
|
|
754
|
-
|
|
755
|
-
## 4. Statistic density
|
|
756
|
-
2-3+ specific numbers per 500 words. Source-attributed where possible.
|
|
757
|
-
|
|
758
|
-
## 5. Named entities
|
|
759
|
-
People, products, places, organizations - named explicitly (not "this product" / "they").
|
|
760
|
-
|
|
761
|
-
## 6. Entity links
|
|
762
|
-
External links to authoritative domains: wikipedia.org, .gov, linkedin.com, crunchbase.com.
|
|
763
|
-
|
|
764
|
-
## 7. Comparison tables
|
|
765
|
-
Side-by-side product/option tables. AI engines extract these directly.
|
|
766
|
-
|
|
767
|
-
## 8. Ordered lists
|
|
768
|
-
Numbered step-by-step. Especially "how to X" content.
|
|
769
|
-
|
|
770
|
-
## 9. Schema author identity
|
|
771
|
-
Author as a Person node with profile URL, not a plain string.
|
|
772
|
-
|
|
773
|
-
## 10. Freshness
|
|
774
|
-
\`dateModified\` within 90 days. Stale content is downranked aggressively.
|
|
775
|
-
|
|
776
|
-
## 11. Canonical hygiene
|
|
777
|
-
Self-referencing canonical, no trailing-slash redirects, \`og:url\` matches canonical.
|
|
778
|
-
|
|
779
|
-
## 12. AI crawler allowance
|
|
780
|
-
robots.txt allows OAI-SearchBot, PerplexityBot, Claude-SearchBot. Blocking training-only crawlers (GPTBot, ClaudeBot) is fine; blocking search crawlers kills citation surface.
|
|
781
|
-
|
|
782
|
-
## 13. Sitemap freshness
|
|
783
|
-
XML sitemap present, lastmod >80% coverage, listed in robots.txt.
|
|
784
|
-
|
|
785
|
-
Use \`
|
|
740
|
+
const SIGNALS_DOC = `# AI-citation signals
|
|
741
|
+
|
|
742
|
+
The 13 signals AI assistants weigh when deciding what to cite.
|
|
743
|
+
|
|
744
|
+
## 1. JSON-LD structured data
|
|
745
|
+
Article / FAQPage / HowTo schema. With Person/Organization author nodes carrying \`sameAs\` links.
|
|
746
|
+
Example:
|
|
747
|
+
\`\`\`json
|
|
748
|
+
{"@context":"https://schema.org","@type":"Article","author":{"@type":"Person","name":"Jane","sameAs":["https://linkedin.com/in/jane"]}}
|
|
749
|
+
\`\`\`
|
|
750
|
+
|
|
751
|
+
## 2. FAQ structure
|
|
752
|
+
H3 headings ending in "?" with concise answer paragraphs. FAQPage JSON-LD doubles the signal.
|
|
753
|
+
|
|
754
|
+
## 3. BLUF (bottom line up front)
|
|
755
|
+
First 1-2 sentences answer the query directly. No throat-clearing.
|
|
756
|
+
|
|
757
|
+
## 4. Statistic density
|
|
758
|
+
2-3+ specific numbers per 500 words. Source-attributed where possible.
|
|
759
|
+
|
|
760
|
+
## 5. Named entities
|
|
761
|
+
People, products, places, organizations - named explicitly (not "this product" / "they").
|
|
762
|
+
|
|
763
|
+
## 6. Entity links
|
|
764
|
+
External links to authoritative domains: wikipedia.org, .gov, linkedin.com, crunchbase.com.
|
|
765
|
+
|
|
766
|
+
## 7. Comparison tables
|
|
767
|
+
Side-by-side product/option tables. AI engines extract these directly.
|
|
768
|
+
|
|
769
|
+
## 8. Ordered lists
|
|
770
|
+
Numbered step-by-step. Especially "how to X" content.
|
|
771
|
+
|
|
772
|
+
## 9. Schema author identity
|
|
773
|
+
Author as a Person node with profile URL, not a plain string.
|
|
774
|
+
|
|
775
|
+
## 10. Freshness
|
|
776
|
+
\`dateModified\` within 90 days. Stale content is downranked aggressively.
|
|
777
|
+
|
|
778
|
+
## 11. Canonical hygiene
|
|
779
|
+
Self-referencing canonical, no trailing-slash redirects, \`og:url\` matches canonical.
|
|
780
|
+
|
|
781
|
+
## 12. AI crawler allowance
|
|
782
|
+
robots.txt allows OAI-SearchBot, PerplexityBot, Claude-SearchBot. Blocking training-only crawlers (GPTBot, ClaudeBot) is fine; blocking search crawlers kills citation surface.
|
|
783
|
+
|
|
784
|
+
## 13. Sitemap freshness
|
|
785
|
+
XML sitemap present, lastmod >80% coverage, listed in robots.txt.
|
|
786
|
+
|
|
787
|
+
Use \`audit_page\` to score a URL across all 13 signals at once.
|
|
786
788
|
`;
|
|
787
|
-
const CRAWLERS_DOC = `# AI crawlers reference
|
|
788
|
-
|
|
789
|
-
The user-agents AI assistants use, what they do, and how to allow / block them in robots.txt.
|
|
790
|
-
|
|
791
|
-
## Training crawlers
|
|
792
|
-
These scrape content to train future model weights. Blocking them does not affect live AI citations.
|
|
793
|
-
|
|
794
|
-
- **GPTBot** - OpenAI training crawler. \`User-agent: GPTBot\`
|
|
795
|
-
- **ClaudeBot** - Anthropic training crawler. \`User-agent: ClaudeBot\`
|
|
796
|
-
- **CCBot** - Common Crawl. Powers many downstream models. \`User-agent: CCBot\`
|
|
797
|
-
- **Meta-ExternalAgent** - Meta training crawler. \`User-agent: Meta-ExternalAgent\`
|
|
798
|
-
- **Amazonbot** - Amazon training crawler. \`User-agent: Amazonbot\`
|
|
799
|
-
|
|
800
|
-
## Search crawlers
|
|
801
|
-
These power live AI search / answer engines. Blocking these directly suppresses your visibility in AI answers.
|
|
802
|
-
|
|
803
|
-
- **OAI-SearchBot** - ChatGPT search index. \`User-agent: OAI-SearchBot\`
|
|
804
|
-
- **Claude-SearchBot** - Claude live search. \`User-agent: Claude-SearchBot\`
|
|
805
|
-
- **PerplexityBot** - Perplexity search index. \`User-agent: PerplexityBot\`
|
|
806
|
-
- **Bingbot** - Bing, also powers Copilot. \`User-agent: Bingbot\`
|
|
807
|
-
- **DuckAssistBot** - DuckDuckGo AI assist. \`User-agent: DuckAssistBot\`
|
|
808
|
-
|
|
809
|
-
## User-triggered fetchers
|
|
810
|
-
On-demand fetches when a user shares a URL in chat. Blocking these breaks user-pasted links.
|
|
811
|
-
|
|
812
|
-
- **ChatGPT-User** - ChatGPT browsing on user request. \`User-agent: ChatGPT-User\`
|
|
813
|
-
- **Claude-User** - Claude browsing on user request. \`User-agent: Claude-User\`
|
|
814
|
-
- **Google-Agent** - Google Search agent fetches. \`User-agent: Google-Agent\`
|
|
815
|
-
|
|
816
|
-
## Robots-token-only signals
|
|
817
|
-
These are not crawlers - they are tokens that modify whether the existing Googlebot / Applebot crawl is used for AI grounding.
|
|
818
|
-
|
|
819
|
-
- **Google-Extended** - opts out of Google AI training/grounding. \`User-agent: Google-Extended\`
|
|
820
|
-
- **Applebot-Extended** - opts out of Apple AI training. \`User-agent: Applebot-Extended\`
|
|
821
|
-
|
|
822
|
-
## Recommended posture
|
|
823
|
-
For most sites that want AI citation surface:
|
|
824
|
-
\`\`\`
|
|
825
|
-
User-agent: GPTBot
|
|
826
|
-
Disallow: /
|
|
827
|
-
|
|
828
|
-
User-agent: ClaudeBot
|
|
829
|
-
Disallow: /
|
|
830
|
-
|
|
831
|
-
User-agent: CCBot
|
|
832
|
-
Disallow: /
|
|
833
|
-
|
|
834
|
-
User-agent: *
|
|
835
|
-
Allow: /
|
|
836
|
-
\`\`\`
|
|
837
|
-
(Blocks training, allows everything else - including search crawlers.)
|
|
838
|
-
|
|
839
|
-
Use \`
|
|
789
|
+
const CRAWLERS_DOC = `# AI crawlers reference
|
|
790
|
+
|
|
791
|
+
The user-agents AI assistants use, what they do, and how to allow / block them in robots.txt.
|
|
792
|
+
|
|
793
|
+
## Training crawlers
|
|
794
|
+
These scrape content to train future model weights. Blocking them does not affect live AI citations.
|
|
795
|
+
|
|
796
|
+
- **GPTBot** - OpenAI training crawler. \`User-agent: GPTBot\`
|
|
797
|
+
- **ClaudeBot** - Anthropic training crawler. \`User-agent: ClaudeBot\`
|
|
798
|
+
- **CCBot** - Common Crawl. Powers many downstream models. \`User-agent: CCBot\`
|
|
799
|
+
- **Meta-ExternalAgent** - Meta training crawler. \`User-agent: Meta-ExternalAgent\`
|
|
800
|
+
- **Amazonbot** - Amazon training crawler. \`User-agent: Amazonbot\`
|
|
801
|
+
|
|
802
|
+
## Search crawlers
|
|
803
|
+
These power live AI search / answer engines. Blocking these directly suppresses your visibility in AI answers.
|
|
804
|
+
|
|
805
|
+
- **OAI-SearchBot** - ChatGPT search index. \`User-agent: OAI-SearchBot\`
|
|
806
|
+
- **Claude-SearchBot** - Claude live search. \`User-agent: Claude-SearchBot\`
|
|
807
|
+
- **PerplexityBot** - Perplexity search index. \`User-agent: PerplexityBot\`
|
|
808
|
+
- **Bingbot** - Bing, also powers Copilot. \`User-agent: Bingbot\`
|
|
809
|
+
- **DuckAssistBot** - DuckDuckGo AI assist. \`User-agent: DuckAssistBot\`
|
|
810
|
+
|
|
811
|
+
## User-triggered fetchers
|
|
812
|
+
On-demand fetches when a user shares a URL in chat. Blocking these breaks user-pasted links.
|
|
813
|
+
|
|
814
|
+
- **ChatGPT-User** - ChatGPT browsing on user request. \`User-agent: ChatGPT-User\`
|
|
815
|
+
- **Claude-User** - Claude browsing on user request. \`User-agent: Claude-User\`
|
|
816
|
+
- **Google-Agent** - Google Search agent fetches. \`User-agent: Google-Agent\`
|
|
817
|
+
|
|
818
|
+
## Robots-token-only signals
|
|
819
|
+
These are not crawlers - they are tokens that modify whether the existing Googlebot / Applebot crawl is used for AI grounding.
|
|
820
|
+
|
|
821
|
+
- **Google-Extended** - opts out of Google AI training/grounding. \`User-agent: Google-Extended\`
|
|
822
|
+
- **Applebot-Extended** - opts out of Apple AI training. \`User-agent: Applebot-Extended\`
|
|
823
|
+
|
|
824
|
+
## Recommended posture
|
|
825
|
+
For most sites that want AI citation surface:
|
|
826
|
+
\`\`\`
|
|
827
|
+
User-agent: GPTBot
|
|
828
|
+
Disallow: /
|
|
829
|
+
|
|
830
|
+
User-agent: ClaudeBot
|
|
831
|
+
Disallow: /
|
|
832
|
+
|
|
833
|
+
User-agent: CCBot
|
|
834
|
+
Disallow: /
|
|
835
|
+
|
|
836
|
+
User-agent: *
|
|
837
|
+
Allow: /
|
|
838
|
+
\`\`\`
|
|
839
|
+
(Blocks training, allows everything else - including search crawlers.)
|
|
840
|
+
|
|
841
|
+
Use \`check_robots\` to see your current posture per crawler.
|
|
840
842
|
`;
|
|
841
843
|
server.resource("ai-citation-signals", "ai-citation://signals", { mimeType: "text/markdown", description: "The 13 signals AI assistants use to decide what to cite, with examples." }, async (uri) => ({
|
|
842
844
|
contents: [{ uri: uri.href, mimeType: "text/markdown", text: SIGNALS_DOC }],
|