pagesight 0.12.2 → 0.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pagesight",
-  "version": "0.12.2",
+  "version": "0.13.0",
   "description": "See your site the way search engines and AI see it.",
   "keywords": [
     "seo",

package/src/tools/ai.ts

@@ -2,6 +2,85 @@ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
 import { auditAiCrawlers, type CrawlerStatus, fetchRobotsTxt, isAllowed, type RobotsTxt } from "../lib/robots.js";
 
+// --- llms.txt detection ---
+
+interface LlmsTxtResult {
+  exists: boolean;
+  size: number | null;
+  firstLine: string | null;
+}
+
+async function checkLlmsTxt(origin: string, path: string): Promise<LlmsTxtResult> {
+  try {
+    const res = await fetch(`${origin}${path}`, {
+      headers: { "User-Agent": "Pagesight/1.0" },
+      redirect: "follow",
+    });
+    if (!res.ok) return { exists: false, size: null, firstLine: null };
+    const text = await res.text();
+    const firstLine =
+      text
+        .split("\n")
+        .find((l) => l.trim().length > 0)
+        ?.trim() ?? null;
+    return { exists: true, size: text.length, firstLine };
+  } catch {
+    return { exists: false, size: null, firstLine: null };
+  }
+}
+
+function formatLlmsTxt(llmsTxt: LlmsTxtResult, llmsFullTxt: LlmsTxtResult): string[] {
+  const lines: string[] = ["--- LLM Visibility ---", ""];
+
+  if (llmsTxt.exists) {
+    const sizeKB = llmsTxt.size ? `${(llmsTxt.size / 1024).toFixed(1)} KB` : "unknown size";
+    lines.push(`llms.txt: FOUND (${sizeKB})`);
+    if (llmsTxt.firstLine) lines.push(`  ${llmsTxt.firstLine}`);
+  } else {
+    lines.push("llms.txt: NOT FOUND — consider adding one for AI-friendly documentation");
+  }
+
+  if (llmsFullTxt.exists) {
+    const sizeKB = llmsFullTxt.size ? `${(llmsFullTxt.size / 1024).toFixed(1)} KB` : "unknown size";
+    lines.push(`llms-full.txt: FOUND (${sizeKB})`);
+  } else if (llmsTxt.exists) {
+    lines.push("llms-full.txt: NOT FOUND — consider adding full docs for AI context windows");
+  }
+
+  return lines;
+}
+
+// --- Category summary ---
+
+function formatCategorySummary(crawlers: CrawlerStatus[]): string[] {
+  const categoryOrder = ["Training", "Search", "Assistant", "Agent", "Other"];
+  const summary = new Map<string, { blocked: number; allowed: number }>();
+
+  for (const bot of crawlers) {
+    const cat = normalizeCategory(bot.category);
+    const existing = summary.get(cat) ?? { blocked: 0, allowed: 0 };
+    if (bot.allowed) existing.allowed++;
+    else existing.blocked++;
+    summary.set(cat, existing);
+  }
+
+  const lines: string[] = ["", "--- AI Crawler Summary by Category ---", ""];
+  for (const cat of categoryOrder) {
+    const s = summary.get(cat);
+    if (!s) continue;
+    const total = s.blocked + s.allowed;
+    if (s.blocked === 0) {
+      lines.push(`${cat}: all ${total} allowed`);
+    } else if (s.allowed === 0) {
+      lines.push(`${cat}: all ${total} BLOCKED`);
+    } else {
+      lines.push(`${cat}: ${s.blocked} blocked, ${s.allowed} allowed (of ${total})`);
+    }
+  }
+
+  return lines;
+}
+
 // Normalize the messy registry categories into clean buckets
 function normalizeCategory(raw: string): string {
   const lower = raw.toLowerCase();
@@ -120,7 +199,7 @@ function formatRobotsAudit(origin: string, robots: RobotsTxt, statusCode: number
 export function registerAiTool(server: McpServer): void {
   server.tool(
     "ai",
-    "Analyze your site's AI visibility. Fetches robots.txt, audits which AI crawlers (training, search, assistant, agent) can access your content, validates syntax per RFC 9309, and checks specific paths. Shows how AI systems see your site.",
+    "Analyze your site's AI visibility. Audits AI crawler access (training, search, assistant, agent) via robots.txt, checks for llms.txt and llms-full.txt, validates syntax per RFC 9309, and tests specific path access. Shows how AI systems see your site.",
     {
       url: z
         .string()
@@ -159,8 +238,21 @@ export function registerAiTool(server: McpServer): void {
           return { content: [{ type: "text", text: lines.join("\n") }] };
         }
 
-        const crawlers = await auditAiCrawlers(robotsTxt);
-        return { content: [{ type: "text", text: formatRobotsAudit(origin, robotsTxt, statusCode, crawlers) }] };
+        const [crawlers, llmsTxt, llmsFullTxt] = await Promise.all([
+          auditAiCrawlers(robotsTxt),
+          checkLlmsTxt(origin, "/llms.txt"),
+          checkLlmsTxt(origin, "/llms-full.txt"),
+        ]);
+
+        const output: string[] = [formatRobotsAudit(origin, robotsTxt, statusCode, crawlers)];
+
+        if (crawlers.length > 0) {
+          output.push(...formatCategorySummary(crawlers));
+        }
+
+        output.push("", ...formatLlmsTxt(llmsTxt, llmsFullTxt));
+
+        return { content: [{ type: "text", text: output.join("\n") }] };
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
         return { content: [{ type: "text", text: `Error analyzing robots.txt: ${msg}` }] };