pagesight 0.12.1 → 0.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pagesight",
-  "version": "0.12.1",
+  "version": "0.13.0",
   "description": "See your site the way search engines and AI see it.",
   "keywords": [
     "seo",

package/src/index.ts

@@ -1,17 +1,12 @@
 #!/usr/bin/env bun
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { registerAiTool } from "./tools/ai.js";
 import { registerAuditTool } from "./tools/audit.js";
-import { registerCruxTool } from "./tools/crux.js";
-import { registerInspectTool } from "./tools/inspect.js";
-import { registerLinksTool } from "./tools/links.js";
-import { registerMetatagsTool } from "./tools/metatags.js";
-import { registerPagespeedTool } from "./tools/pagespeed.js";
-import { registerPerformanceTool } from "./tools/performance.js";
-import { registerRobotsTool } from "./tools/robots.js";
-import { registerSampleInspectTool } from "./tools/sample-inspect.js";
+import { registerPageTool } from "./tools/page.js";
+import { registerSearchTool } from "./tools/search.js";
 import { registerSetupTool } from "./tools/setup.js";
-import { registerSitemapsTool } from "./tools/sitemaps.js";
+import { registerSpeedTool } from "./tools/speed.js";
 
 const pkg = await Bun.file(new URL("../package.json", import.meta.url)).json();
 
@@ -20,17 +15,12 @@ const server = new McpServer({
   version: pkg.version,
 });
 
+registerAiTool(server);
 registerAuditTool(server);
-registerCruxTool(server);
-registerInspectTool(server);
-registerLinksTool(server);
-registerMetatagsTool(server);
-registerPagespeedTool(server);
-registerPerformanceTool(server);
-registerRobotsTool(server);
-registerSampleInspectTool(server);
-registerSitemapsTool(server);
+registerPageTool(server);
+registerSearchTool(server);
 registerSetupTool(server);
+registerSpeedTool(server);
 
 const transport = new StdioServerTransport();
 await server.connect(transport);

package/src/lib/sitemap.ts

@@ -0,0 +1,106 @@
+import { inspectUrl } from "./gsc.js";
+
+export interface SitemapParseResult {
+  urls: string[];
+  isSitemapIndex: boolean;
+  childSitemaps: string[];
+}
+
+export function parseSitemapXml(xml: string): SitemapParseResult {
+  const urls: string[] = [];
+  const childSitemaps: string[] = [];
+
+  const isSitemapIndex = /<sitemapindex/i.test(xml);
+
+  if (isSitemapIndex) {
+    for (const m of xml.matchAll(/<sitemap[^>]*>[\s\S]*?<loc>\s*(.*?)\s*<\/loc>[\s\S]*?<\/sitemap>/gi)) {
+      childSitemaps.push(m[1].trim());
+    }
+  } else {
+    for (const m of xml.matchAll(/<url[^>]*>[\s\S]*?<loc>\s*(.*?)\s*<\/loc>[\s\S]*?<\/url>/gi)) {
+      urls.push(m[1].trim());
+    }
+  }
+
+  return { urls, isSitemapIndex, childSitemaps };
+}
+
+export async function fetchSitemap(sitemapUrl: string): Promise<SitemapParseResult> {
+  const res = await fetch(sitemapUrl, {
+    headers: { "User-Agent": "Pagesight/1.0" },
+  });
+
+  if (!res.ok) {
+    throw new Error(`Failed to fetch sitemap ${sitemapUrl}: HTTP ${res.status}`);
+  }
+
+  const xml = await res.text();
+  return parseSitemapXml(xml);
+}
+
+export function sampleUrls(urls: string[], count: number, strategy: string): string[] {
+  if (urls.length <= count) return [...urls];
+
+  if (strategy === "first") {
+    return urls.slice(0, count);
+  }
+
+  if (strategy === "spread") {
+    const step = Math.floor(urls.length / count);
+    const sampled: string[] = [];
+    for (let i = 0; i < count; i++) {
+      sampled.push(urls[i * step]);
+    }
+    return sampled;
+  }
+
+  // random (default)
+  const shuffled = [...urls];
+  for (let i = shuffled.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+  }
+  return shuffled.slice(0, count);
+}
+
+export interface InspectionSummary {
+  url: string;
+  verdict: string;
+  coverageState: string;
+  pageFetchState: string;
+  robotsTxtState: string;
+  indexingState: string;
+  lastCrawlTime: string | null;
+  googleCanonical: string | null;
+  error: string | null;
+}
+
+export async function inspectSingle(url: string, siteUrl: string): Promise<InspectionSummary> {
+  try {
+    const r = await inspectUrl(url, siteUrl);
+    const idx = r.indexStatusResult;
+    return {
+      url,
+      verdict: idx.verdict,
+      coverageState: idx.coverageState,
+      pageFetchState: idx.pageFetchState,
+      robotsTxtState: idx.robotsTxtState,
+      indexingState: idx.indexingState,
+      lastCrawlTime: idx.lastCrawlTime ?? null,
+      googleCanonical: idx.googleCanonical ?? null,
+      error: null,
+    };
+  } catch (err) {
+    return {
+      url,
+      verdict: "ERROR",
+      coverageState: "ERROR",
+      pageFetchState: "ERROR",
+      robotsTxtState: "ERROR",
+      indexingState: "ERROR",
+      lastCrawlTime: null,
+      googleCanonical: null,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}

package/src/tools/{robots.ts → ai.ts}

@@ -2,6 +2,85 @@ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
 import { auditAiCrawlers, type CrawlerStatus, fetchRobotsTxt, isAllowed, type RobotsTxt } from "../lib/robots.js";
 
+// --- llms.txt detection ---
+
+interface LlmsTxtResult {
+  exists: boolean;
+  size: number | null;
+  firstLine: string | null;
+}
+
+async function checkLlmsTxt(origin: string, path: string): Promise<LlmsTxtResult> {
+  try {
+    const res = await fetch(`${origin}${path}`, {
+      headers: { "User-Agent": "Pagesight/1.0" },
+      redirect: "follow",
+    });
+    if (!res.ok) return { exists: false, size: null, firstLine: null };
+    const text = await res.text();
+    const firstLine =
+      text
+        .split("\n")
+        .find((l) => l.trim().length > 0)
+        ?.trim() ?? null;
+    return { exists: true, size: text.length, firstLine };
+  } catch {
+    return { exists: false, size: null, firstLine: null };
+  }
+}
+
+function formatLlmsTxt(llmsTxt: LlmsTxtResult, llmsFullTxt: LlmsTxtResult): string[] {
+  const lines: string[] = ["--- LLM Visibility ---", ""];
+
+  if (llmsTxt.exists) {
+    const sizeKB = llmsTxt.size ? `${(llmsTxt.size / 1024).toFixed(1)} KB` : "unknown size";
+    lines.push(`llms.txt: FOUND (${sizeKB})`);
+    if (llmsTxt.firstLine) lines.push(`  ${llmsTxt.firstLine}`);
+  } else {
+    lines.push("llms.txt: NOT FOUND — consider adding one for AI-friendly documentation");
+  }
+
+  if (llmsFullTxt.exists) {
+    const sizeKB = llmsFullTxt.size ? `${(llmsFullTxt.size / 1024).toFixed(1)} KB` : "unknown size";
+    lines.push(`llms-full.txt: FOUND (${sizeKB})`);
+  } else if (llmsTxt.exists) {
+    lines.push("llms-full.txt: NOT FOUND — consider adding full docs for AI context windows");
+  }
+
+  return lines;
+}
+
+// --- Category summary ---
+
+function formatCategorySummary(crawlers: CrawlerStatus[]): string[] {
+  const categoryOrder = ["Training", "Search", "Assistant", "Agent", "Other"];
+  const summary = new Map<string, { blocked: number; allowed: number }>();
+
+  for (const bot of crawlers) {
+    const cat = normalizeCategory(bot.category);
+    const existing = summary.get(cat) ?? { blocked: 0, allowed: 0 };
+    if (bot.allowed) existing.allowed++;
+    else existing.blocked++;
+    summary.set(cat, existing);
+  }
+
+  const lines: string[] = ["", "--- AI Crawler Summary by Category ---", ""];
+  for (const cat of categoryOrder) {
+    const s = summary.get(cat);
+    if (!s) continue;
+    const total = s.blocked + s.allowed;
+    if (s.blocked === 0) {
+      lines.push(`${cat}: all ${total} allowed`);
+    } else if (s.allowed === 0) {
+      lines.push(`${cat}: all ${total} BLOCKED`);
+    } else {
+      lines.push(`${cat}: ${s.blocked} blocked, ${s.allowed} allowed (of ${total})`);
+    }
+  }
+
+  return lines;
+}
+
 // Normalize the messy registry categories into clean buckets
 function normalizeCategory(raw: string): string {
   const lower = raw.toLowerCase();
@@ -117,10 +196,10 @@ function formatRobotsAudit(origin: string, robots: RobotsTxt, statusCode: number
   return lines.join("\n");
 }
 
-export function registerRobotsTool(server: McpServer): void {
+export function registerAiTool(server: McpServer): void {
   server.tool(
-    "robots",
-    "Fetch and analyze a site's robots.txt. Validates syntax per RFC 9309, audits AI crawler access (139+ bots), lists sitemaps. Shows blocked bots in detail, summarizes allowed.",
+    "ai",
+    "Analyze your site's AI visibility. Audits AI crawler access (training, search, assistant, agent) via robots.txt, checks for llms.txt and llms-full.txt, validates syntax per RFC 9309, and tests specific path access. Shows how AI systems see your site.",
     {
       url: z
         .string()
@@ -159,8 +238,21 @@ export function registerRobotsTool(server: McpServer): void {
           return { content: [{ type: "text", text: lines.join("\n") }] };
         }
 
-        const crawlers = await auditAiCrawlers(robotsTxt);
-        return { content: [{ type: "text", text: formatRobotsAudit(origin, robotsTxt, statusCode, crawlers) }] };
+        const [crawlers, llmsTxt, llmsFullTxt] = await Promise.all([
+          auditAiCrawlers(robotsTxt),
+          checkLlmsTxt(origin, "/llms.txt"),
+          checkLlmsTxt(origin, "/llms-full.txt"),
+        ]);
+
+        const output: string[] = [formatRobotsAudit(origin, robotsTxt, statusCode, crawlers)];
+
+        if (crawlers.length > 0) {
+          output.push(...formatCategorySummary(crawlers));
+        }
+
+        output.push("", ...formatLlmsTxt(llmsTxt, llmsFullTxt));
+
+        return { content: [{ type: "text", text: output.join("\n") }] };
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
         return { content: [{ type: "text", text: `Error analyzing robots.txt: ${msg}` }] };

package/src/tools/audit.ts

@@ -3,7 +3,7 @@ import { z } from "zod";
 import { inspectUrl, listSitemaps } from "../lib/gsc.js";
 import { type PsiCategoryType, type PsiResult, runPagespeed } from "../lib/psi.js";
 import { auditAiCrawlers, fetchRobotsTxt, isAllowed } from "../lib/robots.js";
-import { fetchSitemap, inspectSingle, sampleUrls } from "./sample-inspect.js";
+import { fetchSitemap, inspectSingle, sampleUrls } from "../lib/sitemap.js";
 
 type Severity = "HIGH" | "MEDIUM" | "LOW";