pagesight 0.12.0 → 0.12.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pagesight",
-  "version": "0.12.0",
+  "version": "0.12.2",
   "description": "See your site the way search engines and AI see it.",
   "keywords": [
     "seo",

package/src/index.ts

@@ -1,17 +1,12 @@
 #!/usr/bin/env bun
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { registerAiTool } from "./tools/ai.js";
 import { registerAuditTool } from "./tools/audit.js";
-import { registerCruxTool } from "./tools/crux.js";
-import { registerInspectTool } from "./tools/inspect.js";
-import { registerLinksTool } from "./tools/links.js";
-import { registerMetatagsTool } from "./tools/metatags.js";
-import { registerPagespeedTool } from "./tools/pagespeed.js";
-import { registerPerformanceTool } from "./tools/performance.js";
-import { registerRobotsTool } from "./tools/robots.js";
-import { registerSampleInspectTool } from "./tools/sample-inspect.js";
+import { registerPageTool } from "./tools/page.js";
+import { registerSearchTool } from "./tools/search.js";
 import { registerSetupTool } from "./tools/setup.js";
-import { registerSitemapsTool } from "./tools/sitemaps.js";
+import { registerSpeedTool } from "./tools/speed.js";
 
 const pkg = await Bun.file(new URL("../package.json", import.meta.url)).json();
 
@@ -20,17 +15,12 @@ const server = new McpServer({
   version: pkg.version,
 });
 
+registerAiTool(server);
 registerAuditTool(server);
-registerCruxTool(server);
-registerInspectTool(server);
-registerLinksTool(server);
-registerMetatagsTool(server);
-registerPagespeedTool(server);
-registerPerformanceTool(server);
-registerRobotsTool(server);
-registerSampleInspectTool(server);
-registerSitemapsTool(server);
+registerPageTool(server);
+registerSearchTool(server);
 registerSetupTool(server);
+registerSpeedTool(server);
 
 const transport = new StdioServerTransport();
 await server.connect(transport);

package/src/lib/psi.ts

@@ -79,6 +79,10 @@ export interface PsiResult {
 export type PsiStrategy = "mobile" | "desktop";
 export type PsiCategoryType = "performance" | "accessibility" | "best-practices" | "seo";
 
+export function hasApiKey(): boolean {
+  return !!process.env.GOOGLE_API_KEY;
+}
+
 export async function runPagespeed(
   url: string,
   options?: {

package/src/lib/sitemap.ts

@@ -0,0 +1,106 @@
+import { inspectUrl } from "./gsc.js";
+
+export interface SitemapParseResult {
+  urls: string[];
+  isSitemapIndex: boolean;
+  childSitemaps: string[];
+}
+
+export function parseSitemapXml(xml: string): SitemapParseResult {
+  const urls: string[] = [];
+  const childSitemaps: string[] = [];
+
+  const isSitemapIndex = /<sitemapindex/i.test(xml);
+
+  if (isSitemapIndex) {
+    for (const m of xml.matchAll(/<sitemap[^>]*>[\s\S]*?<loc>\s*(.*?)\s*<\/loc>[\s\S]*?<\/sitemap>/gi)) {
+      childSitemaps.push(m[1].trim());
+    }
+  } else {
+    for (const m of xml.matchAll(/<url[^>]*>[\s\S]*?<loc>\s*(.*?)\s*<\/loc>[\s\S]*?<\/url>/gi)) {
+      urls.push(m[1].trim());
+    }
+  }
+
+  return { urls, isSitemapIndex, childSitemaps };
+}
+
+export async function fetchSitemap(sitemapUrl: string): Promise<SitemapParseResult> {
+  const res = await fetch(sitemapUrl, {
+    headers: { "User-Agent": "Pagesight/1.0" },
+  });
+
+  if (!res.ok) {
+    throw new Error(`Failed to fetch sitemap ${sitemapUrl}: HTTP ${res.status}`);
+  }
+
+  const xml = await res.text();
+  return parseSitemapXml(xml);
+}
+
+export function sampleUrls(urls: string[], count: number, strategy: string): string[] {
+  if (urls.length <= count) return [...urls];
+
+  if (strategy === "first") {
+    return urls.slice(0, count);
+  }
+
+  if (strategy === "spread") {
+    const step = Math.floor(urls.length / count);
+    const sampled: string[] = [];
+    for (let i = 0; i < count; i++) {
+      sampled.push(urls[i * step]);
+    }
+    return sampled;
+  }
+
+  // random (default)
+  const shuffled = [...urls];
+  for (let i = shuffled.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+  }
+  return shuffled.slice(0, count);
+}
+
+export interface InspectionSummary {
+  url: string;
+  verdict: string;
+  coverageState: string;
+  pageFetchState: string;
+  robotsTxtState: string;
+  indexingState: string;
+  lastCrawlTime: string | null;
+  googleCanonical: string | null;
+  error: string | null;
+}
+
+export async function inspectSingle(url: string, siteUrl: string): Promise<InspectionSummary> {
+  try {
+    const r = await inspectUrl(url, siteUrl);
+    const idx = r.indexStatusResult;
+    return {
+      url,
+      verdict: idx.verdict,
+      coverageState: idx.coverageState,
+      pageFetchState: idx.pageFetchState,
+      robotsTxtState: idx.robotsTxtState,
+      indexingState: idx.indexingState,
+      lastCrawlTime: idx.lastCrawlTime ?? null,
+      googleCanonical: idx.googleCanonical ?? null,
+      error: null,
+    };
+  } catch (err) {
+    return {
+      url,
+      verdict: "ERROR",
+      coverageState: "ERROR",
+      pageFetchState: "ERROR",
+      robotsTxtState: "ERROR",
+      indexingState: "ERROR",
+      lastCrawlTime: null,
+      googleCanonical: null,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}

package/src/tools/{robots.ts → ai.ts}

@@ -117,10 +117,10 @@ function formatRobotsAudit(origin: string, robots: RobotsTxt, statusCode: number
   return lines.join("\n");
 }
 
-export function registerRobotsTool(server: McpServer): void {
+export function registerAiTool(server: McpServer): void {
   server.tool(
-    "robots",
-    "Fetch and analyze a site's robots.txt. Validates syntax per RFC 9309, audits AI crawler access (139+ bots), lists sitemaps. Shows blocked bots in detail, summarizes allowed.",
+    "ai",
+    "Analyze your site's AI visibility. Fetches robots.txt, audits which AI crawlers (training, search, assistant, agent) can access your content, validates syntax per RFC 9309, and checks specific paths. Shows how AI systems see your site.",
     {
       url: z
         .string()

package/src/tools/audit.ts

@@ -3,7 +3,7 @@ import { z } from "zod";
 import { inspectUrl, listSitemaps } from "../lib/gsc.js";
 import { type PsiCategoryType, type PsiResult, runPagespeed } from "../lib/psi.js";
 import { auditAiCrawlers, fetchRobotsTxt, isAllowed } from "../lib/robots.js";
-import { fetchSitemap, inspectSingle, sampleUrls } from "./sample-inspect.js";
+import { fetchSitemap, inspectSingle, sampleUrls } from "../lib/sitemap.js";
 
 type Severity = "HIGH" | "MEDIUM" | "LOW";
 
@@ -252,26 +252,28 @@ function addInspectFindings(verdict: string, coverageState: string, findings: Fi
 function formatAudit(url: string, findings: Finding[], errors: string[]): string {
   const lines: string[] = [`=== Site Audit: ${url} ===`, ""];
 
+  // Lead with errors so partial results are obvious
+  if (errors.length > 0) {
+    lines.push(`${errors.length} check${errors.length > 1 ? "s" : ""} failed — results below are partial:`, "");
+    for (const e of errors) {
+      lines.push(`  FAIL  ${e}`);
+    }
+    lines.push("");
+  }
+
   // Sort: HIGH first, then MEDIUM, then LOW
   const order: Record<string, number> = { HIGH: 0, MEDIUM: 1, LOW: 2 };
   findings.sort((a, b) => order[a.severity] - order[b.severity]);
 
-  if (findings.length === 0) {
+  if (findings.length === 0 && errors.length === 0) {
     lines.push("No issues found.");
-  } else {
-    lines.push(`${findings.length} findings:`, "");
+  } else if (findings.length > 0) {
+    lines.push(`${findings.length} finding${findings.length > 1 ? "s" : ""}:`, "");
     for (const f of findings) {
       lines.push(`${f.severity.padEnd(6)}  ${f.message}`);
     }
   }
 
-  if (errors.length > 0) {
-    lines.push("", "--- Errors (partial audit) ---", "");
-    for (const e of errors) {
-      lines.push(`- ${e}`);
-    }
-  }
-
   return lines.join("\n");
 }