pagesight 0.1.0

package/src/lib/robots.ts

@@ -0,0 +1,279 @@
+/**
+ * robots.txt parser per RFC 9309
+ * https://www.rfc-editor.org/rfc/rfc9309
+ *
+ * AI crawler registry sourced from:
+ * https://github.com/ai-robots-txt/ai.robots.txt (robots.json)
+ */
+
+export interface RobotsGroup {
+  userAgents: string[];
+  rules: Array<{ type: "allow" | "disallow"; path: string }>;
+}
+
+export interface RobotsTxt {
+  groups: RobotsGroup[];
+  sitemaps: string[];
+  raw: string;
+  errors: string[];
+}
+
+export interface CrawlerStatus {
+  name: string;
+  company: string;
+  category: string;
+  respectsRobotsTxt: string;
+  description: string;
+  allowed: boolean;
+  matchedRule?: { type: "allow" | "disallow"; path: string } | null;
+  matchedGroup?: string;
+}
+
+export interface CrawlerInfo {
+  token: string;
+  operator: string;
+  respect: string;
+  function: string;
+  description: string;
+}
+
+// --- Remote Registry ---
+
+const REGISTRY_URL = "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json";
+const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
+
+let cachedRegistry: CrawlerInfo[] | null = null;
+let cacheTimestamp = 0;
+
+function stripMarkdownLinks(text: string): string {
+  return text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
+}
+
+function parseRespect(raw: string): string {
+  const clean = stripMarkdownLinks(raw).trim().toLowerCase();
+  if (clean.startsWith("yes")) return "yes";
+  if (clean.startsWith("no")) return "no";
+  return "unclear";
+}
+
+async function fetchRegistry(): Promise<CrawlerInfo[]> {
+  if (cachedRegistry && Date.now() - cacheTimestamp < CACHE_TTL_MS) {
+    return cachedRegistry;
+  }
+
+  try {
+    const res = await fetch(REGISTRY_URL, {
+      headers: { "User-Agent": "Pagesight/0.1" },
+      signal: AbortSignal.timeout(10_000),
+    });
+
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+
+    const data = (await res.json()) as Record<string, Record<string, string>>;
+
+    cachedRegistry = Object.entries(data).map(([token, info]) => ({
+      token,
+      operator: stripMarkdownLinks(info.operator ?? "Unknown"),
+      respect: info.respect ?? "Unclear",
+      function: info.function ?? "Unknown",
+      description: stripMarkdownLinks(info.description ?? ""),
+    }));
+
+    cacheTimestamp = Date.now();
+    return cachedRegistry;
+  } catch {
+    // Fall back to cached or empty
+    return cachedRegistry ?? [];
+  }
+}
+
+// --- Parser ---
+
+export function parseRobotsTxt(raw: string): RobotsTxt {
+  const errors: string[] = [];
+  const groups: RobotsGroup[] = [];
+  const sitemaps: string[] = [];
+
+  let currentGroup: RobotsGroup | null = null;
+
+  const lines = raw.split(/\r?\n/);
+
+  for (let i = 0; i < lines.length; i++) {
+    const lineNum = i + 1;
+    let line = lines[i];
+
+    // Strip comments
+    const commentIdx = line.indexOf("#");
+    if (commentIdx !== -1) line = line.substring(0, commentIdx);
+    line = line.trim();
+
+    if (!line) continue;
+
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) {
+      errors.push(`Line ${lineNum}: Missing colon — "${line}"`);
+      continue;
+    }
+
+    const directive = line.substring(0, colonIdx).trim().toLowerCase();
+    const value = line.substring(colonIdx + 1).trim();
+
+    if (directive === "user-agent") {
+      if (!value) {
+        errors.push(`Line ${lineNum}: Empty user-agent value`);
+        continue;
+      }
+      if (!currentGroup || currentGroup.rules.length > 0) {
+        currentGroup = { userAgents: [value], rules: [] };
+        groups.push(currentGroup);
+      } else {
+        currentGroup.userAgents.push(value);
+      }
+    } else if (directive === "disallow") {
+      if (!currentGroup) {
+        errors.push(`Line ${lineNum}: Disallow before any User-agent`);
+        continue;
+      }
+      currentGroup.rules.push({ type: "disallow", path: value });
+    } else if (directive === "allow") {
+      if (!currentGroup) {
+        errors.push(`Line ${lineNum}: Allow before any User-agent`);
+        continue;
+      }
+      currentGroup.rules.push({ type: "allow", path: value });
+    } else if (directive === "sitemap") {
+      if (value) sitemaps.push(value);
+      else errors.push(`Line ${lineNum}: Empty sitemap URL`);
+    } else if (directive === "crawl-delay" || directive === "host") {
+      // Known non-standard directives — ignore silently
+    } else {
+      errors.push(`Line ${lineNum}: Unknown directive "${directive}"`);
+    }
+  }
+
+  return { groups, sitemaps, raw, errors };
+}
+
+// --- Matching (per RFC 9309) ---
+
+function pathMatches(pattern: string, path: string): boolean {
+  if (!pattern) return false;
+
+  let regex = "^";
+  for (let i = 0; i < pattern.length; i++) {
+    const c = pattern[i];
+    if (c === "*") {
+      regex += ".*";
+    } else if (c === "$" && i === pattern.length - 1) {
+      regex += "$";
+    } else {
+      regex += c.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    }
+  }
+  if (!pattern.endsWith("$")) regex += ".*";
+
+  try {
+    return new RegExp(regex).test(path);
+  } catch {
+    return false;
+  }
+}
+
+export function isAllowed(
+  robots: RobotsTxt,
+  userAgent: string,
+  path: string,
+): {
+  allowed: boolean;
+  matchedRule: { type: "allow" | "disallow"; path: string } | null;
+  matchedGroup: string | null;
+} {
+  const ua = userAgent.toLowerCase();
+
+  let matchingGroup: RobotsGroup | null = null;
+  let matchedGroupName: string | null = null;
+
+  for (const group of robots.groups) {
+    for (const agent of group.userAgents) {
+      if (agent.toLowerCase() === ua) {
+        matchingGroup = group;
+        matchedGroupName = agent;
+        break;
+      }
+    }
+    if (matchingGroup) break;
+  }
+
+  if (!matchingGroup) {
+    for (const group of robots.groups) {
+      if (group.userAgents.some((a) => a === "*")) {
+        matchingGroup = group;
+        matchedGroupName = "*";
+        break;
+      }
+    }
+  }
+
+  if (!matchingGroup) return { allowed: true, matchedRule: null, matchedGroup: null };
+
+  let bestRule: { type: "allow" | "disallow"; path: string } | null = null;
+  let bestLength = -1;
+
+  for (const rule of matchingGroup.rules) {
+    if (pathMatches(rule.path, path)) {
+      const ruleLength = rule.path.length;
+      if (ruleLength > bestLength || (ruleLength === bestLength && rule.type === "allow")) {
+        bestRule = rule;
+        bestLength = ruleLength;
+      }
+    }
+  }
+
+  if (!bestRule) return { allowed: true, matchedRule: null, matchedGroup: matchedGroupName };
+
+  return {
+    allowed: bestRule.type === "allow",
+    matchedRule: bestRule,
+    matchedGroup: matchedGroupName,
+  };
+}
+
+// --- AI Crawler Audit ---
+
+export async function auditAiCrawlers(robots: RobotsTxt): Promise<CrawlerStatus[]> {
+  const registry = await fetchRegistry();
+
+  return registry.map((crawler) => {
+    const result = isAllowed(robots, crawler.token, "/");
+    return {
+      name: crawler.token,
+      company: crawler.operator,
+      category: crawler.function,
+      respectsRobotsTxt: parseRespect(crawler.respect),
+      description: crawler.description.slice(0, 120),
+      allowed: result.allowed,
+      matchedRule: result.matchedRule,
+      matchedGroup: result.matchedGroup ?? undefined,
+    };
+  });
+}
+
+// --- Fetch ---
+
+export async function fetchRobotsTxt(origin: string): Promise<{ robotsTxt: RobotsTxt; statusCode: number }> {
+  const url = new URL("/robots.txt", origin).href;
+
+  const res = await fetch(url, {
+    headers: { "User-Agent": "Pagesight/0.1" },
+    redirect: "follow",
+  });
+
+  if (res.status >= 400) {
+    return { robotsTxt: { groups: [], sitemaps: [], raw: "", errors: [] }, statusCode: res.status };
+  }
+
+  const raw = await res.text();
+  return { robotsTxt: parseRobotsTxt(raw), statusCode: res.status };
+}
+
+export { fetchRegistry as loadCrawlerRegistry };

package/src/tools/crux.ts

@@ -0,0 +1,288 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import {
+  type CruxFormFactor,
+  type CruxHistoryResponse,
+  type CruxResponse,
+  queryCrux,
+  queryCruxHistory,
+} from "../lib/crux.js";
+
+function formatDate(d: { year: number; month: number; day: number }): string {
+  return `${d.year}-${String(d.month).padStart(2, "0")}-${String(d.day).padStart(2, "0")}`;
+}
+
+const METRIC_LABELS: Record<string, string> = {
+  cumulative_layout_shift: "CLS",
+  first_contentful_paint: "FCP",
+  interaction_to_next_paint: "INP",
+  largest_contentful_paint: "LCP",
+  experimental_time_to_first_byte: "TTFB",
+  round_trip_time: "RTT",
+  navigation_types: "Navigation Types",
+  form_factors: "Form Factors",
+};
+
+function formatCrux(target: string, result: CruxResponse): string {
+  const r = result.record;
+  const period = r.collectionPeriod;
+  const lines: string[] = [
+    `=== CrUX: ${target} ===`,
+    `Form factor: ${r.key.formFactor ?? "all"}`,
+    `Period: ${formatDate(period.firstDate)} to ${formatDate(period.lastDate)}`,
+    "",
+  ];
+
+  if (result.urlNormalizationDetails) {
+    const norm = result.urlNormalizationDetails;
+    if (norm.originalUrl !== norm.normalizedUrl) {
+      lines.push(`Normalized: ${norm.originalUrl} → ${norm.normalizedUrl}`, "");
+    }
+  }
+
+  lines.push("--- Metrics (p75) ---", "");
+
+  for (const [key, metric] of Object.entries(r.metrics)) {
+    const label = METRIC_LABELS[key] ?? key;
+
+    if (metric.percentiles) {
+      const val = metric.percentiles.p75;
+      const unit = key === "cumulative_layout_shift" ? "" : "ms";
+      lines.push(`${label}: ${val}${unit}`);
+
+      if (metric.histogram) {
+        const buckets = metric.histogram.map((b) => `${Math.round(b.density * 100)}%`).join(" / ");
+        lines.push(`  Distribution (good/needs improvement/poor): ${buckets}`);
+      }
+    } else if (metric.fractions) {
+      lines.push(`${label}:`);
+      for (const [fKey, fVal] of Object.entries(metric.fractions)) {
+        lines.push(`  ${fKey}: ${(fVal * 100).toFixed(1)}%`);
+      }
+    }
+  }
+
+  return lines.join("\n");
+}
+
+function formatCruxHistory(target: string, result: CruxHistoryResponse): string {
+  const r = result.record;
+  const periods = r.collectionPeriods;
+  const lines: string[] = [
+    `=== CrUX History: ${target} ===`,
+    `Form factor: ${r.key.formFactor ?? "all"}`,
+    `Periods: ${periods.length} (${formatDate(periods[0].firstDate)} to ${formatDate(periods[periods.length - 1].lastDate)})`,
+    "",
+  ];
+
+  if (result.urlNormalizationDetails) {
+    const norm = result.urlNormalizationDetails;
+    if (norm.originalUrl !== norm.normalizedUrl) {
+      lines.push(`Normalized: ${norm.originalUrl} → ${norm.normalizedUrl}`, "");
+    }
+  }
+
+  lines.push("--- p75 Trend ---", "");
+
+  for (const [key, metric] of Object.entries(r.metrics)) {
+    const label = METRIC_LABELS[key] ?? key;
+
+    if (metric.percentilesTimeseries) {
+      const values = metric.percentilesTimeseries.p75s;
+      const first = values[0];
+      const last = values[values.length - 1];
+      const unit = key === "cumulative_layout_shift" ? "" : "ms";
+
+      if (first === null && last === null) {
+        lines.push(`${label}: insufficient data`);
+        continue;
+      }
+
+      lines.push(
+        `${label}: ${first ?? "N/A"}${first !== null ? unit : ""} → ${last ?? "N/A"}${last !== null ? unit : ""} (${values.length} points)`,
+      );
+
+      // Show trend direction
+      if (first !== null && last !== null) {
+        const f = Number(first);
+        const l = Number(last);
+        if (!Number.isNaN(f) && !Number.isNaN(l)) {
+          const change = ((l - f) / f) * 100;
+          const dir = change > 5 ? "worse" : change < -5 ? "improved" : "stable";
+          lines.push(`  Trend: ${change > 0 ? "+" : ""}${change.toFixed(1)}% (${dir})`);
+        }
+      }
+    } else if (metric.fractionTimeseries) {
+      lines.push(`${label}: (fraction timeseries, ${periods.length} points)`);
+      for (const [fKey, fData] of Object.entries(metric.fractionTimeseries)) {
+        const fracs = fData.fractions;
+        const first = fracs[0];
+        const last = fracs[fracs.length - 1];
+        if (first !== null && last !== null && !Number.isNaN(first) && !Number.isNaN(last)) {
+          lines.push(`  ${fKey}: ${(first * 100).toFixed(1)}% → ${(last * 100).toFixed(1)}%`);
+        }
+      }
+    }
+  }
+
+  // Show last 5 data points as table for core metrics
+  const coreMetrics = ["largest_contentful_paint", "interaction_to_next_paint", "cumulative_layout_shift"];
+  const available = coreMetrics.filter((m) => r.metrics[m]?.percentilesTimeseries);
+
+  if (available.length > 0 && periods.length >= 5) {
+    lines.push("", "--- Recent Data Points ---", "");
+    const lastN = 5;
+    const startIdx = periods.length - lastN;
+
+    lines.push(`${"Date".padEnd(12)} ${available.map((m) => (METRIC_LABELS[m] ?? m).padEnd(10)).join(" ")}`);
+    for (let i = startIdx; i < periods.length; i++) {
+      const date = formatDate(periods[i].lastDate);
+      const vals = available.map((m) => {
+        const v = r.metrics[m].percentilesTimeseries?.p75s[i];
+        return String(v ?? "N/A").padEnd(10);
+      });
+      lines.push(`${date.padEnd(12)} ${vals.join(" ")}`);
+    }
+  }
+
+  return lines.join("\n");
+}
+
+export function registerCruxTool(server: McpServer): void {
+  server.tool(
+    "crux",
+    "Query Chrome UX Report (CrUX) for real-world Core Web Vitals data. Returns p75 metrics from actual Chrome users over a 28-day rolling window.",
+    {
+      url: z.string().optional().describe("Specific page URL. Provide either url or origin."),
+      origin: z.string().optional().describe("Origin URL (e.g., 'https://example.com'). Provide either url or origin."),
+      form_factor: z
+        .enum(["DESKTOP", "PHONE", "TABLET"])
+        .optional()
+        .describe("Filter by device type. Omit for all devices."),
+      metrics: z
+        .array(
+          z.enum([
+            "cumulative_layout_shift",
+            "first_contentful_paint",
+            "interaction_to_next_paint",
+            "largest_contentful_paint",
+            "experimental_time_to_first_byte",
+            "round_trip_time",
+            "navigation_types",
+            "form_factors",
+          ]),
+        )
+        .optional()
+        .describe("Specific metrics to return. Default: all available."),
+    },
+    async ({ url, origin, form_factor, metrics }) => {
+      if (!url && !origin) {
+        return { content: [{ type: "text", text: "Error: provide either url or origin." }] };
+      }
+      try {
+        const result = await queryCrux({
+          url,
+          origin,
+          formFactor: form_factor as CruxFormFactor | undefined,
+          metrics,
+        });
+        return { content: [{ type: "text", text: formatCrux(url ?? origin ?? "", result) }] };
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        if (msg.includes("404")) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `No CrUX data available for ${url ?? origin}. The page may not have enough Chrome user traffic.`,
+              },
+            ],
+          };
+        }
+        if (msg.includes("SERVICE_DISABLED") || msg.includes("API_KEY_SERVICE_BLOCKED")) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: "Chrome UX Report API is not enabled or the API key doesn't have access. Enable the API at: https://console.cloud.google.com/apis/library/chromeuxreport.googleapis.com — and ensure your API key allows it (Credentials > API key > API restrictions).",
+              },
+            ],
+          };
+        }
+        return { content: [{ type: "text", text: `Error querying CrUX: ${msg}` }] };
+      }
+    },
+  );
+
+  server.tool(
+    "crux_history",
+    "Query CrUX History API for Core Web Vitals trends over time. Returns up to 40 weekly data points (~10 months) of real-world performance data.",
+    {
+      url: z.string().optional().describe("Specific page URL. Provide either url or origin."),
+      origin: z.string().optional().describe("Origin URL (e.g., 'https://example.com'). Provide either url or origin."),
+      form_factor: z
+        .enum(["DESKTOP", "PHONE", "TABLET"])
+        .optional()
+        .describe("Filter by device type. Omit for all devices."),
+      metrics: z
+        .array(
+          z.enum([
+            "cumulative_layout_shift",
+            "first_contentful_paint",
+            "interaction_to_next_paint",
+            "largest_contentful_paint",
+            "experimental_time_to_first_byte",
+            "round_trip_time",
+            "navigation_types",
+            "form_factors",
+          ]),
+        )
+        .optional()
+        .describe("Specific metrics to return. Default: all available."),
+      periods: z
+        .number()
+        .min(1)
+        .max(40)
+        .optional()
+        .describe("Number of collection periods (1-40). Default: 25. Each is a 28-day window."),
+    },
+    async ({ url, origin, form_factor, metrics, periods }) => {
+      if (!url && !origin) {
+        return { content: [{ type: "text", text: "Error: provide either url or origin." }] };
+      }
+      try {
+        const result = await queryCruxHistory({
+          url,
+          origin,
+          formFactor: form_factor as CruxFormFactor | undefined,
+          metrics,
+          collectionPeriodCount: periods,
+        });
+        return { content: [{ type: "text", text: formatCruxHistory(url ?? origin ?? "", result) }] };
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        if (msg.includes("404")) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `No CrUX history data for ${url ?? origin}. The page may not have enough Chrome user traffic.`,
+              },
+            ],
+          };
+        }
+        if (msg.includes("SERVICE_DISABLED")) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: "Chrome UX Report API is not enabled. Enable it at: https://console.cloud.google.com/apis/library/chromeuxreport.googleapis.com",
+              },
+            ],
+          };
+        }
+        return { content: [{ type: "text", text: `Error querying CrUX History: ${msg}` }] };
+      }
+    },
+  );
+}

package/src/tools/inspect.ts

@@ -0,0 +1,105 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { type InspectionResult, inspectUrl } from "../lib/gsc.js";
+
+function formatInspection(url: string, siteUrl: string, r: InspectionResult): string {
+  const idx = r.indexStatusResult;
+  const lines: string[] = [
+    `=== URL Inspection: ${url} ===`,
+    `Property: ${siteUrl}`,
+    "",
+    "--- Index Status ---",
+    "",
+    `Verdict: ${idx.verdict}`,
+    `Coverage: ${idx.coverageState}`,
+    `Page fetch: ${idx.pageFetchState}`,
+    `Robots.txt: ${idx.robotsTxtState}`,
+    `Indexing: ${idx.indexingState}`,
+    `Crawled as: ${idx.crawledAs ?? "unknown"}`,
+  ];
+
+  if (idx.lastCrawlTime) lines.push(`Last crawled: ${idx.lastCrawlTime}`);
+  if (idx.userCanonical) lines.push(`Your canonical: ${idx.userCanonical}`);
+  if (idx.googleCanonical) lines.push(`Google's canonical: ${idx.googleCanonical}`);
+
+  if (idx.userCanonical && idx.googleCanonical && idx.userCanonical !== idx.googleCanonical) {
+    lines.push(`\n⚠ CANONICAL MISMATCH: You declared "${idx.userCanonical}" but Google chose "${idx.googleCanonical}"`);
+  }
+
+  if (idx.sitemap && idx.sitemap.length > 0) {
+    lines.push(`\nSitemaps: ${idx.sitemap.join(", ")}`);
+  }
+
+  if (idx.referringUrls && idx.referringUrls.length > 0) {
+    lines.push(`\nReferring URLs: ${idx.referringUrls.join(", ")}`);
+  }
+
+  // Rich Results
+  if (r.richResultsResult) {
+    lines.push("", "--- Rich Results ---", "");
+    lines.push(`Verdict: ${r.richResultsResult.verdict}`);
+
+    for (const item of r.richResultsResult.detectedItems ?? []) {
+      lines.push(`\nType: ${item.richResultType}`);
+      for (const instance of item.items ?? []) {
+        if (instance.name) lines.push(`  Name: ${instance.name}`);
+        const issues = instance.issues ?? [];
+        if (issues.length === 0) {
+          lines.push("  Status: PASS");
+        } else {
+          for (const issue of issues) {
+            lines.push(`  ${issue.severity}: ${issue.issueMessage}`);
+          }
+        }
+      }
+    }
+  }
+
+  // Mobile Usability (deprecated but still returned)
+  if (r.mobileUsabilityResult) {
+    lines.push("", "--- Mobile Usability (deprecated) ---", "");
+    lines.push(`Verdict: ${r.mobileUsabilityResult.verdict}`);
+    if (r.mobileUsabilityResult.issues) {
+      for (const issue of r.mobileUsabilityResult.issues) {
+        lines.push(`  ${issue.issueType}${issue.message ? `: ${issue.message}` : ""}`);
+      }
+    }
+  }
+
+  // Inspection link
+  if (r.inspectionResultLink) {
+    lines.push("", `Full report: ${r.inspectionResultLink}`);
+  }
+
+  return lines.join("\n");
+}
+
+export function registerInspectTool(server: McpServer): void {
+  server.tool(
+    "inspect",
+    "Inspect a URL using Google Search Console. Returns index status, canonical, crawl status, rich results validation, and more — directly from Google's index.",
+    {
+      url: z.string().url().describe("The URL to inspect"),
+      site_url: z.string().describe("The GSC property (e.g., 'https://example.com/' or 'sc-domain:example.com')"),
+    },
+    async ({ url, site_url }) => {
+      try {
+        const result = await inspectUrl(url, site_url);
+        return { content: [{ type: "text", text: formatInspection(url, site_url, result) }] };
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        if (msg.includes("PERMISSION_DENIED")) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Error: URL "${url}" is not part of property "${site_url}", or you don't have access.`,
+              },
+            ],
+          };
+        }
+        return { content: [{ type: "text", text: `Error inspecting URL: ${msg}` }] };
+      }
+    },
+  );
+}