pagesight 0.6.0 → 0.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pagesight",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "description": "See your site the way search engines and AI see it.",
   "keywords": [
     "seo",

package/src/index.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { registerAuditTool } from "./tools/audit.js";
 import { registerCruxTool } from "./tools/crux.js";
 import { registerInspectTool } from "./tools/inspect.js";
 import { registerMetatagsTool } from "./tools/metatags.js";
@@ -18,6 +19,7 @@ const server = new McpServer({
   version: pkg.version,
 });
 
+registerAuditTool(server);
 registerCruxTool(server);
 registerInspectTool(server);
 registerMetatagsTool(server);

package/src/tools/audit.ts

@@ -0,0 +1,330 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { inspectUrl, listSitemaps } from "../lib/gsc.js";
+import { type PsiCategoryType, type PsiResult, runPagespeed } from "../lib/psi.js";
+import { auditAiCrawlers, fetchRobotsTxt, isAllowed } from "../lib/robots.js";
+
+type Severity = "HIGH" | "MEDIUM" | "LOW";
+
+interface Finding {
+  severity: Severity;
+  message: string;
+  source: string;
+}
+
+// --- Metatags (inline, no external dep needed) ---
+
+interface MetaCheckResult {
+  title: string | null;
+  description: string | null;
+  canonical: string | null;
+  ogImage: string | null;
+  ogTitle: string | null;
+  twitterCard: string | null;
+  hasJsonLd: boolean;
+  redirectCount: number;
+  finalUrl: string;
+}
+
+async function checkMeta(url: string): Promise<MetaCheckResult> {
+  let current = url;
+  let redirectCount = 0;
+
+  for (let i = 0; i < 10; i++) {
+    const res = await fetch(current, {
+      headers: { "User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1)", Accept: "text/html" },
+      redirect: "manual",
+    });
+    if (res.status >= 300 && res.status < 400) {
+      const location = res.headers.get("location");
+      if (!location) break;
+      current = new URL(location, current).href;
+      redirectCount++;
+      continue;
+    }
+
+    const html = await res.text();
+    const head = html.match(/<head[^>]*>([\s\S]*?)<\/head>/i)?.[1] ?? html;
+
+    const getMeta = (key: string) => {
+      const nameMatch = head.match(new RegExp(`<meta[^>]*name=["']${key}["'][^>]*content=["']([^"']*?)["']`, "i"));
+      const propMatch = head.match(new RegExp(`<meta[^>]*property=["']${key}["'][^>]*content=["']([^"']*?)["']`, "i"));
+      return nameMatch?.[1] ?? propMatch?.[1] ?? null;
+    };
+
+    return {
+      title: head.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim() ?? null,
+      description: getMeta("description"),
+      canonical: head.match(/<link[^>]*rel=["']canonical["'][^>]*href=["']([^"']+)["']/i)?.[1] ?? null,
+      ogImage: getMeta("og:image"),
+      ogTitle: getMeta("og:title"),
+      twitterCard: getMeta("twitter:card"),
+      hasJsonLd: /<script[^>]*type=["']application\/ld\+json["']/i.test(html),
+      redirectCount,
+      finalUrl: current,
+    };
+  }
+
+  // Exhausted redirects
+  return {
+    title: null,
+    description: null,
+    canonical: null,
+    ogImage: null,
+    ogTitle: null,
+    twitterCard: null,
+    hasJsonLd: false,
+    redirectCount,
+    finalUrl: current,
+  };
+}
+
+// --- Audit logic ---
+
+function addMetaFindings(meta: MetaCheckResult, findings: Finding[]) {
+  if (!meta.title) findings.push({ severity: "HIGH", message: "Missing <title> tag", source: "metatags" });
+  if (!meta.description) findings.push({ severity: "HIGH", message: "Missing meta description", source: "metatags" });
+  if (!meta.canonical) findings.push({ severity: "HIGH", message: "Missing canonical URL", source: "metatags" });
+  if (!meta.ogImage)
+    findings.push({ severity: "MEDIUM", message: "Missing og:image — no social preview image", source: "metatags" });
+  if (!meta.ogTitle) findings.push({ severity: "MEDIUM", message: "Missing Open Graph tags", source: "metatags" });
+  if (!meta.twitterCard) findings.push({ severity: "LOW", message: "Missing Twitter Card tags", source: "metatags" });
+  if (!meta.hasJsonLd)
+    findings.push({ severity: "LOW", message: "No structured data (JSON-LD) found", source: "metatags" });
+  if (meta.redirectCount > 2)
+    findings.push({
+      severity: "MEDIUM",
+      message: `Redirect chain has ${meta.redirectCount} hops — consider reducing`,
+      source: "metatags",
+    });
+}
+
+function addPagespeedFindings(result: PsiResult, findings: Finding[]) {
+  const lhr = result.lighthouseResult;
+
+  for (const cat of Object.values(lhr.categories)) {
+    const score = cat.score !== null ? Math.round(cat.score * 100) : null;
+    if (score === null) continue;
+
+    if (cat.id === "performance" && score < 50) {
+      findings.push({ severity: "HIGH", message: `Performance score: ${score}/100`, source: "pagespeed" });
+    } else if (cat.id === "performance" && score < 90) {
+      findings.push({ severity: "MEDIUM", message: `Performance score: ${score}/100`, source: "pagespeed" });
+    }
+
+    if (cat.id === "accessibility" && score < 90) {
+      const sev = score < 70 ? "HIGH" : "MEDIUM";
+      findings.push({ severity: sev, message: `Accessibility score: ${score}/100`, source: "pagespeed" });
+    }
+
+    if (cat.id === "seo" && score < 90) {
+      findings.push({ severity: "MEDIUM", message: `SEO score: ${score}/100`, source: "pagespeed" });
+    }
+  }
+
+  // Flag specific high-impact audits
+  const renderBlocking = lhr.audits["render-blocking-insight"] ?? lhr.audits["render-blocking-resources"];
+  if (renderBlocking?.score === 0 && renderBlocking.details?.items) {
+    const fontItems = renderBlocking.details.items.filter((i) => i.url && String(i.url).includes("fonts"));
+    if (fontItems.length > 0) {
+      const wastedMs = fontItems.reduce((sum, i) => sum + (i.wastedMs ? Number(i.wastedMs) : 0), 0);
+      findings.push({
+        severity: "MEDIUM",
+        message: `Render-blocking Google Fonts (${Math.round(wastedMs)}ms wasted)`,
+        source: "pagespeed",
+      });
+    }
+  }
+
+  // FCP / LCP
+  const fcp = lhr.audits["first-contentful-paint"];
+  if (fcp?.numericValue && fcp.numericValue > 2500) {
+    findings.push({
+      severity: "MEDIUM",
+      message: `FCP ${(fcp.numericValue / 1000).toFixed(1)}s — above 2.5s threshold`,
+      source: "pagespeed",
+    });
+  }
+
+  const lcp = lhr.audits["largest-contentful-paint"];
+  if (lcp?.numericValue && lcp.numericValue > 2500) {
+    findings.push({
+      severity: lcp.numericValue > 4000 ? "HIGH" : "MEDIUM",
+      message: `LCP ${(lcp.numericValue / 1000).toFixed(1)}s — above 2.5s threshold`,
+      source: "pagespeed",
+    });
+  }
+}
+
+function addRobotsFindings(
+  robotsTxt: string | null,
+  blockedAiCrawlers: number,
+  totalAiCrawlers: number,
+  googlebotAllowed: boolean,
+  findings: Finding[],
+) {
+  if (robotsTxt === null) {
+    findings.push({ severity: "LOW", message: "No robots.txt found", source: "robots" });
+    return;
+  }
+  if (!googlebotAllowed) {
+    findings.push({ severity: "HIGH", message: "Googlebot is blocked by robots.txt", source: "robots" });
+  }
+  if (blockedAiCrawlers > 0) {
+    findings.push({
+      severity: "LOW",
+      message: `${blockedAiCrawlers}/${totalAiCrawlers} AI crawlers blocked`,
+      source: "robots",
+    });
+  }
+}
+
+function addSitemapFindings(sitemapCount: number, totalSubmitted: number, totalIndexed: number, findings: Finding[]) {
+  if (sitemapCount === 0) {
+    findings.push({ severity: "MEDIUM", message: "No sitemaps submitted to GSC", source: "sitemaps" });
+    return;
+  }
+  if (totalSubmitted > 0 && totalIndexed === 0) {
+    findings.push({
+      severity: "HIGH",
+      message: `${totalSubmitted.toLocaleString()} sitemap URLs submitted, 0 indexed`,
+      source: "sitemaps",
+    });
+  } else if (totalSubmitted > 0) {
+    const pct = Math.round((totalIndexed / totalSubmitted) * 100);
+    if (pct < 50) {
+      findings.push({
+        severity: "HIGH",
+        message: `${totalSubmitted.toLocaleString()} sitemap URLs, only ${pct}% indexed (${totalIndexed.toLocaleString()})`,
+        source: "sitemaps",
+      });
+    } else if (pct < 80) {
+      findings.push({
+        severity: "MEDIUM",
+        message: `${totalSubmitted.toLocaleString()} sitemap URLs, ${pct}% indexed (${totalIndexed.toLocaleString()})`,
+        source: "sitemaps",
+      });
+    }
+  }
+}
+
+function addInspectFindings(verdict: string, coverageState: string, findings: Finding[]) {
+  if (verdict === "FAIL") {
+    findings.push({ severity: "HIGH", message: `URL not indexed: ${coverageState}`, source: "inspect" });
+  } else if (verdict !== "PASS") {
+    findings.push({ severity: "MEDIUM", message: `Index status: ${coverageState}`, source: "inspect" });
+  }
+}
+
+function formatAudit(url: string, findings: Finding[], errors: string[]): string {
+  const lines: string[] = [`=== Site Audit: ${url} ===`, ""];
+
+  // Sort: HIGH first, then MEDIUM, then LOW
+  const order: Record<string, number> = { HIGH: 0, MEDIUM: 1, LOW: 2 };
+  findings.sort((a, b) => order[a.severity] - order[b.severity]);
+
+  if (findings.length === 0) {
+    lines.push("No issues found.");
+  } else {
+    lines.push(`${findings.length} findings:`, "");
+    for (const f of findings) {
+      lines.push(`${f.severity.padEnd(6)}  ${f.message}`);
+    }
+  }
+
+  if (errors.length > 0) {
+    lines.push("", "--- Errors (partial audit) ---", "");
+    for (const e of errors) {
+      lines.push(`- ${e}`);
+    }
+  }
+
+  return lines.join("\n");
+}
+
+export function registerAuditTool(server: McpServer): void {
+  server.tool(
+    "audit",
+    "Run a comprehensive site audit. Checks PageSpeed scores, meta tags, robots.txt, sitemaps, and index status in one call. Returns a prioritized list of findings.",
+    {
+      url: z.string().url().describe("The URL to audit (e.g., 'https://example.com')."),
+      site_url: z
+        .string()
+        .optional()
+        .describe("GSC property for index/sitemap checks (e.g., 'sc-domain:example.com'). Omit to skip GSC checks."),
+      strategy: z.enum(["mobile", "desktop"]).optional().describe("PageSpeed device strategy. Default: 'mobile'."),
+    },
+    async ({ url, site_url, strategy }) => {
+      const findings: Finding[] = [];
+      const errors: string[] = [];
+      const origin = new URL(url).origin;
+
+      // Run independent checks in parallel
+      const [metaResult, pagespeedResult, robotsResult, sitemapResult, inspectResult] = await Promise.allSettled([
+        checkMeta(url),
+        runPagespeed(url, {
+          strategy: (strategy as "mobile" | "desktop") ?? "mobile",
+          categories: ["performance", "accessibility", "seo", "best-practices"] as PsiCategoryType[],
+        }),
+        fetchRobotsTxt(origin).then(async ({ robotsTxt }) => {
+          const crawlers = await auditAiCrawlers(robotsTxt);
+          const { allowed: googlebotAllowed } = isAllowed(robotsTxt, "Googlebot", "/");
+          return { robotsTxt, crawlers, googlebotAllowed };
+        }),
+        site_url ? listSitemaps(site_url) : Promise.resolve(null),
+        site_url ? inspectUrl(url, site_url).catch(() => null) : Promise.resolve(null),
+      ]);
+
+      // Process meta
+      if (metaResult.status === "fulfilled") {
+        addMetaFindings(metaResult.value, findings);
+      } else {
+        errors.push(`Meta tags: ${metaResult.reason}`);
+      }
+
+      // Process pagespeed
+      if (pagespeedResult.status === "fulfilled") {
+        addPagespeedFindings(pagespeedResult.value, findings);
+      } else {
+        errors.push(`PageSpeed: ${pagespeedResult.reason}`);
+      }
+
+      // Process robots
+      if (robotsResult.status === "fulfilled") {
+        const { robotsTxt, crawlers, googlebotAllowed } = robotsResult.value;
+        const blocked = crawlers.filter((c) => !c.allowed).length;
+        addRobotsFindings(robotsTxt.raw || null, blocked, crawlers.length, googlebotAllowed, findings);
+      } else {
+        errors.push(`Robots.txt: ${robotsResult.reason}`);
+      }
+
+      // Process sitemaps
+      if (sitemapResult.status === "fulfilled" && sitemapResult.value) {
+        const sitemaps = sitemapResult.value;
+        let totalSubmitted = 0;
+        let totalIndexed = 0;
+        for (const sm of sitemaps) {
+          for (const c of sm.contents ?? []) {
+            totalSubmitted += Number(c.submitted ?? 0);
+            totalIndexed += Number(c.indexed ?? 0);
+          }
+        }
+        addSitemapFindings(sitemaps.length, totalSubmitted, totalIndexed, findings);
+      } else if (sitemapResult.status === "rejected") {
+        errors.push(`Sitemaps: ${sitemapResult.reason}`);
+      }
+
+      // Process inspect
+      if (inspectResult.status === "fulfilled" && inspectResult.value) {
+        const idx = inspectResult.value.indexStatusResult;
+        addInspectFindings(idx.verdict, idx.coverageState, findings);
+      } else if (inspectResult.status === "rejected") {
+        errors.push(`Inspect: ${inspectResult.reason}`);
+      }
+
+      return {
+        content: [{ type: "text", text: formatAudit(url, findings, errors) }],
+      };
+    },
+  );
+}

package/src/tools/metatags.ts

@@ -134,6 +134,168 @@ function formatJsonLd(data: unknown, indent = 0): string[] {
   return lines;
 }
 
+// --- Structured data validation (per Google Rich Results docs) ---
+
+interface SchemaRule {
+  required: string[];
+  recommended: string[];
+  imageFields: string[];
+}
+
+// Required/recommended fields sourced from Google's Rich Results documentation
+// https://developers.google.com/search/docs/appearance/structured-data
+const SCHEMA_RULES: Record<string, SchemaRule> = {
+  WebSite: {
+    required: ["name", "url"],
+    recommended: ["potentialAction"],
+    imageFields: [],
+  },
+  Organization: {
+    required: ["name", "url"],
+    recommended: ["logo", "sameAs", "contactPoint"],
+    imageFields: ["logo"],
+  },
+  Article: {
+    required: ["headline", "image", "datePublished", "author"],
+    recommended: ["dateModified", "publisher"],
+    imageFields: ["image"],
+  },
+  NewsArticle: {
+    required: ["headline", "image", "datePublished", "author"],
+    recommended: ["dateModified", "publisher"],
+    imageFields: ["image"],
+  },
+  Product: {
+    required: ["name", "image"],
+    recommended: ["description", "offers", "brand", "review", "aggregateRating"],
+    imageFields: ["image"],
+  },
+  LocalBusiness: {
+    required: ["name", "address"],
+    recommended: ["telephone", "openingHoursSpecification", "image", "url"],
+    imageFields: ["image"],
+  },
+  FAQPage: {
+    required: ["mainEntity"],
+    recommended: [],
+    imageFields: [],
+  },
+  BreadcrumbList: {
+    required: ["itemListElement"],
+    recommended: [],
+    imageFields: [],
+  },
+  Event: {
+    required: ["name", "startDate", "location"],
+    recommended: ["endDate", "image", "description", "offers", "organizer"],
+    imageFields: ["image"],
+  },
+  Recipe: {
+    required: ["name", "image"],
+    recommended: ["author", "datePublished", "description", "recipeIngredient", "recipeInstructions"],
+    imageFields: ["image"],
+  },
+  VideoObject: {
+    required: ["name", "description", "thumbnailUrl", "uploadDate"],
+    recommended: ["duration", "contentUrl", "embedUrl"],
+    imageFields: ["thumbnailUrl"],
+  },
+};
+
+interface ValidationIssue {
+  type: string;
+  level: "required" | "recommended";
+  field: string;
+}
+
+function getNestedValue(obj: Record<string, unknown>, field: string): unknown {
+  const val = obj[field];
+  if (val !== undefined && val !== null && val !== "") return val;
+  // Check if it's a nested object with a value (e.g., logo might be {url: "..."} or a string)
+  if (typeof val === "object" && val !== null) return val;
+  return undefined;
+}
+
+function validateJsonLd(blocks: unknown[]): { issues: ValidationIssue[]; imageUrls: string[] } {
+  const issues: ValidationIssue[] = [];
+  const imageUrls: string[] = [];
+
+  function validateBlock(data: unknown) {
+    if (Array.isArray(data)) {
+      for (const item of data) validateBlock(item);
+      return;
+    }
+    if (!data || typeof data !== "object") return;
+
+    const obj = data as Record<string, unknown>;
+    const rawType = obj["@type"];
+    const types = Array.isArray(rawType) ? rawType : rawType ? [rawType] : [];
+
+    for (const type of types) {
+      const rule = SCHEMA_RULES[String(type)];
+      if (!rule) continue;
+
+      for (const field of rule.required) {
+        if (getNestedValue(obj, field) === undefined) {
+          issues.push({ type: String(type), level: "required", field });
+        }
+      }
+
+      for (const field of rule.recommended) {
+        if (getNestedValue(obj, field) === undefined) {
+          issues.push({ type: String(type), level: "recommended", field });
+        }
+      }
+
+      // Collect image URLs for validation
+      for (const field of rule.imageFields) {
+        const val = obj[field];
+        if (typeof val === "string" && val.startsWith("http")) {
+          imageUrls.push(val);
+        } else if (val && typeof val === "object") {
+          const nested = val as Record<string, unknown>;
+          const url = nested.url ?? nested.contentUrl;
+          if (typeof url === "string" && url.startsWith("http")) {
+            imageUrls.push(url);
+          }
+        }
+      }
+    }
+
+    // Recurse into nested objects
+    for (const val of Object.values(obj)) {
+      if (val && typeof val === "object" && !Array.isArray(val)) {
+        const nested = val as Record<string, unknown>;
+        if (nested["@type"]) validateBlock(nested);
+      }
+    }
+  }
+
+  for (const block of blocks) validateBlock(block);
+  return { issues, imageUrls };
+}
+
+function formatValidation(issues: ValidationIssue[]): string[] {
+  if (issues.length === 0) return ["All validated types have their required fields."];
+
+  const lines: string[] = [];
+  const required = issues.filter((i) => i.level === "required");
+  const recommended = issues.filter((i) => i.level === "recommended");
+
+  if (required.length > 0) {
+    for (const i of required) {
+      lines.push(`MISSING  ${i.type}.${i.field} (required for Rich Results)`);
+    }
+  }
+  if (recommended.length > 0) {
+    for (const i of recommended) {
+      lines.push(`OPTIONAL ${i.type}.${i.field} (recommended)`);
+    }
+  }
+
+  return lines;
+}
+
 interface RedirectHop {
   url: string;
   status: number;
@@ -381,6 +543,31 @@ export function registerMetatagsTool(server: McpServer): void {
           }
         }
 
+        // Structured data validation
+        if (parsed.jsonLd.length > 0) {
+          const { issues, imageUrls } = validateJsonLd(parsed.jsonLd);
+          if (issues.length > 0 || imageUrls.length > 0) {
+            output.push("", "--- Structured Data Validation ---", "");
+            output.push(...formatValidation(issues));
+
+            // HEAD-check image URLs from structured data
+            if (imageUrls.length > 0) {
+              const deduped = [...new Set(imageUrls)];
+              const imgChecks = await Promise.all(deduped.map((u) => checkImage(u, "schema")));
+              for (const check of imgChecks) {
+                if (check.error) {
+                  output.push(`BROKEN   Image: ${check.url} — ${check.error}`);
+                } else if (check.status && check.status >= 400) {
+                  output.push(`BROKEN   Image: ${check.url} — HTTP ${check.status}`);
+                } else {
+                  const sizeKB = check.contentLength ? Math.round(check.contentLength / 1024) : null;
+                  output.push(`OK       Image: ${check.url}${sizeKB ? ` (${sizeKB} KB)` : ""}`);
+                }
+              }
+            }
+          }
+        }
+
         return {
           content: [{ type: "text", text: output.join("\n") }],
         };

package/src/tools/performance.ts

@@ -111,7 +111,7 @@ function formatComparison(
     `Clicks:      ${String(cur.clicks.toLocaleString()).padEnd(12)} ${String(prev.clicks.toLocaleString()).padEnd(12)} ${pctChange(cur.clicks, prev.clicks)}`,
     `Impressions: ${String(cur.impressions.toLocaleString()).padEnd(12)} ${String(prev.impressions.toLocaleString()).padEnd(12)} ${pctChange(cur.impressions, prev.impressions)}`,
     `Avg CTR:     ${`${(cur.ctr * 100).toFixed(1)}%`.padEnd(12)} ${`${(prev.ctr * 100).toFixed(1)}%`.padEnd(12)} ${((cur.ctr - prev.ctr) * 100).toFixed(1)}pp`,
-    `Avg Position:${String(cur.position.toFixed(1)).padEnd(13)} ${String(prev.position.toFixed(1)).padEnd(12)} ${cur.position < prev.position ? "improved" : cur.position > prev.position ? "regressed" : "stable"} (${(cur.position - prev.position).toFixed(1)})`,
+    `Avg Position:${String(cur.position.toFixed(1)).padEnd(13)} ${String(prev.position.toFixed(1)).padEnd(12)} ${prev.position === 0 && cur.position > 0 ? "new" : cur.position < prev.position ? "improved" : cur.position > prev.position ? "regressed" : "stable"} (${(cur.position - prev.position).toFixed(1)})`,
     "",
   ];