pagesight 0.3.1 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pagesight",
-  "version": "0.3.1",
+  "version": "0.5.0",
   "description": "See your site the way search engines and AI see it.",
   "keywords": [
     "seo",

package/src/index.ts

@@ -7,6 +7,7 @@ import { registerMetatagsTool } from "./tools/metatags.js";
 import { registerPagespeedTool } from "./tools/pagespeed.js";
 import { registerPerformanceTool } from "./tools/performance.js";
 import { registerRobotsTool } from "./tools/robots.js";
+import { registerSampleInspectTool } from "./tools/sample-inspect.js";
 import { registerSetupTool } from "./tools/setup.js";
 import { registerSitemapsTool } from "./tools/sitemaps.js";
 
@@ -23,6 +24,7 @@ registerMetatagsTool(server);
 registerPagespeedTool(server);
 registerPerformanceTool(server);
 registerRobotsTool(server);
+registerSampleInspectTool(server);
 registerSitemapsTool(server);
 registerSetupTool(server);
 

package/src/tools/crux.ts

@@ -193,14 +193,15 @@ export function registerCruxTool(server: McpServer): void {
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
         if (msg.includes("404")) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `No CrUX data available for ${url ?? origin}. The page may not have enough Chrome user traffic.`,
-              },
-            ],
-          };
+          const target = url ?? origin ?? "";
+          const lines = [`No CrUX data for ${target}.`, ""];
+          lines.push("CrUX requires sufficient Chrome user traffic (roughly 1,000+ monthly visits).");
+          if (url) {
+            const originUrl = new URL(url).origin;
+            lines.push(`Try origin-level data instead: origin "${originUrl}"`);
+          }
+          lines.push("For lab metrics without traffic requirements, use the pagespeed tool.");
+          return { content: [{ type: "text", text: lines.join("\n") }] };
         }
         if (msg.includes("SERVICE_DISABLED") || msg.includes("API_KEY_SERVICE_BLOCKED")) {
           return {
@@ -268,14 +269,15 @@ export function registerCruxTool(server: McpServer): void {
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
         if (msg.includes("404")) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `No CrUX history data for ${url ?? origin}. The page may not have enough Chrome user traffic.`,
-              },
-            ],
-          };
+          const target = url ?? origin ?? "";
+          const lines = [`No CrUX history data for ${target}.`, ""];
+          lines.push("CrUX requires sufficient Chrome user traffic (roughly 1,000+ monthly visits).");
+          if (url) {
+            const originUrl = new URL(url).origin;
+            lines.push(`Try origin-level data instead: origin "${originUrl}"`);
+          }
+          lines.push("For lab metrics without traffic requirements, use the pagespeed tool.");
+          return { content: [{ type: "text", text: lines.join("\n") }] };
         }
         if (msg.includes("SERVICE_DISABLED")) {
           return {

package/src/tools/metatags.ts

@@ -134,6 +134,77 @@ function formatJsonLd(data: unknown, indent = 0): string[] {
   return lines;
 }
 
+interface RedirectHop {
+  url: string;
+  status: number;
+}
+
+interface ImageCheck {
+  url: string;
+  tag: string;
+  status: number | null;
+  contentType: string | null;
+  contentLength: number | null;
+  error: string | null;
+}
+
+async function followRedirects(
+  url: string,
+  ua: string,
+  maxHops = 10,
+): Promise<{ chain: RedirectHop[]; response: Response }> {
+  const chain: RedirectHop[] = [];
+  let current = url;
+
+  for (let i = 0; i < maxHops; i++) {
+    const res = await fetch(current, {
+      headers: { "User-Agent": ua, Accept: "text/html" },
+      redirect: "manual",
+    });
+
+    chain.push({ url: current, status: res.status });
+
+    if (res.status >= 300 && res.status < 400) {
+      const location = res.headers.get("location");
+      if (!location) break;
+      current = new URL(location, current).href;
+      continue;
+    }
+
+    return { chain, response: res };
+  }
+
+  // If we exhausted hops, do a final follow-redirect fetch
+  const res = await fetch(current, {
+    headers: { "User-Agent": ua, Accept: "text/html" },
+    redirect: "follow",
+  });
+  return { chain, response: res };
+}
+
+async function checkImage(imageUrl: string, tag: string): Promise<ImageCheck> {
+  try {
+    const res = await fetch(imageUrl, { method: "HEAD", redirect: "follow" });
+    return {
+      url: imageUrl,
+      tag,
+      status: res.status,
+      contentType: res.headers.get("content-type"),
+      contentLength: res.headers.has("content-length") ? Number(res.headers.get("content-length")) : null,
+      error: null,
+    };
+  } catch (err) {
+    return {
+      url: imageUrl,
+      tag,
+      status: null,
+      contentType: null,
+      contentLength: null,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
 function formatMetatags(url: string, parsed: ParsedHead): string {
   const lines: string[] = [`=== Meta Tags: ${url} ===`, ""];
 
@@ -231,13 +302,7 @@ export function registerMetatagsTool(server: McpServer): void {
     async ({ url, user_agent }) => {
       try {
         const ua = user_agent ?? "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
-        const res = await fetch(url, {
-          headers: {
-            "User-Agent": ua,
-            Accept: "text/html",
-          },
-          redirect: "follow",
-        });
+        const { chain, response: res } = await followRedirects(url, ua);
 
         if (!res.ok) {
           return {
@@ -252,15 +317,50 @@ export function registerMetatagsTool(server: McpServer): void {
 
         const html = await res.text();
         const parsed = parseHead(html);
-        const finalUrl = res.url !== url ? `(redirected to ${res.url})\n\n` : "";
+        const output: string[] = [];
+
+        // Redirect chain (only if there were redirects)
+        if (chain.length > 1) {
+          output.push("--- Redirect Chain ---", "");
+          for (let i = 0; i < chain.length; i++) {
+            const hop = chain[i];
+            const prefix = i === chain.length - 1 ? "" : `${hop.status} → `;
+            output.push(`${i + 1}. ${prefix}${hop.url}`);
+          }
+          output.push("");
+        }
+
+        // Main meta tag report
+        const finalUrl = chain.length > 1 ? chain[chain.length - 1].url : url;
+        output.push(formatMetatags(finalUrl, parsed));
+
+        // OG/Twitter image validation
+        const ogImage = getMeta(parsed.meta, "og:image");
+        const twitterImage = getMeta(parsed.meta, "twitter:image");
+        const imagesToCheck: Array<{ url: string; tag: string }> = [];
+        if (ogImage) imagesToCheck.push({ url: ogImage, tag: "og:image" });
+        if (twitterImage && twitterImage !== ogImage) imagesToCheck.push({ url: twitterImage, tag: "twitter:image" });
+
+        if (imagesToCheck.length > 0) {
+          const checks = await Promise.all(imagesToCheck.map((img) => checkImage(img.url, img.tag)));
+          output.push("", "--- Image Validation ---", "");
+          for (const check of checks) {
+            if (check.error) {
+              output.push(`${check.tag}: FAILED — ${check.error}`);
+              output.push(`  URL: ${check.url}`);
+            } else if (check.status && check.status >= 400) {
+              output.push(`${check.tag}: BROKEN — HTTP ${check.status}`);
+              output.push(`  URL: ${check.url}`);
+            } else {
+              const size = check.contentLength ? ` (${Math.round(check.contentLength / 1024)} KB)` : "";
+              const type = check.contentType ? ` ${check.contentType}` : "";
+              output.push(`${check.tag}: OK —${type}${size}`);
+            }
+          }
+        }
 
         return {
-          content: [
-            {
-              type: "text",
-              text: `${finalUrl}${formatMetatags(res.url, parsed)}`,
-            },
-          ],
+          content: [{ type: "text", text: output.join("\n") }],
         };
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err);

package/src/tools/sample-inspect.ts

@@ -0,0 +1,275 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { inspectUrl, listSitemaps } from "../lib/gsc.js";
+
+interface SitemapParseResult {
+  urls: string[];
+  isSitemapIndex: boolean;
+  childSitemaps: string[];
+}
+
+function parseSitemapXml(xml: string): SitemapParseResult {
+  const urls: string[] = [];
+  const childSitemaps: string[] = [];
+
+  // Check if it's a sitemap index
+  const isSitemapIndex = /<sitemapindex/i.test(xml);
+
+  if (isSitemapIndex) {
+    for (const m of xml.matchAll(/<sitemap[^>]*>[\s\S]*?<loc>\s*(.*?)\s*<\/loc>[\s\S]*?<\/sitemap>/gi)) {
+      childSitemaps.push(m[1].trim());
+    }
+  } else {
+    for (const m of xml.matchAll(/<url[^>]*>[\s\S]*?<loc>\s*(.*?)\s*<\/loc>[\s\S]*?<\/url>/gi)) {
+      urls.push(m[1].trim());
+    }
+  }
+
+  return { urls, isSitemapIndex, childSitemaps };
+}
+
+async function fetchSitemap(sitemapUrl: string): Promise<SitemapParseResult> {
+  const res = await fetch(sitemapUrl, {
+    headers: { "User-Agent": "Pagesight/1.0" },
+  });
+
+  if (!res.ok) {
+    throw new Error(`Failed to fetch sitemap ${sitemapUrl}: HTTP ${res.status}`);
+  }
+
+  const xml = await res.text();
+  return parseSitemapXml(xml);
+}
+
+function sampleUrls(urls: string[], count: number, strategy: string): string[] {
+  if (urls.length <= count) return [...urls];
+
+  if (strategy === "first") {
+    return urls.slice(0, count);
+  }
+
+  if (strategy === "spread") {
+    const step = Math.floor(urls.length / count);
+    const sampled: string[] = [];
+    for (let i = 0; i < count; i++) {
+      sampled.push(urls[i * step]);
+    }
+    return sampled;
+  }
+
+  // random (default)
+  const shuffled = [...urls];
+  for (let i = shuffled.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+  }
+  return shuffled.slice(0, count);
+}
+
+interface InspectionSummary {
+  url: string;
+  verdict: string;
+  coverageState: string;
+  pageFetchState: string;
+  robotsTxtState: string;
+  indexingState: string;
+  lastCrawlTime: string | null;
+  googleCanonical: string | null;
+  error: string | null;
+}
+
+async function inspectSingle(url: string, siteUrl: string): Promise<InspectionSummary> {
+  try {
+    const r = await inspectUrl(url, siteUrl);
+    const idx = r.indexStatusResult;
+    return {
+      url,
+      verdict: idx.verdict,
+      coverageState: idx.coverageState,
+      pageFetchState: idx.pageFetchState,
+      robotsTxtState: idx.robotsTxtState,
+      indexingState: idx.indexingState,
+      lastCrawlTime: idx.lastCrawlTime ?? null,
+      googleCanonical: idx.googleCanonical ?? null,
+      error: null,
+    };
+  } catch (err) {
+    return {
+      url,
+      verdict: "ERROR",
+      coverageState: "ERROR",
+      pageFetchState: "ERROR",
+      robotsTxtState: "ERROR",
+      indexingState: "ERROR",
+      lastCrawlTime: null,
+      googleCanonical: null,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+function formatResults(siteUrl: string, sitemapUrl: string, totalUrls: number, results: InspectionSummary[]): string {
+  const lines: string[] = [
+    `=== Sample Inspection: ${siteUrl} ===`,
+    `Sitemap: ${sitemapUrl} (${totalUrls.toLocaleString()} URLs)`,
+    `Sampled: ${results.length}`,
+    "",
+  ];
+
+  // Summary
+  const verdictCounts: Record<string, number> = {};
+  const coverageCounts: Record<string, number> = {};
+  const fetchCounts: Record<string, number> = {};
+  let indexed = 0;
+  let errors = 0;
+
+  for (const r of results) {
+    if (r.error) {
+      errors++;
+      continue;
+    }
+    verdictCounts[r.verdict] = (verdictCounts[r.verdict] ?? 0) + 1;
+    coverageCounts[r.coverageState] = (coverageCounts[r.coverageState] ?? 0) + 1;
+    fetchCounts[r.pageFetchState] = (fetchCounts[r.pageFetchState] ?? 0) + 1;
+    if (r.verdict === "PASS") indexed++;
+  }
+
+  const inspected = results.length - errors;
+  lines.push("--- Summary ---", "");
+  lines.push(`Indexed: ${indexed}/${inspected}`);
+
+  if (indexed < inspected) {
+    lines.push(`Not indexed: ${inspected - indexed}/${inspected}`);
+    for (const [state, count] of Object.entries(coverageCounts)) {
+      if (state !== "Submitted and indexed" && state !== "Indexing allowed") {
+        lines.push(`  ${state}: ${count}`);
+      }
+    }
+  }
+
+  // Page fetch issues
+  const fetchIssues = Object.entries(fetchCounts).filter(([s]) => s !== "SUCCESSFUL");
+  if (fetchIssues.length > 0) {
+    lines.push("");
+    lines.push("Page fetch issues:");
+    for (const [state, count] of fetchIssues) {
+      lines.push(`  ${state}: ${count}`);
+    }
+  }
+
+  if (errors > 0) {
+    lines.push(`\nInspection errors: ${errors}`);
+  }
+
+  // Individual results
+  lines.push("", "--- Details ---", "");
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i];
+    lines.push(`${i + 1}. ${r.url}`);
+    if (r.error) {
+      lines.push(`   Error: ${r.error}`);
+    } else {
+      lines.push(`   Verdict: ${r.verdict}`);
+      lines.push(`   Coverage: ${r.coverageState}`);
+      lines.push(`   Page fetch: ${r.pageFetchState}`);
+      if (r.robotsTxtState !== "ALLOWED") lines.push(`   Robots.txt: ${r.robotsTxtState}`);
+      if (r.indexingState !== "INDEXING_ALLOWED") lines.push(`   Indexing: ${r.indexingState}`);
+      if (r.lastCrawlTime) lines.push(`   Last crawled: ${r.lastCrawlTime}`);
+      if (r.googleCanonical && r.googleCanonical !== r.url) {
+        lines.push(`   Google canonical: ${r.googleCanonical}`);
+      }
+    }
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}
+
+export function registerSampleInspectTool(server: McpServer): void {
+  server.tool(
+    "sample_inspect",
+    "Sample URLs from a sitemap and batch-inspect them via Google Search Console. Diagnoses indexing issues by revealing patterns across multiple URLs — why pages aren't indexed, common fetch errors, robots.txt blocks.",
+    {
+      site_url: z.string().describe("GSC property (e.g., 'https://example.com/' or 'sc-domain:example.com')"),
+      sitemap_url: z
+        .string()
+        .url()
+        .optional()
+        .describe("Sitemap URL to sample from. If omitted, discovers sitemaps from GSC."),
+      sample_size: z.number().min(1).max(10).optional().describe("Number of URLs to inspect. Default: 5. Max: 10."),
+      strategy: z
+        .enum(["random", "first", "spread"])
+        .optional()
+        .describe("Sampling strategy. 'random' (default), 'first' (first N), 'spread' (evenly spaced)."),
+    },
+    async ({ site_url, sitemap_url, sample_size, strategy }) => {
+      const count = sample_size ?? 5;
+      const sampleStrategy = strategy ?? "random";
+
+      try {
+        // Discover sitemap URL if not provided
+        let resolvedSitemapUrl = sitemap_url;
+        if (!resolvedSitemapUrl) {
+          const sitemaps = await listSitemaps(site_url);
+          if (sitemaps.length === 0) {
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: `No sitemaps found for ${site_url} in GSC. Provide a sitemap_url directly.`,
+                },
+              ],
+            };
+          }
+          // Pick the first non-index sitemap, or the first one
+          const nonIndex = sitemaps.find((s) => !s.isSitemapsIndex);
+          resolvedSitemapUrl = (nonIndex ?? sitemaps[0]).path;
+        }
+
+        // Fetch and parse sitemap
+        let parsed = await fetchSitemap(resolvedSitemapUrl);
+
+        // If it's a sitemap index, fetch the first child
+        if (parsed.isSitemapIndex && parsed.childSitemaps.length > 0) {
+          const childUrl = parsed.childSitemaps[0];
+          parsed = await fetchSitemap(childUrl);
+          resolvedSitemapUrl = `${resolvedSitemapUrl} → ${childUrl}`;
+        }
+
+        if (parsed.urls.length === 0) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Sitemap ${resolvedSitemapUrl} contains no URLs.`,
+              },
+            ],
+          };
+        }
+
+        // Sample URLs
+        const sampled = sampleUrls(parsed.urls, count, sampleStrategy);
+
+        // Inspect each URL sequentially (API rate limits)
+        const results: InspectionSummary[] = [];
+        for (const url of sampled) {
+          results.push(await inspectSingle(url, site_url));
+        }
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: formatResults(site_url, resolvedSitemapUrl, parsed.urls.length, results),
+            },
+          ],
+        };
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return {
+          content: [{ type: "text", text: `Error: ${msg}` }],
+        };
+      }
+    },
+  );
+}