pagesight 0.16.0 → 0.18.0

package/src/tools/audit.ts

@@ -1,9 +1,9 @@
 import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
-import { inspectUrl, listSitemaps } from "../lib/gsc.js";
-import { type PsiCategoryType, type PsiResult, runPagespeed } from "../lib/psi.js";
-import { auditAiCrawlers, fetchRobotsTxt, isAllowed } from "../lib/robots.js";
-import { fetchSitemap, inspectSingle, sampleUrls } from "../lib/sitemap.js";
+import { type GscSitemap, inspectUrl, listSitemaps } from "../providers/gsc.js";
+import { RequestError } from "../shared/http.js";
+import { type PsiCategoryType, type PsiResult, runPagespeed } from "../providers/pagespeed.js";
+import { auditAiCrawlers, fetchRobotsTxt, isAllowed } from "../web/robots.js";
 
 type Severity = "HIGH" | "MEDIUM" | "LOW";
 
@@ -13,8 +13,6 @@ interface Finding {
   source: string;
 }
 
-// --- Metatags (inline, no external dep needed) ---
-
 interface MetaCheckResult {
   title: string | null;
   description: string | null;
@@ -66,7 +64,7 @@ async function checkMeta(url: string): Promise<MetaCheckResult> {
     };
   }
 
-  // Exhausted redirects
+  // The hop limit or a redirect without Location prevented a page response.
   return {
     title: null,
     description: null,
@@ -80,8 +78,6 @@ async function checkMeta(url: string): Promise<MetaCheckResult> {
   };
 }
 
-// --- Audit logic ---
-
 function addMetaFindings(meta: MetaCheckResult, findings: Finding[]) {
   if (!meta.title) findings.push({ severity: "HIGH", message: "Missing <title> tag", source: "metatags" });
   if (!meta.description) findings.push({ severity: "HIGH", message: "Missing meta description", source: "metatags" });
@@ -182,70 +178,30 @@ function addRobotsFindings(
   }
 }
 
-function addSitemapFindings(sitemapCount: number, totalSubmitted: number, totalIndexed: number, findings: Finding[]) {
-  if (sitemapCount === 0) {
+export function addSitemapFindings(sitemaps: GscSitemap[], findings: Finding[]) {
+  if (sitemaps.length === 0) {
     findings.push({ severity: "MEDIUM", message: "No sitemaps submitted to GSC", source: "sitemaps" });
-    return;
   }
-  if (totalSubmitted > 0 && totalIndexed === 0) {
-    findings.push({
-      severity: "HIGH",
-      message: `${totalSubmitted.toLocaleString()} sitemap URLs submitted, 0 indexed`,
-      source: "sitemaps",
-    });
-  } else if (totalSubmitted > 0) {
-    const pct = Math.round((totalIndexed / totalSubmitted) * 100);
-    if (pct < 50) {
+  // contents[].indexed is deprecated; sitemap submission is not indexing evidence.
+  // https://developers.google.com/webmaster-tools/v1/sitemaps
+  for (const sitemap of sitemaps) {
+    if (Number(sitemap.errors) > 0) {
       findings.push({
         severity: "HIGH",
-        message: `${totalSubmitted.toLocaleString()} sitemap URLs, only ${pct}% indexed (${totalIndexed.toLocaleString()})`,
+        message: `${sitemap.path}: ${sitemap.errors} sitemap errors`,
         source: "sitemaps",
       });
-    } else if (pct < 80) {
+    }
+    if (Number(sitemap.warnings) > 0) {
       findings.push({
         severity: "MEDIUM",
-        message: `${totalSubmitted.toLocaleString()} sitemap URLs, ${pct}% indexed (${totalIndexed.toLocaleString()})`,
+        message: `${sitemap.path}: ${sitemap.warnings} sitemap warnings`,
         source: "sitemaps",
       });
     }
   }
 }
 
-function formatDrillDown(
-  inspections: Array<{ url: string; verdict: string; coverageState: string; error: string | null }>,
-): string {
-  const valid = inspections.filter((r) => !r.error);
-  if (valid.length === 0) return "";
-
-  const indexed = valid.filter((r) => r.verdict === "PASS").length;
-  const lines: string[] = [`        Auto-inspected ${valid.length} URLs:`];
-  lines.push(`        - ${indexed}/${valid.length} indexed`);
-
-  const stateCounts: Record<string, { count: number; urls: string[] }> = {};
-  for (const r of valid) {
-    if (r.verdict !== "PASS") {
-      let path: string;
-      try {
-        path = new URL(r.url).pathname;
-      } catch {
-        path = r.url;
-      }
-      const existing = stateCounts[r.coverageState];
-      if (existing) {
-        existing.count++;
-        existing.urls.push(path);
-      } else {
-        stateCounts[r.coverageState] = { count: 1, urls: [path] };
-      }
-    }
-  }
-  for (const [state, { count, urls }] of Object.entries(stateCounts)) {
-    lines.push(`        - ${count}/${valid.length} ${state}: ${urls.join(", ")}`);
-  }
-
-  return lines.join("\n");
-}
-
 function addInspectFindings(verdict: string, coverageState: string, findings: Finding[]) {
   if (verdict === "FAIL") {
     findings.push({ severity: "HIGH", message: `URL not indexed: ${coverageState}`, source: "inspect" });
@@ -266,7 +222,6 @@ function formatAudit(url: string, findings: Finding[], errors: string[]): string
     lines.push("");
   }
 
-  // Sort: HIGH first, then MEDIUM, then LOW
   const order: Record<string, number> = { HIGH: 0, MEDIUM: 1, LOW: 2 };
   findings.sort((a, b) => order[a.severity] - order[b.severity]);
 
@@ -299,7 +254,6 @@ export function registerAuditTool(server: McpServer): void {
       const errors: string[] = [];
       const origin = new URL(url).origin;
 
-      // Run independent checks in parallel
       const [metaResult, pagespeedResult, robotsResult, sitemapResult, inspectResult] = await Promise.allSettled([
         checkMeta(url),
         runPagespeed(url, {
@@ -312,7 +266,7 @@ export function registerAuditTool(server: McpServer): void {
           return { robotsTxt, crawlers, googlebotAllowed };
         }),
         site_url ? listSitemaps(site_url) : Promise.resolve(null),
-        site_url ? inspectUrl(url, site_url).catch(() => null) : Promise.resolve(null),
+        site_url ? inspectUrl(url, site_url) : Promise.resolve(null),
       ]);
 
       // Process meta
@@ -326,7 +280,15 @@ export function registerAuditTool(server: McpServer): void {
       if (pagespeedResult.status === "fulfilled") {
         addPagespeedFindings(pagespeedResult.value, findings);
       } else {
-        errors.push(`PageSpeed: ${pagespeedResult.reason}`);
+        const psiErr = String(pagespeedResult.reason);
+        const status = pagespeedResult.reason instanceof RequestError ? pagespeedResult.reason.status : null;
+        if (status === 403) {
+          errors.push("PageSpeed: SKIPPED (API key not authorized — enable PageSpeed Insights API in Google Cloud)");
+        } else if (status === 429) {
+          errors.push("PageSpeed: SKIPPED (rate limited — try again later or set GOOGLE_API_KEY)");
+        } else {
+          errors.push(`PageSpeed: ${psiErr.replace(/:\s*\{[\s\S]*$/, "")}`);
+        }
       }
 
       // Process robots
@@ -340,40 +302,7 @@ export function registerAuditTool(server: McpServer): void {
 
       // Process sitemaps
       if (sitemapResult.status === "fulfilled" && sitemapResult.value) {
-        const sitemaps = sitemapResult.value;
-        let totalSubmitted = 0;
-        let totalIndexed = 0;
-        for (const sm of sitemaps) {
-          for (const c of sm.contents ?? []) {
-            totalSubmitted += Number(c.submitted ?? 0);
-            totalIndexed += Number(c.indexed ?? 0);
-          }
-        }
-        addSitemapFindings(sitemaps.length, totalSubmitted, totalIndexed, findings);
-
-        // Auto-drill-down: when indexing is low, sample-inspect to explain why
-        const indexPct = totalSubmitted > 0 ? (totalIndexed / totalSubmitted) * 100 : 100;
-        if (site_url && indexPct < 50 && sitemaps.length > 0) {
-          try {
-            const sitemapPath = (sitemaps.find((s) => !s.isSitemapsIndex) ?? sitemaps[0]).path;
-            let parsed = await fetchSitemap(sitemapPath);
-            if (parsed.isSitemapIndex && parsed.childSitemaps.length > 0) {
-              parsed = await fetchSitemap(parsed.childSitemaps[0]);
-            }
-            if (parsed.urls.length > 0) {
-              const sampled = sampleUrls(parsed.urls, 5, "spread");
-              const inspections = await Promise.all(sampled.map((u) => inspectSingle(u, site_url)));
-              const drillDown = formatDrillDown(inspections);
-              // Append drill-down to the sitemap finding
-              const sitemapFinding = findings.find((f) => f.source === "sitemaps" && f.severity === "HIGH");
-              if (sitemapFinding) {
-                sitemapFinding.message += `\n${drillDown}`;
-              }
-            }
-          } catch {
-            // Drill-down is best-effort — don't fail the audit
-          }
-        }
+        addSitemapFindings(sitemapResult.value, findings);
       } else if (sitemapResult.status === "rejected") {
         errors.push(`Sitemaps: ${sitemapResult.reason}`);
       }

package/src/tools/observe.ts

@@ -0,0 +1,28 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { execute, operationSchema } from "../api/index.js";
+import { RequestError } from "../shared/http.js";
+
+export function registerObserveTool(server: McpServer): void {
+  server.tool(
+    "observe",
+    "Run the shared Pagesight API. Read-only operations: discover; bing.sites/queries/pages/traffic/crawl-stats/crawl-issues/url-info/link-counts/url-links; evidence.import (unverified UI findings); gsc.sites/sitemaps/inspect/report; ga.accounts/property/key-events/report; page (metadata and image ALT evidence); speed.psi/crux/history; doctor; snapshot; compare. Pass the API operation object as request. Returns structured evidence, exact effective requests, provider responses and partial errors. See the CLI --help and README for report/config shapes.",
+    { request: operationSchema },
+    async ({ request }) => {
+      try {
+        const result = await execute(request);
+        return {
+          content: [{ type: "text" as const, text: JSON.stringify(result) }],
+          structuredContent: { ...result },
+          isError: result.status === "error",
+        };
+      } catch (error) {
+        return {
+          content: [
+            { type: "text" as const, text: error instanceof RequestError ? error.message : "Invalid operation" },
+          ],
+          isError: true,
+        };
+      }
+    },
+  );
+}

package/src/tools/page/analyze.ts

@@ -0,0 +1,194 @@
+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+import { parseHead, getMeta, formatMetatags } from "./metadata.js";
+import { validateJsonLd, formatValidation } from "./structured-data.js";
+import {
+  followRedirects,
+  checkImage,
+  checkLink,
+  extractInternalLinks,
+  formatLinkResults,
+  type LinkResult,
+} from "./links.js";
+import { parseHex, relativeLuminance, contrastRatio, wcagLevel, toHex, findNearestPassing } from "./contrast.js";
+
+export async function analyzePage(
+  url: string,
+  options: {
+    check_links?: boolean;
+    user_agent?: string;
+    foreground?: string;
+    background?: string;
+    large_text?: boolean;
+  },
+): Promise<CallToolResult> {
+  const { check_links, user_agent, foreground, background, large_text } = options;
+  try {
+    const ua = user_agent ?? "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
+    const { chain, response: res } = await followRedirects(url, ua);
+
+    if (!res.ok) {
+      return {
+        content: [
+          {
+            type: "text",
+            text: `Error fetching ${url}: HTTP ${res.status} ${res.statusText}`,
+          },
+        ],
+      };
+    }
+
+    const html = await res.text();
+    const parsed = parseHead(html);
+    const output: string[] = [];
+
+    // Redirect chain (only if there were redirects)
+    if (chain.length > 1) {
+      output.push("--- Redirect Chain ---", "");
+      const parts: string[] = [];
+      for (const hop of chain) {
+        parts.push(hop.url);
+        parts.push(String(hop.status));
+      }
+      // Format: URL → status → URL → status (drop trailing status)
+      parts.pop();
+      output.push(parts.join(" → "));
+      output.push(`Hops: ${chain.length - 1}`);
+      output.push("");
+    }
+
+    // Main meta tag report
+    const finalUrl = chain.length > 1 ? chain[chain.length - 1].url : url;
+    output.push(formatMetatags(finalUrl, parsed));
+
+    // OG/Twitter image validation
+    const ogImage = getMeta(parsed.meta, "og:image");
+    const twitterImage = getMeta(parsed.meta, "twitter:image");
+    const imagesToCheck: Array<{ url: string; tag: string }> = [];
+    if (ogImage) imagesToCheck.push({ url: ogImage, tag: "og:image" });
+    if (twitterImage && twitterImage !== ogImage) imagesToCheck.push({ url: twitterImage, tag: "twitter:image" });
+
+    if (imagesToCheck.length > 0) {
+      const checks = await Promise.all(imagesToCheck.map((img) => checkImage(img.url, img.tag)));
+      output.push("", "--- Image Validation ---", "");
+      for (const check of checks) {
+        if (check.error) {
+          output.push(`${check.tag}: FAILED — ${check.error}`);
+          output.push(`  URL: ${check.url}`);
+        } else if (check.status && check.status >= 400) {
+          output.push(`${check.tag}: BROKEN — HTTP ${check.status}`);
+          output.push(`  URL: ${check.url}`);
+        } else {
+          const sizeKB = check.contentLength ? Math.round(check.contentLength / 1024) : null;
+          const size = sizeKB ? ` (${sizeKB} KB)` : "";
+          const type = check.contentType ? ` ${check.contentType}` : "";
+          output.push(`${check.tag}: OK —${type}${size}`);
+          if (sizeKB && sizeKB > 1024) {
+            output.push(`  Warning: ${sizeKB} KB is large for social previews — consider compressing below 1 MB`);
+          }
+        }
+
+        // Check declared dimensions from meta tags
+        if (check.tag === "og:image") {
+          const w = getMeta(parsed.meta, "og:image:width");
+          const h = getMeta(parsed.meta, "og:image:height");
+          if (w && h) {
+            output.push(`  Declared dimensions: ${w}x${h}`);
+            const wn = Number(w);
+            const hn = Number(h);
+            if (wn && hn && (wn < 1200 || hn < 630)) {
+              output.push("  Note: recommended minimum for og:image is 1200x630");
+            }
+          }
+        }
+      }
+    }
+
+    // Structured data validation
+    if (parsed.jsonLd.length > 0) {
+      const { issues, imageUrls, validatedTypes } = validateJsonLd(parsed.jsonLd);
+      if (validatedTypes.size > 0 || imageUrls.length > 0) {
+        output.push("", "--- Structured Data Validation ---", "");
+        output.push(...formatValidation(issues, validatedTypes));
+
+        // HEAD-check image URLs from structured data
+        if (imageUrls.length > 0) {
+          const deduped = [...new Set(imageUrls)];
+          const imgChecks = await Promise.all(deduped.map((u) => checkImage(u, "schema")));
+          for (const check of imgChecks) {
+            if (check.error) {
+              output.push(`BROKEN   Image: ${check.url} — ${check.error}`);
+            } else if (check.status && check.status >= 400) {
+              output.push(`BROKEN   Image: ${check.url} — HTTP ${check.status}`);
+            } else {
+              const sizeKB = check.contentLength ? Math.round(check.contentLength / 1024) : null;
+              output.push(`OK       Image: ${check.url}${sizeKB ? ` (${sizeKB} KB)` : ""}`);
+            }
+          }
+        }
+      }
+    }
+
+    if (foreground && background) {
+      const fg = parseHex(foreground);
+      const bg = parseHex(background);
+      if (fg && bg) {
+        const fgLum = relativeLuminance(...fg);
+        const bgLum = relativeLuminance(...bg);
+        const ratio = contrastRatio(fgLum, bgLum);
+        const isLarge = large_text ?? false;
+        const level = wcagLevel(ratio, isLarge);
+        const aaThreshold = isLarge ? 3 : 4.5;
+        const aaaThreshold = isLarge ? 4.5 : 7;
+        output.push(
+          "",
+          "--- Contrast Check ---",
+          "",
+          `Foreground: ${toHex(...fg)}  Background: ${toHex(...bg)}`,
+          `Ratio: ${ratio.toFixed(2)}:1  AA: ${ratio >= aaThreshold ? "PASS" : "FAIL"}  AAA: ${ratio >= aaaThreshold ? "PASS" : "FAIL"}  Result: ${level}`,
+        );
+        if (ratio < aaThreshold) {
+          const suggested = findNearestPassing(fg, bg, aaThreshold);
+          output.push(`Nearest AA-passing foreground: ${suggested}`);
+        }
+      }
+    }
+
+    // --- Link check (last section) ---
+    if (check_links !== false) {
+      const origin = new URL(url).origin;
+      const links = extractInternalLinks(html, origin);
+
+      if (links.length === 0) {
+        output.push(
+          "",
+          "--- Internal Links ---",
+          "",
+          "No internal links found.",
+          "Note: this page may use client-side rendering (SPA). Only static <a href> links in the HTML are detected.",
+        );
+      } else {
+        const concurrency = 5;
+        const results: LinkResult[] = [];
+        let i = 0;
+
+        while (i < links.length) {
+          const batch = links.slice(i, i + concurrency);
+          const settled = await Promise.all(batch.map((href) => checkLink(href)));
+          results.push(...settled);
+          i += concurrency;
+        }
+
+        output.push("", formatLinkResults(url, results));
+      }
+    }
+
+    return {
+      content: [{ type: "text", text: output.join("\n") }],
+    };
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return {
+      content: [{ type: "text", text: `Error analyzing page: ${msg}` }],
+    };
+  }
+}

package/src/tools/page/batch.ts

@@ -0,0 +1,163 @@
+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+import { parseHead, getMeta } from "./metadata.js";
+import { formatJsonLdType, validateJsonLd } from "./structured-data.js";
+import { followRedirects, checkLink, extractInternalLinks } from "./links.js";
+
+export async function analyzeBatch(
+  urls: string[],
+  options: { check_links?: boolean; user_agent?: string },
+): Promise<CallToolResult> {
+  const { check_links, user_agent } = options;
+  const ua = user_agent ?? "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
+  const results: Array<{
+    url: string;
+    title: string;
+    description: string;
+    canonical: string;
+    jsonLd: string;
+    links: string;
+    issues: string[];
+    error?: string;
+  }> = [];
+
+  for (const pageUrl of urls) {
+    try {
+      const { chain, response: res } = await followRedirects(pageUrl, ua);
+      if (!res.ok) {
+        results.push({
+          url: pageUrl,
+          title: "-",
+          description: "-",
+          canonical: "-",
+          jsonLd: "-",
+          links: "-",
+          issues: [`HTTP ${res.status}`],
+        });
+        continue;
+      }
+      const html = await res.text();
+      const parsed = parseHead(html);
+      const finalUrl = chain.length > 1 ? chain[chain.length - 1].url : pageUrl;
+
+      const issues: string[] = [];
+      if (!parsed.title) issues.push("no title");
+      if (!getMeta(parsed.meta, "description")) issues.push("no description");
+      else if ((getMeta(parsed.meta, "description") ?? "").length > 155) issues.push("description >155 chars");
+      if (!parsed.canonical) issues.push("no canonical");
+      if (parsed.jsonLd.length === 0) issues.push("no JSON-LD");
+      if (chain.length > 2) issues.push(`${chain.length - 1} redirects`);
+
+      // Quick link check
+      let linkSummary = "-";
+      if (check_links !== false) {
+        const origin = new URL(finalUrl).origin;
+        const links = extractInternalLinks(html, origin);
+        if (links.length > 0) {
+          const checked = await Promise.all(links.slice(0, 20).map((href) => checkLink(href)));
+          const broken = checked.filter((r) => r.error || (r.status && r.status >= 400)).length;
+          const redirected = checked.filter(
+            (r) => !r.error && r.redirectChain.length > 1 && r.status && r.status < 400,
+          ).length;
+          const parts: string[] = [`${links.length} found`];
+          if (broken > 0) parts.push(`${broken} broken`);
+          if (redirected > 0) parts.push(`${redirected} redirected`);
+          if (broken === 0 && redirected === 0) parts.push("all OK");
+          linkSummary = parts.join(", ");
+          if (broken > 0) issues.push(`${broken} broken links`);
+          if (redirected > 0) issues.push(`${redirected} redirect chains`);
+        } else {
+          linkSummary = "0 (SPA?)";
+        }
+      }
+
+      const jsonLdTypes = parsed.jsonLd
+        .map((block) => {
+          if (block && typeof block === "object" && "@type" in block)
+            return formatJsonLdType((block as Record<string, unknown>)["@type"], ",");
+          return null;
+        })
+        .filter(Boolean);
+
+      // Run structured data validation for batch summary
+      let jsonLdSummary = "none";
+      if (parsed.jsonLd.length > 0) {
+        const { issues: valIssues } = validateJsonLd(parsed.jsonLd);
+        const requiredMissing = valIssues.filter((i) => i.level === "required").length;
+        if (jsonLdTypes.length > 0) {
+          jsonLdSummary = `${jsonLdTypes.join(", ")} (selected fields only)`;
+          if (requiredMissing > 0) {
+            jsonLdSummary += ` (${requiredMissing} required missing)`;
+            issues.push(`${requiredMissing} missing required fields in checked JSON-LD subset`);
+          }
+        }
+      }
+
+      // Description with length
+      const descVal = getMeta(parsed.meta, "description");
+      let descSummary: string;
+      if (descVal) {
+        descSummary = descVal.length > 155 ? `yes (${descVal.length} chars ⚠)` : `yes (${descVal.length})`;
+      } else {
+        descSummary = "no";
+      }
+
+      results.push({
+        url: finalUrl,
+        title: parsed.title
+          ? parsed.title.length > 60
+            ? `${parsed.title.slice(0, parsed.title.lastIndexOf(" ", 60) > 20 ? parsed.title.lastIndexOf(" ", 60) : 60)}...`
+            : parsed.title
+          : "(missing)",
+        description: descSummary,
+        canonical: parsed.canonical ? "yes" : "no",
+        jsonLd: jsonLdSummary,
+        links: linkSummary,
+        issues,
+      });
+    } catch (err) {
+      results.push({
+        url: pageUrl,
+        title: "-",
+        description: "-",
+        canonical: "-",
+        jsonLd: "-",
+        links: "-",
+        issues: [err instanceof Error ? err.message : String(err)],
+      });
+    }
+  }
+
+  const lines: string[] = [`=== Batch Page Analysis (${results.length} URLs) ===`, ""];
+
+  for (const r of results) {
+    let label: string;
+    try {
+      const u = new URL(r.url);
+      const allSameHost = results.every((x) => {
+        try {
+          return new URL(x.url).hostname === u.hostname;
+        } catch {
+          return false;
+        }
+      });
+      label = allSameHost ? u.pathname : `${u.hostname}${u.pathname}`;
+    } catch {
+      label = r.url;
+    }
+    lines.push(label);
+    lines.push(`  Title: ${r.title}`);
+    lines.push(`  Description: ${r.description}  Canonical: ${r.canonical}  JSON-LD: ${r.jsonLd}`);
+    if (check_links !== false) lines.push(`  Links: ${r.links}`);
+    if (r.issues.length > 0) {
+      lines.push(`  Issues: ${r.issues.join(", ")}`);
+    } else {
+      lines.push("  Issues: none");
+    }
+    lines.push("");
+  }
+
+  const issueCount = results.reduce((sum, r) => sum + r.issues.length, 0);
+  lines.push(`Total: ${results.length} pages, ${issueCount} issues`);
+
+  return { content: [{ type: "text", text: lines.join("\n") }] };
+}