pagesight 0.14.0 → 0.14.2

package/README.md

@@ -8,34 +8,40 @@ See your site the way search engines and AI see it.
 npm install pagesight
 ```
 
-Your AI assistant can write your code. Now it can see your site. Index status, performance, real-user metrics, search traffic, meta tags, structured data, AI crawler access — one package, one call.
+Your AI assistant can write your code. Now it can see your site. Index status, performance, real-user metrics, search traffic, meta tags, structured data, AI crawler access, link health — one package, one call.
 
 ```
 === Site Audit: https://example.com ===
 
+2 checks failed — results below are partial:
+
+  FAIL  PageSpeed: quota exceeded
+  FAIL  Sitemaps: permission denied
+
+5 findings:
+
 HIGH    Missing canonical URL
-HIGH    7,772 sitemap URLs submitted, 0 indexed
+HIGH    22 sitemap URLs submitted, 0 indexed
+        Auto-inspected 5 URLs:
+        - 2/5 indexed
+        - 3/5 Discovered - currently not indexed: /docs/, /pricing/, /about/
 MEDIUM  Missing og:image — no social preview image
-MEDIUM  Accessibility score: 89/100
 LOW     Missing Twitter Card tags
-LOW     No structured data (JSON-LD) found
+LOW     6/139 AI crawlers blocked
 ```
 
 ## Tools
 
-| Tool | What it does |
-|------|-------------|
-| `audit` | One-call site audit. Runs all checks in parallel. Returns prioritized findings. |
-| `pagespeed` | Lighthouse scores, Core Web Vitals, opportunities, failing audits with fix links. |
-| `metatags` | OG, Twitter Card, canonical, JSON-LD with schema validation, redirect chain, image validation. |
-| `inspect` | Google index status, canonical choice, crawl state, rich results. |
-| `sample_inspect` | Sample URLs from a sitemap and batch-inspect. Diagnoses indexing patterns. |
-| `performance` | Search analytics — clicks, impressions, CTR, position. `compare: true` for period-over-period. |
-| `crux` | Real-user Core Web Vitals (p75, histograms). |
-| `crux_history` | CWV trends over time — up to 40 weekly data points. |
-| `robots` | robots.txt validation (RFC 9309) + AI crawler audit (139+ bots). |
-| `sitemaps` | Search Console properties and sitemaps with submitted/indexed counts. |
-| `setup` | Auth status and OAuth setup. |
+6 tools organized by intent:
+
+| Tool | Intent | What it does |
+|------|--------|-------------|
+| `audit` | How's my site? | One-call site audit. Runs all checks in parallel. Prioritized findings with auto-drill-down on indexing issues. |
+| `page` | What's on this URL? | Meta tags, OG, Twitter Card, JSON-LD validation (19 schema types), internal link health, redirect chains, WCAG contrast checker. Batch mode for multiple URLs. |
+| `speed` | How fast is it? | PageSpeed single/batch/compare with Lighthouse scores and opportunities. CrUX real-user metrics (snapshot + history trends). |
+| `search` | How's Google seeing me? | URL inspection, sample-inspect from sitemaps, sitemap management, search analytics with period-over-period comparison. |
+| `ai` | How's AI seeing me? | AI crawler audit (139+ bots by category), robots.txt validation (RFC 9309), llms.txt detection, path access checks. |
+| `setup` | Auth config | Auth status check and OAuth setup flow. |
 
 ## Setup
 
@@ -58,7 +64,7 @@ Add to your MCP config:
 }
 ```
 
-`robots`, `metatags`, and `pagespeed` work without credentials.
+`page`, `speed`, and `ai` work without credentials. `search` and `audit` (for GSC checks) require OAuth or a service account.
 
 ### Full setup
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pagesight",
-  "version": "0.14.0",
+  "version": "0.14.2",
   "description": "See your site the way search engines and AI see it.",
   "keywords": [
     "seo",

package/src/lib/auth.ts

@@ -15,6 +15,9 @@ let cachedToken: { token: string; expiresAt: number } | null = null;
 
 async function getServiceAccountToken(keyPath: string): Promise<string> {
   const keyFile = JSON.parse(await Bun.file(keyPath).text());
+  if (!keyFile.client_email || !keyFile.private_key) {
+    throw new Error("Service account key file missing client_email or private_key");
+  }
   const now = Math.floor(Date.now() / 1000);
 
   const header = toBase64Url(JSON.stringify({ alg: "RS256", typ: "JWT" }));

package/src/lib/crux.ts

@@ -68,14 +68,18 @@ export interface CruxHistoryResponse {
   urlNormalizationDetails?: { originalUrl: string; normalizedUrl: string };
 }
 
-// --- CrUX Daily API ---
-
-export async function queryCrux(options: {
-  url?: string;
-  origin?: string;
-  formFactor?: CruxFormFactor;
-  metrics?: string[];
-}): Promise<CruxResponse> {
+// --- Shared fetch helper ---
+
+async function cruxFetch<T>(
+  endpoint: string,
+  options: {
+    url?: string;
+    origin?: string;
+    formFactor?: CruxFormFactor;
+    metrics?: string[];
+    collectionPeriodCount?: number;
+  },
+): Promise<T> {
   const key = getApiKey();
 
   const body: Record<string, unknown> = {};
@@ -83,8 +87,11 @@ export async function queryCrux(options: {
   if (options.origin) body.origin = options.origin;
   if (options.formFactor) body.formFactor = options.formFactor;
   if (options.metrics) body.metrics = options.metrics;
+  if (options.collectionPeriodCount !== undefined && options.collectionPeriodCount !== 0) {
+    body.collectionPeriodCount = options.collectionPeriodCount;
+  }
 
-  const res = await fetch(`${CRUX_API}:queryRecord?key=${key}`, {
+  const res = await fetch(`${CRUX_API}:${endpoint}?key=${key}`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
     body: JSON.stringify(body),
@@ -95,37 +102,28 @@ export async function queryCrux(options: {
     throw new Error(`CrUX API error (${res.status}): ${err}`);
   }
 
-  return res.json() as Promise<CruxResponse>;
+  return res.json() as Promise<T>;
+}
+
+// --- CrUX Daily API ---
+
+export function queryCrux(options: {
+  url?: string;
+  origin?: string;
+  formFactor?: CruxFormFactor;
+  metrics?: string[];
+}): Promise<CruxResponse> {
+  return cruxFetch<CruxResponse>("queryRecord", options);
 }
 
 // --- CrUX History API ---
 
-export async function queryCruxHistory(options: {
+export function queryCruxHistory(options: {
   url?: string;
   origin?: string;
   formFactor?: CruxFormFactor;
   metrics?: string[];
   collectionPeriodCount?: number;
 }): Promise<CruxHistoryResponse> {
-  const key = getApiKey();
-
-  const body: Record<string, unknown> = {};
-  if (options.url) body.url = options.url;
-  if (options.origin) body.origin = options.origin;
-  if (options.formFactor) body.formFactor = options.formFactor;
-  if (options.metrics) body.metrics = options.metrics;
-  if (options.collectionPeriodCount) body.collectionPeriodCount = options.collectionPeriodCount;
-
-  const res = await fetch(`${CRUX_API}:queryHistoryRecord?key=${key}`, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify(body),
-  });
-
-  if (!res.ok) {
-    const err = await res.text();
-    throw new Error(`CrUX History API error (${res.status}): ${err}`);
-  }
-
-  return res.json() as Promise<CruxHistoryResponse>;
+  return cruxFetch<CruxHistoryResponse>("queryHistoryRecord", options);
 }

package/src/lib/gsc.ts

@@ -21,7 +21,11 @@ async function gscFetch(url: string, body?: unknown): Promise<Record<string, unk
     throw new Error(`GSC API error (${res.status}): ${err}`);
   }
 
-  return res.json();
+  try {
+    return (await res.json()) as Record<string, unknown>;
+  } catch {
+    throw new Error(`GSC API returned invalid JSON (${res.status})`);
+  }
 }
 
 // --- URL Inspection ---
@@ -62,6 +66,9 @@ export async function inspectUrl(inspectionUrl: string, siteUrl: string): Promis
     inspectionUrl,
     siteUrl,
   });
+  if (!data.inspectionResult) {
+    throw new Error("GSC API returned no inspection result");
+  }
   return data.inspectionResult as InspectionResult;
 }
 

package/src/lib/psi.ts

@@ -115,5 +115,9 @@ export async function runPagespeed(
     throw new Error(`PageSpeed API error (${res.status}): ${err}`);
   }
 
-  return res.json() as Promise<PsiResult>;
+  try {
+    return (await res.json()) as PsiResult;
+  } catch {
+    throw new Error(`PageSpeed API returned invalid JSON (${res.status})`);
+  }
 }

package/src/lib/robots.ts

@@ -157,6 +157,13 @@ export function parseRobotsTxt(raw: string): RobotsTxt {
 function pathMatches(pattern: string, path: string): boolean {
   if (!pattern) return false;
 
+  // RFC 9309 §2.2.2: decode percent-encoded characters for comparison
+  try {
+    path = decodeURIComponent(path);
+  } catch {
+    // malformed encoding, use as-is
+  }
+
   let regex = "^";
   for (let i = 0; i < pattern.length; i++) {
     const c = pattern[i];

package/src/lib/sitemap.ts

@@ -27,14 +27,22 @@ export function parseSitemapXml(xml: string): SitemapParseResult {
 
 export async function fetchSitemap(sitemapUrl: string): Promise<SitemapParseResult> {
   const res = await fetch(sitemapUrl, {
-    headers: { "User-Agent": "Pagesight/1.0" },
+    headers: { "User-Agent": "Pagesight/1.0", "Accept-Encoding": "gzip, deflate" },
   });
 
   if (!res.ok) {
     throw new Error(`Failed to fetch sitemap ${sitemapUrl}: HTTP ${res.status}`);
   }
 
-  const xml = await res.text();
+  let xml: string;
+  const contentType = res.headers.get("content-type") ?? "";
+  if (sitemapUrl.endsWith(".gz") || contentType.includes("gzip") || contentType.includes("application/x-gzip")) {
+    const buffer = await res.arrayBuffer();
+    const decompressed = Bun.gunzipSync(new Uint8Array(buffer));
+    xml = new TextDecoder().decode(decompressed);
+  } else {
+    xml = await res.text();
+  }
   return parseSitemapXml(xml);
 }
 

package/src/tools/ai.ts

@@ -12,12 +12,24 @@ interface LlmsTxtResult {
 
 async function checkLlmsTxt(origin: string, path: string): Promise<LlmsTxtResult> {
   try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 10_000);
     const res = await fetch(`${origin}${path}`, {
       headers: { "User-Agent": "Pagesight/1.0" },
       redirect: "follow",
+      signal: controller.signal,
     });
+    clearTimeout(timeout);
     if (!res.ok) return { exists: false, size: null, firstLine: null };
+    // Cap at 1MB to avoid OOM on large responses
+    const contentLength = Number(res.headers.get("content-length") ?? 0);
+    if (contentLength > 1_048_576) {
+      return { exists: true, size: contentLength, firstLine: "(file too large to preview)" };
+    }
     const text = await res.text();
+    if (text.length > 1_048_576) {
+      return { exists: true, size: text.length, firstLine: "(file too large to preview)" };
+    }
     const firstLine =
       text
         .split("\n")
@@ -215,6 +227,7 @@ export function registerAiTool(server: McpServer): void {
     {
       url: z
         .string()
+        .url()
         .describe("Site URL or origin (e.g., 'https://example.com'). Fetches /robots.txt from this origin."),
       check_path: z
         .string()

package/src/tools/audit.ts

@@ -224,7 +224,12 @@ function formatDrillDown(
   const stateCounts: Record<string, { count: number; urls: string[] }> = {};
   for (const r of valid) {
     if (r.verdict !== "PASS") {
-      const path = new URL(r.url).pathname;
+      let path: string;
+      try {
+        path = new URL(r.url).pathname;
+      } catch {
+        path = r.url;
+      }
       const existing = stateCounts[r.coverageState];
       if (existing) {
         existing.count++;
@@ -357,10 +362,7 @@ export function registerAuditTool(server: McpServer): void {
             }
             if (parsed.urls.length > 0) {
               const sampled = sampleUrls(parsed.urls, 5, "spread");
-              const inspections = [];
-              for (const u of sampled) {
-                inspections.push(await inspectSingle(u, site_url));
-              }
+              const inspections = await Promise.all(sampled.map((u) => inspectSingle(u, site_url)));
               const drillDown = formatDrillDown(inspections);
               // Append drill-down to the sitemap finding
               const sitemapFinding = findings.find((f) => f.source === "sitemaps" && f.severity === "HIGH");

package/src/tools/page.ts

@@ -105,6 +105,15 @@ function parseHead(html: string): ParsedHead {
     hreflang.push({ lang: hm[2], href: hm[1] });
   }
 
+  // Dedup hreflang entries
+  const seen = new Set<string>();
+  const dedupedHreflang = hreflang.filter((h) => {
+    const key = `${h.lang}:${h.href}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+
   // JSON-LD
   const jsonLd: unknown[] = [];
   for (const ld of html.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
@@ -115,7 +124,7 @@ function parseHead(html: string): ParsedHead {
     }
   }
 
-  return { title, charset, canonical, meta, hreflang, jsonLd };
+  return { title, charset, canonical, meta, hreflang: dedupedHreflang, jsonLd };
 }
 
 function getMeta(meta: MetaTag[], key: string): string | null {
@@ -276,8 +285,6 @@ const SCHEMA_RULES: Record<string, SchemaRule> = {
 function getNestedValue(obj: Record<string, unknown>, field: string): unknown {
   const val = obj[field];
   if (val !== undefined && val !== null && val !== "") return val;
-  // Check if it's a nested object with a value (e.g., logo might be {url: "..."} or a string)
-  if (typeof val === "object" && val !== null) return val;
   return undefined;
 }
 
@@ -438,10 +445,14 @@ async function followRedirects(
   let current = url;
 
   for (let i = 0; i < maxHops; i++) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 15_000);
     const res = await fetch(current, {
       headers: { "User-Agent": ua, Accept: "text/html" },
       redirect: "manual",
+      signal: controller.signal,
     });
+    clearTimeout(timeout);
 
     chain.push({ url: current, status: res.status });
 
@@ -465,7 +476,10 @@ async function followRedirects(
 
 async function checkImage(imageUrl: string, tag: string): Promise<ImageCheck> {
   try {
-    const res = await fetch(imageUrl, { method: "HEAD", redirect: "follow" });
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 10_000);
+    const res = await fetch(imageUrl, { method: "HEAD", redirect: "follow", signal: controller.signal });
+    clearTimeout(timeout);
     return {
       url: imageUrl,
       tag,
@@ -614,11 +628,15 @@ async function checkLink(href: string): Promise<LinkResult> {
 
   try {
     for (let i = 0; i < 10; i++) {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), 10_000);
       const res = await fetch(current, {
-        method: "GET",
+        method: "HEAD",
         headers: { "User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1)", Accept: "text/html" },
         redirect: "manual",
+        signal: controller.signal,
       });
+      clearTimeout(timeout);
 
       chain.push({ url: current, status: res.status });
 
@@ -935,9 +953,20 @@ export function registerPageTool(server: McpServer): void {
         const lines: string[] = [`=== Batch Page Analysis (${results.length} URLs) ===`, ""];
 
         for (const r of results) {
-          const u = new URL(r.url);
-          const allSameHost = results.every((x) => new URL(x.url).hostname === u.hostname);
-          const label = allSameHost ? u.pathname : `${u.hostname}${u.pathname}`;
+          let label: string;
+          try {
+            const u = new URL(r.url);
+            const allSameHost = results.every((x) => {
+              try {
+                return new URL(x.url).hostname === u.hostname;
+              } catch {
+                return false;
+              }
+            });
+            label = allSameHost ? u.pathname : `${u.hostname}${u.pathname}`;
+          } catch {
+            label = r.url;
+          }
           lines.push(label);
           lines.push(`  Title: ${r.title}`);
           lines.push(`  Description: ${r.description}  Canonical: ${r.canonical}  JSON-LD: ${r.jsonLd}`);

package/src/tools/search.ts

@@ -430,10 +430,16 @@ function formatComparison(
 }
 
 function daysAgo(n: number): string {
-  // GSC dates are in PT (Pacific Time). Use UTC-8 as a stable approximation.
-  const now = new Date(Date.now() - 8 * 60 * 60 * 1000);
-  now.setDate(now.getDate() - n);
-  return now.toISOString().split("T")[0];
+  // GSC dates are in Pacific Time. Use Intl to handle DST correctly.
+  const d = new Date();
+  d.setDate(d.getDate() - n);
+  const parts = new Intl.DateTimeFormat("en-CA", {
+    timeZone: "America/Los_Angeles",
+    year: "numeric",
+    month: "2-digit",
+    day: "2-digit",
+  }).format(d);
+  return parts; // en-CA formats as YYYY-MM-DD
 }
 
 // ── Tool registration ──
@@ -630,6 +636,13 @@ export function registerSearchTool(server: McpServer): void {
 
           const startDate = start_date ?? daysAgo(28);
           const endDate = end_date ?? daysAgo(3);
+
+          if (start_date && Number.isNaN(new Date(start_date).getTime())) {
+            return textResult(`Error: invalid start_date "${start_date}". Use YYYY-MM-DD format.`);
+          }
+          if (end_date && Number.isNaN(new Date(end_date).getTime())) {
+            return textResult(`Error: invalid end_date "${end_date}". Use YYYY-MM-DD format.`);
+          }
           const dims = dimensions ?? ["query", "page"];
 
           const filterGroups =

package/src/tools/speed.ts

@@ -312,6 +312,7 @@ function formatDelta(a: number | null, b: number | null): string {
 }
 
 function formatBatchCompare(results: Array<{ url: string; result: PsiResult }>, strategy: string): string {
+  if (results.length < 2) return "Error: compare requires at least 2 results.";
   const [a, b] = results;
   const lhrA = a.result.lighthouseResult;
   const lhrB = b.result.lighthouseResult;
@@ -395,6 +396,7 @@ function formatBatchCompare(results: Array<{ url: string; result: PsiResult }>,
 }
 
 function formatBatchTable(results: Array<{ url: string; result: PsiResult }>, strategy: string): string {
+  if (results.length === 0) return "Error: no results to display.";
   const lines: string[] = [`=== Batch PageSpeed (${results.length} URLs, ${strategy}) ===`, ""];
 
   // Collect all category IDs from first result
@@ -813,10 +815,10 @@ export function registerSpeedTool(server: McpServer): void {
         const cruxOrigin = origin;
 
         if (!cruxUrl && !cruxOrigin) {
-          return { content: [{ type: "text" as const, text: "Error: provide either url or origin, not both." }] };
+          return { content: [{ type: "text" as const, text: "Error: provide url or origin for CrUX data." }] };
         }
         if (cruxUrl && cruxOrigin) {
-          return { content: [{ type: "text" as const, text: "Error: provide either url or origin, not both." }] };
+          return { content: [{ type: "text" as const, text: "Error: provide url or origin, not both." }] };
         }
 
         try {
@@ -860,10 +862,10 @@ export function registerSpeedTool(server: McpServer): void {
         const histOrigin = origin;
 
         if (!histUrl && !histOrigin) {
-          return { content: [{ type: "text" as const, text: "Error: provide either url or origin, not both." }] };
+          return { content: [{ type: "text" as const, text: "Error: provide url or origin for CrUX history." }] };
         }
         if (histUrl && histOrigin) {
-          return { content: [{ type: "text" as const, text: "Error: provide either url or origin, not both." }] };
+          return { content: [{ type: "text" as const, text: "Error: provide url or origin, not both." }] };
         }
 
         try {