pagesight 0.17.0 → 0.18.0

package/src/providers/ga.ts

@@ -0,0 +1,114 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { getOAuthToken, getServiceAccountToken } from "./google-tokens.js";
+import { RequestError, requestJson } from "../shared/http.js";
+
+const SCOPE = "https://www.googleapis.com/auth/analytics.readonly";
+let cached: { path: string; token: string; expiresAt: number } | undefined;
+let pending: Promise<string> | undefined;
+
+export function gaCredentialPath(): string {
+  return (
+    process.env.PAGESIGHT_GA_CREDENTIALS ||
+    process.env.GOOGLE_APPLICATION_CREDENTIALS ||
+    join(homedir(), ".config/gcloud/application_default_credentials.json")
+  );
+}
+
+export async function gaAccessToken(): Promise<string> {
+  const path = gaCredentialPath();
+  if (cached?.path === path && cached.expiresAt > Date.now()) return cached.token;
+  if (pending) return pending;
+  pending = (async () => {
+    if (!(await Bun.file(path).exists()))
+      throw new RequestError(
+        "Set PAGESIGHT_GA_CREDENTIALS or GOOGLE_APPLICATION_CREDENTIALS to a Google credential JSON file",
+        null,
+        "not_configured",
+      );
+    let key: { type?: string; client_id?: string; client_secret?: string; refresh_token?: string };
+    try {
+      key = await Bun.file(path).json();
+      if (!key || typeof key !== "object") throw new Error();
+    } catch {
+      throw new RequestError("Cannot parse configured GA credential JSON", null, "invalid_credentials");
+    }
+    let token: string;
+    if (key.type === "service_account") token = await getServiceAccountToken(path, SCOPE);
+    else if (
+      key.type === "authorized_user" &&
+      typeof key.client_id === "string" &&
+      key.client_id &&
+      typeof key.client_secret === "string" &&
+      key.client_secret &&
+      typeof key.refresh_token === "string" &&
+      key.refresh_token
+    ) {
+      token = await getOAuthToken(key.client_id, key.client_secret, key.refresh_token);
+    } else
+      throw new RequestError(
+        "Supported GA credentials: service_account or authorized_user ADC",
+        null,
+        "unsupported_credentials",
+      );
+    cached = { path, token, expiresAt: Date.now() + 3_000_000 };
+    return token;
+  })();
+  try {
+    return await pending;
+  } finally {
+    pending = undefined;
+  }
+}
+
+export async function gaFetch<T>(path: string, body?: unknown, admin = false): Promise<T> {
+  const token = await gaAccessToken();
+  const base = admin ? "https://analyticsadmin.googleapis.com/v1beta/" : "https://analyticsdata.googleapis.com/v1beta/";
+  try {
+    return await requestJson<T>(base + path, {
+      method: body ? "POST" : "GET",
+      headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
+      body: body ? JSON.stringify(body) : undefined,
+    });
+  } catch (error) {
+    if (error instanceof RequestError && error.status === 401) cached = undefined;
+    throw error;
+  }
+}
+
+export function normalizeGaProperty(id: string): string {
+  const number = id.replace(/^properties\//, "");
+  if (!/^\d+$/.test(number)) throw new RequestError("GA property must be a numeric ID", null, "invalid_input");
+  return `properties/${number}`;
+}
+
+export interface GaReport {
+  rows?: Array<{ dimensionValues?: Array<{ value?: string }>; metricValues?: Array<{ value?: string }> }>;
+  rowCount?: number;
+  metadata?: {
+    timeZone?: string;
+    dataLossFromOtherRow?: boolean;
+    subjectToThresholding?: boolean;
+    samplingMetadatas?: unknown[];
+    [key: string]: unknown;
+  };
+  [key: string]: unknown;
+}
+
+export async function gaCredentialInfo() {
+  const source = process.env.PAGESIGHT_GA_CREDENTIALS
+    ? "PAGESIGHT_GA_CREDENTIALS"
+    : process.env.GOOGLE_APPLICATION_CREDENTIALS
+      ? "GOOGLE_APPLICATION_CREDENTIALS"
+      : "gcloud_adc";
+  try {
+    const key = await Bun.file(gaCredentialPath()).json();
+    return {
+      source,
+      type: typeof key.type === "string" ? key.type : null,
+      clientEmail: typeof key.client_email === "string" ? key.client_email : null,
+    };
+  } catch {
+    return { source, type: null, clientEmail: null };
+  }
+}

package/src/providers/google-tokens.ts

@@ -0,0 +1,86 @@
+import { RequestError, requestJson } from "../shared/http.js";
+
+export const TOKEN_URL = "https://oauth2.googleapis.com/token";
+
+interface TokenResponse {
+  access_token: string;
+  expires_in: number;
+  token_type: string;
+}
+
+export async function getServiceAccountToken(keyPath: string, scope: string): Promise<string> {
+  const keyFile = JSON.parse(await Bun.file(keyPath).text());
+  if (!keyFile.client_email || !keyFile.private_key) {
+    throw new Error("Service account key file missing client_email or private_key");
+  }
+  const now = Math.floor(Date.now() / 1000);
+
+  const header = toBase64Url(JSON.stringify({ alg: "RS256", typ: "JWT" }));
+  const payload = toBase64Url(
+    JSON.stringify({
+      iss: keyFile.client_email,
+      scope,
+      aud: TOKEN_URL,
+      iat: now,
+      exp: now + 3600,
+    }),
+  );
+
+  const signingInput = `${header}.${payload}`;
+  const key = await crypto.subtle.importKey(
+    "pkcs8",
+    pemToBuffer(keyFile.private_key),
+    { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
+    false,
+    ["sign"],
+  );
+  const signature = await crypto.subtle.sign("RSASSA-PKCS1-v1_5", key, new TextEncoder().encode(signingInput));
+  const jwt = `${signingInput}.${bufferToBase64Url(signature)}`;
+
+  const data = await requestJson<TokenResponse>(TOKEN_URL, {
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams({ grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", assertion: jwt }),
+  });
+  if (!data.access_token) throw new RequestError("Token exchange returned no access token", null, "invalid_response");
+  return data.access_token;
+}
+
+function pemToBuffer(pem: string): ArrayBuffer {
+  const b64 = pem
+    .replace(/-----BEGIN PRIVATE KEY-----/g, "")
+    .replace(/-----END PRIVATE KEY-----/g, "")
+    .replace(/\s/g, "");
+  const binary = atob(b64);
+  const buf = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) buf[i] = binary.charCodeAt(i);
+  return buf.buffer;
+}
+
+function toBase64Url(input: string): string {
+  return btoa(input).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+}
+
+function bufferToBase64Url(buf: ArrayBuffer): string {
+  const bytes = new Uint8Array(buf);
+  let binary = "";
+  for (const b of bytes) binary += String.fromCharCode(b);
+  return toBase64Url(binary);
+}
+
+// --- OAuth Refresh Token Auth ---
+
+export async function getOAuthToken(clientId: string, clientSecret: string, refreshToken: string): Promise<string> {
+  const data = await requestJson<TokenResponse>(TOKEN_URL, {
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams({
+      grant_type: "refresh_token",
+      client_id: clientId,
+      client_secret: clientSecret,
+      refresh_token: refreshToken,
+    }),
+  });
+  if (!data.access_token) throw new RequestError("Token refresh returned no access token", null, "invalid_response");
+  return data.access_token;
+}

package/src/providers/gsc-auth.ts

@@ -0,0 +1,93 @@
+import { getOAuthToken, getServiceAccountToken, TOKEN_URL } from "./google-tokens.js";
+import { existsSync } from "node:fs";
+import { RequestError } from "../shared/http.js";
+
+const SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly"];
+let cachedToken: { token: string; expiresAt: number } | null = null;
+
+// --- Service Account Auth ---
+
+// --- Public API ---
+
+export async function getAccessToken(): Promise<string> {
+  if (cachedToken && Date.now() < cachedToken.expiresAt) {
+    return cachedToken.token;
+  }
+
+  // Try service account first
+  const saKeyPath = process.env.GSC_SERVICE_ACCOUNT_KEY;
+  if (saKeyPath && existsSync(saKeyPath)) {
+    const token = await getServiceAccountToken(saKeyPath, SCOPES.join(" "));
+    cachedToken = { token, expiresAt: Date.now() + 3500_000 }; // ~58 min
+    return token;
+  }
+
+  // Try OAuth refresh token
+  const clientId = process.env.GSC_CLIENT_ID;
+  const clientSecret = process.env.GSC_CLIENT_SECRET;
+  const refreshToken = process.env.GSC_REFRESH_TOKEN;
+
+  if (clientId && clientSecret && refreshToken) {
+    const token = await getOAuthToken(clientId, clientSecret, refreshToken);
+    cachedToken = { token, expiresAt: Date.now() + 3500_000 };
+    return token;
+  }
+
+  throw new RequestError(
+    "No GSC credentials configured. Set either:\n" +
+      "  - GSC_SERVICE_ACCOUNT_KEY (path to service account JSON)\n" +
+      "  - GSC_CLIENT_ID + GSC_CLIENT_SECRET + GSC_REFRESH_TOKEN (OAuth 2.0)",
+    null,
+    "not_configured",
+  );
+}
+
+export function clearTokenCache(): void {
+  cachedToken = null;
+}
+
+export function getAuthMethod(): string {
+  if (process.env.GSC_SERVICE_ACCOUNT_KEY) return "service_account";
+  if (process.env.GSC_REFRESH_TOKEN) return "oauth";
+  return "none";
+}
+
+// --- OAuth Setup Helper ---
+
+export function getOAuthSetupUrl(clientId: string): string {
+  const params = new URLSearchParams({
+    client_id: clientId,
+    redirect_uri: "http://localhost",
+    response_type: "code",
+    scope: SCOPES.join(" "),
+    access_type: "offline",
+    prompt: "consent",
+  });
+  return `https://accounts.google.com/o/oauth2/auth?${params}`;
+}
+
+export async function exchangeCodeForToken(
+  clientId: string,
+  clientSecret: string,
+  code: string,
+): Promise<{ refreshToken: string; accessToken: string }> {
+  const res = await fetch(TOKEN_URL, {
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams({
+      grant_type: "authorization_code",
+      client_id: clientId,
+      client_secret: clientSecret,
+      code,
+      redirect_uri: "http://localhost",
+    }).toString(),
+  });
+
+  if (!res.ok) {
+    const err = await res.text();
+    throw new Error(`Code exchange failed: ${err}`);
+  }
+
+  const data = (await res.json()) as Record<string, string>;
+  return { refreshToken: data.refresh_token, accessToken: data.access_token };
+}

package/src/{lib → providers}/gsc.ts

@@ -1,30 +1,20 @@
-import { clearTokenCache, getAccessToken } from "./auth.js";
+import { clearTokenCache, getAccessToken } from "./gsc-auth.js";
+import { RequestError, requestJson } from "../shared/http.js";
 
 const GSC_API = "https://searchconsole.googleapis.com/v1";
 const WEBMASTERS_API = "https://www.googleapis.com/webmasters/v3";
 
 async function gscFetch(url: string, body?: unknown): Promise<Record<string, unknown>> {
   const token = await getAccessToken();
-  const res = await fetch(url, {
-    method: body ? "POST" : "GET",
-    headers: {
-      Authorization: `Bearer ${token}`,
-      "Content-Type": "application/json",
-    },
-    body: body ? JSON.stringify(body) : undefined,
-  });
-
-  if (!res.ok) {
-    // Clear cached token on 401 so next call gets a fresh one
-    if (res.status === 401) clearTokenCache();
-    const err = await res.text();
-    throw new Error(`GSC API error (${res.status}): ${err}`);
-  }
-
   try {
-    return (await res.json()) as Record<string, unknown>;
-  } catch {
-    throw new Error(`GSC API returned invalid JSON (${res.status})`);
+    return await requestJson<Record<string, unknown>>(url, {
+      method: body ? "POST" : "GET",
+      headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
+      body: body ? JSON.stringify(body) : undefined,
+    });
+  } catch (error) {
+    if (error instanceof RequestError && error.status === 401) clearTokenCache();
+    throw error;
   }
 }
 
@@ -62,6 +52,11 @@ export interface InspectionResult {
 }
 
 export async function inspectUrl(inspectionUrl: string, siteUrl: string): Promise<InspectionResult> {
+  const data = await inspectUrlResponse(inspectionUrl, siteUrl);
+  return data.inspectionResult as InspectionResult;
+}
+
+export async function inspectUrlResponse(inspectionUrl: string, siteUrl: string): Promise<Record<string, unknown>> {
   const data = await gscFetch(`${GSC_API}/urlInspection/index:inspect`, {
     inspectionUrl,
     siteUrl,
@@ -69,7 +64,7 @@ export async function inspectUrl(inspectionUrl: string, siteUrl: string): Promis
   if (!data.inspectionResult) {
     throw new Error("GSC API returned no inspection result");
   }
-  return data.inspectionResult as InspectionResult;
+  return data;
 }
 
 // --- Search Analytics ---
@@ -121,7 +116,7 @@ export async function querySearchAnalytics(
     dimensions: options.dimensions ?? ["query", "page"],
     type: options.type ?? "web",
     rowLimit: options.rowLimit ?? 1000,
-    dataState: options.dataState ?? "all",
+    dataState: options.dataState ?? "final",
   };
 
   if (options.startRow !== undefined) body.startRow = options.startRow;
@@ -140,10 +135,14 @@ export interface GscSite {
 }
 
 export async function listSites(): Promise<GscSite[]> {
-  const data = await gscFetch(`${WEBMASTERS_API}/sites`);
+  const data = await listSitesResponse();
   return (data.siteEntry as GscSite[] | undefined) ?? [];
 }
 
+export function listSitesResponse(): Promise<Record<string, unknown>> {
+  return gscFetch(`${WEBMASTERS_API}/sites`);
+}
+
 export async function getSite(siteUrl: string): Promise<GscSite> {
   const data = await gscFetch(`${WEBMASTERS_API}/sites/${encodeURIComponent(siteUrl)}`);
   return data as unknown as GscSite;
@@ -170,10 +169,14 @@ export interface GscSitemap {
 }
 
 export async function listSitemaps(siteUrl: string): Promise<GscSitemap[]> {
-  const data = await gscFetch(`${WEBMASTERS_API}/sites/${encodeURIComponent(siteUrl)}/sitemaps`);
+  const data = await listSitemapsResponse(siteUrl);
   return (data.sitemap as GscSitemap[] | undefined) ?? [];
 }
 
+export function listSitemapsResponse(siteUrl: string): Promise<Record<string, unknown>> {
+  return gscFetch(`${WEBMASTERS_API}/sites/${encodeURIComponent(siteUrl)}/sitemaps`);
+}
+
 export async function getSitemap(siteUrl: string, feedpath: string): Promise<GscSitemap> {
   const data = await gscFetch(
     `${WEBMASTERS_API}/sites/${encodeURIComponent(siteUrl)}/sitemaps/${encodeURIComponent(feedpath)}`,

package/src/{lib/psi.ts → providers/pagespeed.ts}

@@ -1,3 +1,5 @@
+import { requestJson } from "../shared/http.js";
+
 const PSI_API = "https://www.googleapis.com/pagespeedonline/v5/runPagespeed";
 
 export interface PsiCategory {
@@ -106,18 +108,5 @@ export async function runPagespeed(
 
   if (options?.locale) params.set("locale", options.locale);
 
-  const res = await fetch(`${PSI_API}?${params}`, {
-    headers: { "User-Agent": "Pagesight/0.1" },
-  });
-
-  if (!res.ok) {
-    const err = await res.text();
-    throw new Error(`PageSpeed API error (${res.status}): ${err}`);
-  }
-
-  try {
-    return (await res.json()) as PsiResult;
-  } catch {
-    throw new Error(`PageSpeed API returned invalid JSON (${res.status})`);
-  }
+  return requestJson<PsiResult>(`${PSI_API}?${params}`, { headers: { "User-Agent": "Pagesight/0.17" } }, 60_000);
 }

package/src/shared/dates.ts

@@ -0,0 +1,32 @@
+import { z } from "zod";
+
+export const dateSchema = z
+  .string()
+  .regex(/^\d{4}-\d{2}-\d{2}$/)
+  .refine((value) => {
+    const date = new Date(value);
+    return !Number.isNaN(date.getTime()) && date.toISOString().slice(0, 10) === value;
+  }, "Use a real YYYY-MM-DD date");
+
+export function pacificDate(now = new Date()): string {
+  return new Intl.DateTimeFormat("en-CA", {
+    timeZone: "America/Los_Angeles",
+    year: "numeric",
+    month: "2-digit",
+    day: "2-digit",
+  }).format(now);
+}
+
+export function defaultDates(now = new Date()): { startDate: string; endDate: string } {
+  const end = new Date(`${pacificDate(now)}T12:00:00Z`);
+  end.setUTCDate(end.getUTCDate() - 3);
+  const start = new Date(end);
+  start.setUTCDate(start.getUTCDate() - 27);
+  return { startDate: start.toISOString().slice(0, 10), endDate: end.toISOString().slice(0, 10) };
+}
+
+export function pacificDaysAgo(days: number, now = new Date()): string {
+  const date = new Date(`${pacificDate(now)}T12:00:00Z`);
+  date.setUTCDate(date.getUTCDate() - days);
+  return date.toISOString().slice(0, 10);
+}

package/src/shared/http.ts

@@ -0,0 +1,63 @@
+export class RequestError extends Error {
+  constructor(
+    message: string,
+    public readonly status: number | null = null,
+    public readonly code = "request_failed",
+  ) {
+    super(message);
+  }
+}
+
+export async function requestJson<T>(url: string, init: RequestInit = {}, timeoutMs = 30_000): Promise<T> {
+  const signal = AbortSignal.timeout(timeoutMs);
+  let response: Response;
+  try {
+    response = await fetch(url, { ...init, signal });
+  } catch {
+    throw new RequestError("Request failed or timed out", null, "network_error");
+  }
+  if (!response.ok) {
+    // Provider bodies and request URLs can contain credentials; never echo them into evidence.
+    const code =
+      response.status === 401
+        ? "unauthenticated"
+        : response.status === 403
+          ? "forbidden"
+          : response.status === 429
+            ? "quota_exceeded"
+            : response.status === 404
+              ? "not_found"
+              : "provider_error";
+    throw new RequestError(`Provider returned HTTP ${response.status}`, response.status, code);
+  }
+  try {
+    return (await response.json()) as T;
+  } catch (error) {
+    if (signal.aborted || !(error instanceof SyntaxError))
+      throw new RequestError("Response body failed or timed out", null, "network_error");
+    throw new RequestError("Provider returned invalid JSON", response.status, "invalid_response");
+  }
+}
+
+export async function readBounded(response: Response, maxBytes: number): Promise<string> {
+  if (Number(response.headers.get("content-length")) > maxBytes) {
+    await response.body?.cancel();
+    throw new RequestError("Response exceeds byte limit", null, "size_limit");
+  }
+  const reader = response.body?.getReader();
+  if (!reader) return "";
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      size += value.byteLength;
+      if (size > maxBytes) throw new RequestError("Response exceeds byte limit", null, "size_limit");
+      chunks.push(value);
+    }
+  } finally {
+    await reader.cancel();
+  }
+  return Buffer.concat(chunks).toString("utf8");
+}

package/src/tools/ai.ts

@@ -1,8 +1,7 @@
 import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
-import { auditAiCrawlers, type CrawlerStatus, fetchRobotsTxt, isAllowed, type RobotsTxt } from "../lib/robots.js";
-
-// --- llms.txt detection ---
+import { readBounded, RequestError } from "../shared/http.js";
+import { auditAiCrawlers, type CrawlerStatus, fetchRobotsTxt, isAllowed, type RobotsTxt } from "../web/robots.js";
 
 interface LlmsTxtResult {
   exists: boolean;
@@ -15,32 +14,20 @@ interface LlmsTxtResult {
 
 async function checkLlmsTxt(origin: string, path: string): Promise<LlmsTxtResult> {
   try {
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 10_000);
     const res = await fetch(`${origin}${path}`, {
       headers: { "User-Agent": "Pagesight/1.0" },
       redirect: "follow",
-      signal: controller.signal,
+      signal: AbortSignal.timeout(10_000),
     });
-    clearTimeout(timeout);
     if (!res.ok) return { exists: false, size: null, firstLine: null, lineCount: null, sections: [], linkCount: null };
-    // Cap at 1MB to avoid OOM on large responses
-    const contentLength = Number(res.headers.get("content-length") ?? 0);
-    if (contentLength > 1_048_576) {
-      return {
-        exists: true,
-        size: contentLength,
-        firstLine: "(file too large to preview)",
-        lineCount: null,
-        sections: [],
-        linkCount: null,
-      };
-    }
-    const text = await res.text();
-    if (text.length > 1_048_576) {
+    let text: string;
+    try {
+      text = await readBounded(res, 1_048_576);
+    } catch (error) {
+      if (!(error instanceof RequestError) || error.code !== "size_limit") throw error;
       return {
         exists: true,
-        size: text.length,
+        size: null,
         firstLine: "(file too large to preview)",
         lineCount: null,
         sections: [],
@@ -51,7 +38,14 @@ async function checkLlmsTxt(origin: string, path: string): Promise<LlmsTxtResult
     const firstLine = textLines.find((l) => l.trim().length > 0)?.trim() ?? null;
     const sections = textLines.filter((l) => /^##\s/.test(l)).map((l) => l.replace(/^##\s+/, "").trim());
     const linkCount = (text.match(/\[.*?\]\(https?:\/\/[^)]+\)/g) ?? []).length;
-    return { exists: true, size: text.length, firstLine, lineCount: textLines.length, sections, linkCount };
+    return {
+      exists: true,
+      size: new TextEncoder().encode(text).byteLength,
+      firstLine,
+      lineCount: textLines.length,
+      sections,
+      linkCount,
+    };
   } catch {
     return { exists: false, size: null, firstLine: null, lineCount: null, sections: [], linkCount: null };
   }
@@ -84,9 +78,7 @@ function formatLlmsTxt(llmsTxt: LlmsTxtResult, llmsFullTxt: LlmsTxtResult): stri
   return lines;
 }
 
-// --- Category summary ---
-
-function formatCategorySummary(crawlers: CrawlerStatus[]): string[] {
+export function formatCategorySummary(crawlers: CrawlerStatus[]): string[] {
   const categoryOrder = ["Training", "Search", "Assistant", "Agent", "Other"];
   const summary = new Map<string, { blocked: number; allowed: number }>();
 
@@ -245,7 +237,7 @@ function formatRobotsAudit(origin: string, robots: RobotsTxt, statusCode: number
 export function registerAiTool(server: McpServer): void {
   server.tool(
     "ai",
-    "Analyze your site's AI visibility. Audits AI crawler access (training, search, assistant, agent) via robots.txt, checks for llms.txt and llms-full.txt, validates syntax per RFC 9309, and tests specific path access. Shows how AI systems see your site.",
+    "Analyze your site's AI visibility. Audits AI crawler access (training, search, assistant, agent) via robots.txt, checks for llms.txt and llms-full.txt, reports robots.txt parse errors, and tests specific path access. Shows how AI systems see your site.",
     {
       url: z
         .string()