pagesight 0.17.0 → 0.18.0

package/src/{lib → web}/robots.ts

@@ -1,5 +1,5 @@
 /**
- * robots.txt parser per RFC 9309
+ * robots.txt parser based on RFC 9309
  * https://www.rfc-editor.org/rfc/rfc9309
  *
  * AI crawler registry sourced from:
@@ -152,17 +152,24 @@ export function parseRobotsTxt(raw: string): RobotsTxt {
   return { groups, sitemaps, raw, errors };
 }
 
-// --- Matching (per RFC 9309) ---
+// --- Matching ---
+
+// RFC 9309 §2.2.2: decode only unreserved ASCII; preserve reserved escapes on both sides.
+function normalizePath(path: string): string {
+  return path.replace(/%[0-9a-f]{2}|\P{ASCII}/giu, (part) => {
+    if (part.startsWith("%")) {
+      const decoded = String.fromCharCode(Number.parseInt(part.slice(1), 16));
+      return /^[a-z0-9._~-]$/i.test(decoded) ? decoded : part.toUpperCase();
+    }
+    return encodeURIComponent(part.toWellFormed());
+  });
+}
 
 function pathMatches(pattern: string, path: string): boolean {
   if (!pattern) return false;
 
-  // RFC 9309 §2.2.2: decode percent-encoded characters for comparison
-  try {
-    path = decodeURIComponent(path);
-  } catch {
-    // malformed encoding, use as-is
-  }
+  path = normalizePath(path);
+  pattern = normalizePath(pattern);
 
   let regex = "^";
   for (let i = 0; i < pattern.length; i++) {
@@ -231,7 +238,7 @@ export function isAllowed(
 
   for (const rule of matchedRules) {
     if (pathMatches(rule.path, path)) {
-      const ruleLength = rule.path.length;
+      const ruleLength = normalizePath(rule.path).length;
       if (ruleLength > bestLength || (ruleLength === bestLength && rule.type === "allow")) {
         bestRule = rule;
         bestLength = ruleLength;
@@ -278,14 +285,15 @@ export async function fetchRobotsTxt(origin: string): Promise<{ robotsTxt: Robot
     redirect: "follow",
   });
 
-  // RFC 9309: 4xx (except 429) = no restrictions (allow all)
-  // 5xx and 429 = assume complete disallow
+  // RFC 9309 §§2.3.1.3–4 distinguish unavailable (4xx) from unreachable (5xx).
+  // Pagesight treats 429 conservatively; Google also excludes it from the allow-on-4xx rule.
+  // https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt#handling-http-status-codes
   if (res.status >= 500 || res.status === 429) {
     const disallowAll: RobotsTxt = {
       groups: [{ userAgents: ["*"], rules: [{ type: "disallow", path: "/" }] }],
       sitemaps: [],
       raw: "",
-      errors: [`Server returned ${res.status} — treating as full disallow per RFC 9309`],
+      errors: [`Server returned ${res.status} — treating as full disallow`],
     };
     return { robotsTxt: disallowAll, statusCode: res.status };
   }

package/src/web/sitemap-inventory.ts

@@ -0,0 +1,64 @@
+import { RequestError } from "../shared/http.js";
+import { parseInventorySitemap } from "./sitemap-parser.js";
+import { fetchText } from "./fetch.js";
+
+export async function observeSitemap(url: string) {
+  const origin = new URL(url).origin;
+  const queue = [url];
+  const scheduled = new Set(queue);
+  let omittedChildReferences = 0;
+  const visited = new Set<string>();
+  const urls = new Set<string>();
+  const documents: Array<{ url: string; status: number; sha256: string; bytes: number }> = [];
+  const errors: Array<{ url: string; code: string }> = [];
+  const futureLastmods = new Set<string>();
+  const today = new Date().toISOString().slice(0, 10);
+  let bytes = 0;
+  while (queue.length && visited.size < 5 && bytes < 8_000_000) {
+    const next = queue.shift();
+    if (!next || visited.has(next)) continue;
+    visited.add(next);
+    try {
+      if (new URL(next).origin !== origin)
+        throw new RequestError("Child sitemap is outside the site origin", null, "outside_origin");
+      const page = await fetchText(next, 8_000_000 - bytes, origin);
+      if (!page.response.ok) throw new RequestError("Sitemap fetch failed", page.response.status);
+      bytes += Buffer.byteLength(page.body);
+      const parsed = parseInventorySitemap(page.body);
+      documents.push({
+        url: page.finalUrl,
+        status: page.response.status,
+        bytes: Buffer.byteLength(page.body),
+        sha256: Bun.CryptoHasher.hash("sha256", page.body, "hex"),
+      });
+      for (const child of parsed.children) {
+        if (scheduled.has(child)) continue;
+        if (scheduled.size >= 5) {
+          omittedChildReferences++;
+          continue;
+        }
+        scheduled.add(child);
+        queue.push(child);
+      }
+      for (const item of parsed.urls) urls.add(item);
+      for (const lastmod of parsed.lastmods) if (lastmod.slice(0, 10) > today) futureLastmods.add(lastmod);
+    } catch (error) {
+      errors.push({ url: next, code: error instanceof RequestError ? error.code : "fetch_failed" });
+    }
+  }
+  return {
+    sitemap: url,
+    complete: queue.length === 0 && errors.length === 0 && omittedChildReferences === 0,
+    documents,
+    urls: [...urls],
+    urlCount: urls.size,
+    bytes,
+    unvisited: queue,
+    omittedChildReferences,
+    errors,
+    futureLastmods: [...futureLastmods],
+    warnings: [
+      "Sitemap membership is not indexing evidence. Inventory is limited to five same-origin XML documents and 8 MB.",
+    ],
+  };
+}

package/src/web/sitemap-parser.ts

@@ -0,0 +1,59 @@
+import { SaxesParser, type SaxesTagNS } from "saxes";
+import { RequestError } from "../shared/http.js";
+
+export function parseInventorySitemap(xml: string) {
+  const parser = new SaxesParser({ xmlns: true });
+  const stack: SaxesTagNS[] = [];
+  const urls: string[] = [];
+  const children: string[] = [];
+  const lastmods: string[] = [];
+  let root = "";
+  let text = "";
+  let locations = 0;
+  const invalid = () => {
+    throw new RequestError("Expected well-formed sitemap XML", null, "invalid_sitemap");
+  };
+  parser.on("error", invalid);
+  parser.on("doctype", invalid);
+  parser.on("opentag", (tag) => {
+    stack.push(tag);
+    if (stack.length === 1) {
+      root = tag.local;
+      if (tag.prefix)
+        throw new RequestError(
+          "Prefixed sitemap XML is not supported by this inventory reader",
+          null,
+          "unsupported_sitemap",
+        );
+      if (
+        !["urlset", "sitemapindex"].includes(root) ||
+        (tag.uri && tag.uri !== "http://www.sitemaps.org/schemas/sitemap/0.9")
+      )
+        invalid();
+    }
+    if (stack.length === 2) {
+      if (tag.local !== (root === "urlset" ? "url" : "sitemap") || tag.uri !== stack[0].uri) invalid();
+      locations = 0;
+    }
+    if (stack.length === 3) text = "";
+    if (stack.length > 3 && ["loc", "lastmod"].includes(stack[2].local) && stack[2].uri === stack[0].uri) invalid();
+  });
+  const append = (value: string) => {
+    if (stack.length === 3) text += value;
+  };
+  parser.on("text", append);
+  parser.on("cdata", append);
+  parser.on("closetag", (tag) => {
+    if (stack.length === 3 && tag.uri === stack[0].uri) {
+      if (tag.local === "loc") {
+        if (++locations !== 1 || !text.trim()) invalid();
+        (root === "urlset" ? urls : children).push(text.trim());
+      }
+      if (tag.local === "lastmod") lastmods.push(text.trim());
+    }
+    if (stack.length === 2 && locations !== 1) invalid();
+    stack.pop();
+  });
+  parser.write(xml).close();
+  return { urls, children, lastmods };
+}

package/src/lib/auth.ts

@@ -1,187 +0,0 @@
-import { existsSync } from "node:fs";
-
-const SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly"];
-const TOKEN_URL = "https://oauth2.googleapis.com/token";
-
-interface TokenResponse {
-  access_token: string;
-  expires_in: number;
-  token_type: string;
-}
-
-let cachedToken: { token: string; expiresAt: number } | null = null;
-
-// --- Service Account Auth ---
-
-async function getServiceAccountToken(keyPath: string): Promise<string> {
-  const keyFile = JSON.parse(await Bun.file(keyPath).text());
-  if (!keyFile.client_email || !keyFile.private_key) {
-    throw new Error("Service account key file missing client_email or private_key");
-  }
-  const now = Math.floor(Date.now() / 1000);
-
-  const header = toBase64Url(JSON.stringify({ alg: "RS256", typ: "JWT" }));
-  const payload = toBase64Url(
-    JSON.stringify({
-      iss: keyFile.client_email,
-      scope: SCOPES.join(" "),
-      aud: TOKEN_URL,
-      iat: now,
-      exp: now + 3600,
-    }),
-  );
-
-  const signingInput = `${header}.${payload}`;
-  const key = await crypto.subtle.importKey(
-    "pkcs8",
-    pemToBuffer(keyFile.private_key),
-    { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
-    false,
-    ["sign"],
-  );
-  const signature = await crypto.subtle.sign("RSASSA-PKCS1-v1_5", key, new TextEncoder().encode(signingInput));
-  const jwt = `${signingInput}.${bufferToBase64Url(signature)}`;
-
-  const res = await fetch(TOKEN_URL, {
-    method: "POST",
-    headers: { "Content-Type": "application/x-www-form-urlencoded" },
-    body: `grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion=${jwt}`,
-  });
-
-  if (!res.ok) {
-    const err = await res.text();
-    throw new Error(`Service account token exchange failed: ${err}`);
-  }
-
-  const data: TokenResponse = await res.json();
-  return data.access_token;
-}
-
-function pemToBuffer(pem: string): ArrayBuffer {
-  const b64 = pem
-    .replace(/-----BEGIN PRIVATE KEY-----/g, "")
-    .replace(/-----END PRIVATE KEY-----/g, "")
-    .replace(/\s/g, "");
-  const binary = atob(b64);
-  const buf = new Uint8Array(binary.length);
-  for (let i = 0; i < binary.length; i++) buf[i] = binary.charCodeAt(i);
-  return buf.buffer;
-}
-
-function toBase64Url(input: string): string {
-  return btoa(input).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
-}
-
-function bufferToBase64Url(buf: ArrayBuffer): string {
-  const bytes = new Uint8Array(buf);
-  let binary = "";
-  for (const b of bytes) binary += String.fromCharCode(b);
-  return toBase64Url(binary);
-}
-
-// --- OAuth Refresh Token Auth ---
-
-async function getOAuthToken(clientId: string, clientSecret: string, refreshToken: string): Promise<string> {
-  const res = await fetch(TOKEN_URL, {
-    method: "POST",
-    headers: { "Content-Type": "application/x-www-form-urlencoded" },
-    body: new URLSearchParams({
-      grant_type: "refresh_token",
-      client_id: clientId,
-      client_secret: clientSecret,
-      refresh_token: refreshToken,
-    }).toString(),
-  });
-
-  if (!res.ok) {
-    const err = await res.text();
-    throw new Error(`OAuth token refresh failed: ${err}`);
-  }
-
-  const data: TokenResponse = await res.json();
-  return data.access_token;
-}
-
-// --- Public API ---
-
-export async function getAccessToken(): Promise<string> {
-  // Return cached token if still valid
-  if (cachedToken && Date.now() < cachedToken.expiresAt) {
-    return cachedToken.token;
-  }
-
-  // Try service account first
-  const saKeyPath = process.env.GSC_SERVICE_ACCOUNT_KEY;
-  if (saKeyPath && existsSync(saKeyPath)) {
-    const token = await getServiceAccountToken(saKeyPath);
-    cachedToken = { token, expiresAt: Date.now() + 3500_000 }; // ~58 min
-    return token;
-  }
-
-  // Try OAuth refresh token
-  const clientId = process.env.GSC_CLIENT_ID;
-  const clientSecret = process.env.GSC_CLIENT_SECRET;
-  const refreshToken = process.env.GSC_REFRESH_TOKEN;
-
-  if (clientId && clientSecret && refreshToken) {
-    const token = await getOAuthToken(clientId, clientSecret, refreshToken);
-    cachedToken = { token, expiresAt: Date.now() + 3500_000 };
-    return token;
-  }
-
-  throw new Error(
-    "No GSC credentials configured. Set either:\n" +
-      "  - GSC_SERVICE_ACCOUNT_KEY (path to service account JSON)\n" +
-      "  - GSC_CLIENT_ID + GSC_CLIENT_SECRET + GSC_REFRESH_TOKEN (OAuth 2.0)",
-  );
-}
-
-export function clearTokenCache(): void {
-  cachedToken = null;
-}
-
-export function getAuthMethod(): string {
-  if (process.env.GSC_SERVICE_ACCOUNT_KEY) return "service_account";
-  if (process.env.GSC_REFRESH_TOKEN) return "oauth";
-  return "none";
-}
-
-// --- OAuth Setup Helper ---
-
-export function getOAuthSetupUrl(clientId: string): string {
-  const params = new URLSearchParams({
-    client_id: clientId,
-    redirect_uri: "http://localhost",
-    response_type: "code",
-    scope: SCOPES.join(" "),
-    access_type: "offline",
-    prompt: "consent",
-  });
-  return `https://accounts.google.com/o/oauth2/auth?${params}`;
-}
-
-export async function exchangeCodeForToken(
-  clientId: string,
-  clientSecret: string,
-  code: string,
-): Promise<{ refreshToken: string; accessToken: string }> {
-  const res = await fetch(TOKEN_URL, {
-    method: "POST",
-    headers: { "Content-Type": "application/x-www-form-urlencoded" },
-    body: new URLSearchParams({
-      grant_type: "authorization_code",
-      client_id: clientId,
-      client_secret: clientSecret,
-      code,
-      redirect_uri: "http://localhost",
-    }).toString(),
-  });
-
-  if (!res.ok) {
-    const err = await res.text();
-    throw new Error(`Code exchange failed: ${err}`);
-  }
-
-  const data = (await res.json()) as Record<string, string>;
-  return { refreshToken: data.refresh_token, accessToken: data.access_token };
-}