pagesight 0.2.1 → 0.2.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pagesight",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "See your site the way search engines and AI see it.",
   "keywords": [
     "seo",

package/src/lib/auth.ts

@@ -17,8 +17,8 @@ async function getServiceAccountToken(keyPath: string): Promise<string> {
   const keyFile = JSON.parse(await Bun.file(keyPath).text());
   const now = Math.floor(Date.now() / 1000);
 
-  const header = btoa(JSON.stringify({ alg: "RS256", typ: "JWT" }));
-  const payload = btoa(
+  const header = toBase64Url(JSON.stringify({ alg: "RS256", typ: "JWT" }));
+  const payload = toBase64Url(
     JSON.stringify({
       iss: keyFile.client_email,
       scope: SCOPES.join(" "),
@@ -65,11 +65,15 @@ function pemToBuffer(pem: string): ArrayBuffer {
   return buf.buffer;
 }
 
+function toBase64Url(input: string): string {
+  return btoa(input).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+}
+
 function bufferToBase64Url(buf: ArrayBuffer): string {
   const bytes = new Uint8Array(buf);
   let binary = "";
   for (const b of bytes) binary += String.fromCharCode(b);
-  return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+  return toBase64Url(binary);
 }
 
 // --- OAuth Refresh Token Auth ---
@@ -140,7 +144,7 @@ export function getAuthMethod(): string {
 export function getOAuthSetupUrl(clientId: string): string {
   const params = new URLSearchParams({
     client_id: clientId,
-    redirect_uri: "urn:ietf:wg:oauth:2.0:oob",
+    redirect_uri: "http://localhost",
     response_type: "code",
     scope: SCOPES.join(" "),
     access_type: "offline",
@@ -162,7 +166,7 @@ export async function exchangeCodeForToken(
       client_id: clientId,
       client_secret: clientSecret,
       code,
-      redirect_uri: "urn:ietf:wg:oauth:2.0:oob",
+      redirect_uri: "http://localhost",
     }).toString(),
   });
 

package/src/lib/robots.ts

@@ -144,10 +144,8 @@ export function parseRobotsTxt(raw: string): RobotsTxt {
     } else if (directive === "sitemap") {
       if (value) sitemaps.push(value);
       else errors.push(`Line ${lineNum}: Empty sitemap URL`);
-    } else if (directive === "crawl-delay" || directive === "host") {
-      // Known non-standard directives — ignore silently
     } else {
-      errors.push(`Line ${lineNum}: Unknown directive "${directive}"`);
+      // RFC 9309 allows parser-specific extensions — ignore unknown directives silently
     }
   }
 
@@ -190,36 +188,41 @@ export function isAllowed(
 } {
   const ua = userAgent.toLowerCase();
 
-  let matchingGroup: RobotsGroup | null = null;
+  // RFC 9309: collect ALL rules from groups matching this user-agent
+  // Try specific user-agent match first, merging all matching groups
+  const matchedRules: Array<{ type: "allow" | "disallow"; path: string }> = [];
   let matchedGroupName: string | null = null;
+  let foundSpecific = false;
 
   for (const group of robots.groups) {
     for (const agent of group.userAgents) {
       if (agent.toLowerCase() === ua) {
-        matchingGroup = group;
+        matchedRules.push(...group.rules);
         matchedGroupName = agent;
-        break;
+        foundSpecific = true;
       }
     }
-    if (matchingGroup) break;
   }
 
-  if (!matchingGroup) {
+  // Fall back to wildcard if no specific match
+  if (!foundSpecific) {
     for (const group of robots.groups) {
       if (group.userAgents.some((a) => a === "*")) {
-        matchingGroup = group;
+        matchedRules.push(...group.rules);
         matchedGroupName = "*";
-        break;
       }
     }
   }
 
-  if (!matchingGroup) return { allowed: true, matchedRule: null, matchedGroup: null };
+  if (matchedRules.length === 0 && !matchedGroupName) {
+    return { allowed: true, matchedRule: null, matchedGroup: null };
+  }
 
+  // Find the most specific (longest path) matching rule
   let bestRule: { type: "allow" | "disallow"; path: string } | null = null;
   let bestLength = -1;
 
-  for (const rule of matchingGroup.rules) {
+  for (const rule of matchedRules) {
     if (pathMatches(rule.path, path)) {
       const ruleLength = rule.path.length;
       if (ruleLength > bestLength || (ruleLength === bestLength && rule.type === "allow")) {
@@ -268,6 +271,18 @@ export async function fetchRobotsTxt(origin: string): Promise<{ robotsTxt: Robot
     redirect: "follow",
   });
 
+  // RFC 9309: 4xx (except 429) = no restrictions (allow all)
+  // 5xx and 429 = assume complete disallow
+  if (res.status >= 500 || res.status === 429) {
+    const disallowAll: RobotsTxt = {
+      groups: [{ userAgents: ["*"], rules: [{ type: "disallow", path: "/" }] }],
+      sitemaps: [],
+      raw: "",
+      errors: [`Server returned ${res.status} — treating as full disallow per RFC 9309`],
+    };
+    return { robotsTxt: disallowAll, statusCode: res.status };
+  }
+
   if (res.status >= 400) {
     return { robotsTxt: { groups: [], sitemaps: [], raw: "", errors: [] }, statusCode: res.status };
   }

package/src/tools/robots.ts

@@ -50,6 +50,11 @@ function formatRobotsAudit(origin: string, robots: RobotsTxt, statusCode: number
   const blocked = crawlers.filter((c) => !c.allowed);
   const allowed = crawlers.filter((c) => c.allowed);
 
+  if (crawlers.length === 0) {
+    lines.push("", "--- AI Crawlers ---", "", "Could not load AI crawler registry. Audit skipped.");
+    return lines.join("\n");
+  }
+
   lines.push(
     "",
     `--- AI Crawlers: ${blocked.length} blocked, ${allowed.length} allowed (of ${crawlers.length} known) ---`,
@@ -58,7 +63,7 @@ function formatRobotsAudit(origin: string, robots: RobotsTxt, statusCode: number
   );
 
   if (blocked.length === 0) {
-    lines.push("", "All 139 known AI crawlers are allowed. No bots are explicitly blocked.");
+    lines.push("", `All ${crawlers.length} known AI crawlers are allowed. No bots are explicitly blocked.`);
   } else if (blocked.length === crawlers.length) {
     lines.push("", "All known AI crawlers are blocked.");
     // Show how they're blocked

package/src/tools/setup.ts

@@ -87,25 +87,30 @@ export function registerSetupTool(server: McpServer): void {
         if (!client_id || !client_secret || !code) {
           return { content: [{ type: "text", text: "Error: client_id, client_secret, and code are all required." }] };
         }
-        const tokens = await exchangeCodeForToken(client_id, client_secret, code);
-        return {
-          content: [
-            {
-              type: "text",
-              text: [
-                "=== OAuth Setup Complete ===",
-                "",
-                "Add these environment variables to your MCP server config:",
-                "",
-                `GSC_CLIENT_ID=${client_id}`,
-                `GSC_CLIENT_SECRET=${client_secret}`,
-                `GSC_REFRESH_TOKEN=${tokens.refreshToken}`,
-                "",
-                "Then restart Pagesight.",
-              ].join("\n"),
-            },
-          ],
-        };
+        try {
+          const tokens = await exchangeCodeForToken(client_id, client_secret, code);
+          return {
+            content: [
+              {
+                type: "text",
+                text: [
+                  "=== OAuth Setup Complete ===",
+                  "",
+                  "Add these environment variables to your MCP server config:",
+                  "",
+                  `GSC_CLIENT_ID=${client_id}`,
+                  "GSC_CLIENT_SECRET=(use the client_secret you already have)",
+                  `GSC_REFRESH_TOKEN=${tokens.refreshToken}`,
+                  "",
+                  "Then restart Pagesight.",
+                ].join("\n"),
+              },
+            ],
+          };
+        } catch (err) {
+          const msg = err instanceof Error ? err.message : String(err);
+          return { content: [{ type: "text", text: `Error exchanging code: ${msg}` }] };
+        }
       }
 
       return { content: [{ type: "text", text: "Unknown action." }] };