pagesight 0.19.0 → 0.20.0

package/src/followup-manifest.ts

@@ -0,0 +1,42 @@
+import { dirname, resolve } from "node:path";
+import { RequestError } from "./shared/http.js";
+
+export async function loadFollowupManifest(path: string | undefined): Promise<unknown[]> {
+  if (!path) throw new RequestError("Missing --manifest", null, "invalid_input");
+  let manifest: unknown;
+  try {
+    manifest = await Bun.file(path).json();
+  } catch {
+    throw new RequestError("Cannot read --manifest JSON file", null, "invalid_input");
+  }
+  if (!Array.isArray(manifest) || !manifest.length || manifest.length > 50)
+    throw new RequestError("Manifest must contain 1–50 experiment entries", null, "invalid_input");
+  const base = dirname(resolve(path));
+  const read = async (value: unknown) => {
+    if (typeof value !== "string" || !value) return null;
+    try {
+      return await Bun.file(resolve(base, value)).json();
+    } catch {
+      return null;
+    }
+  };
+  const entries = [];
+  for (const item of manifest) {
+    if (!item || typeof item !== "object" || Array.isArray(item)) {
+      entries.push(null);
+      continue;
+    }
+    const entry = item as Record<string, unknown>;
+    if (Object.keys(entry).some((key) => !["label", "record", "baseline", "current"].includes(key))) {
+      entries.push(null);
+      continue;
+    }
+    entries.push({
+      label: entry.label,
+      record: await read(entry.record),
+      baseline: await read(entry.baseline),
+      ...(Object.hasOwn(entry, "current") ? { current: await read(entry.current) } : {}),
+    });
+  }
+  return entries;
+}

package/src/followup-text.ts

@@ -0,0 +1,45 @@
+import type { Evidence } from "./api/evidence.js";
+
+export function renderFollowup(evidence: Evidence): string {
+  const data = evidence.pages[0]?.response as
+    | {
+        asOf: string;
+        entries: Array<{
+          index: number;
+          label?: string;
+          id?: string;
+          status: string;
+          reason?: string;
+          confounded?: boolean;
+          reports?: Array<{
+            name: string;
+            status: string;
+            reason?: string;
+            nextCollection?: { date: string; timezone: string };
+          }>;
+        }>;
+      }
+    | undefined;
+  if (!data) return `Experiment follow-up unavailable: ${evidence.error?.message ?? evidence.status}\n`;
+  const lines = [
+    `Experiment follow-up as of ${data.asOf}`,
+    "Collection dates are planning estimates; no data is collected automatically.",
+    "",
+  ];
+  for (const entry of data.entries) {
+    lines.push(`${entry.label ?? `Entry ${entry.index + 1}`}${entry.id ? ` (${entry.id})` : ""}: ${entry.status}`);
+    if (entry.reason) lines.push(`  ${entry.reason}`);
+    if (entry.confounded) lines.push("  Declared overlapping changes confound attribution.");
+    for (const report of entry.reports ?? []) {
+      lines.push(
+        `  ${report.name}: ${report.status}${report.nextCollection ? ` — collect ${report.nextCollection.date} (${report.nextCollection.timezone})` : ""}`,
+      );
+      if (report.reason) lines.push(`    ${report.reason}`);
+    }
+    lines.push("");
+  }
+  lines.push(
+    "Readiness is per report. Review raw JSON warnings and source hashes; no causal SEO conclusion is implied.",
+  );
+  return lines.join("\n") + "\n";
+}

package/src/investigation-text.ts

@@ -0,0 +1,32 @@
+import type { Evidence } from "./api/evidence.js";
+import type { investigationBrief } from "./api/investigation.js";
+
+export function renderInvestigation(evidence: Evidence): string {
+  const response = evidence.pages[0]?.response as {
+    requestedDates: unknown;
+    context: unknown;
+    brief: ReturnType<typeof investigationBrief>;
+  };
+  const { brief } = response;
+  const quote = (value: unknown) =>
+    JSON.stringify(value).replace(
+      /[\u007f-\u009f\u2028\u2029]/gu,
+      (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`,
+    );
+  return [
+    `Pagesight investigation: ${quote(evidence.target)} (${evidence.status})`,
+    `Reporting window: ${quote(response.requestedDates)}`,
+    `Caller context (unverified): ${quote(response.context)}`,
+    ...brief.findings.map((f) => `Finding [${f.source}]: ${quote(f.statement)}`),
+    ...brief.tables.flatMap((t) => [
+      `Evidence [${t.source}]: ${quote({ displayPolicy: t.displayPolicy, dimensions: t.dimensions, metrics: t.metrics, rows: t.rows, observedRows: t.observedRows, omittedRows: t.omittedRows, unusableRows: t.unusableRows, paginationExhausted: t.paginationExhausted, collectedAt: t.collectedAt, metadata: t.metadata })}`,
+      ...t.warnings.map((w) => `Warning [${t.source}]: ${quote(w)}`),
+    ]),
+    ...brief.technical.map((t) => `Technical [${t.source}]: ${quote(t)}`),
+    "Full raw requests/responses and errors are available in JSON output.",
+    ...brief.unknowns.map((v) => `Unknown: ${quote(v)}`),
+    ...brief.nextChecks.map((v) => `Next check: ${quote(v)}`),
+    ...brief.limitations.map((v) => `Limitation: ${quote(v)}`),
+    "",
+  ].join("\n");
+}

package/src/providers/cloudflare.ts

@@ -0,0 +1,46 @@
+import { RequestError, readBounded } from "../shared/http.js";
+
+export type CloudflareQuery = { query: string; variables: Record<string, unknown> };
+export async function cloudflareGraphql(request: CloudflareQuery): Promise<unknown> {
+  const token = process.env.CLOUDFLARE_API_TOKEN;
+  if (!token)
+    throw new RequestError(
+      "Set CLOUDFLARE_API_TOKEN with read access to the selected zone's analytics",
+      null,
+      "not_configured",
+    );
+  const signal = AbortSignal.timeout(30_000);
+  let response: Response;
+  try {
+    response = await fetch("https://api.cloudflare.com/client/v4/graphql", {
+      method: "POST",
+      signal,
+      headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
+      body: JSON.stringify(request),
+    });
+  } catch {
+    throw new RequestError("Cloudflare request failed or timed out", null, "network_error");
+  }
+  if (!response.ok) {
+    await response.body?.cancel();
+    throw new RequestError(
+      `Cloudflare returned HTTP ${response.status}`,
+      response.status,
+      response.status === 401
+        ? "unauthenticated"
+        : response.status === 403
+          ? "forbidden"
+          : response.status === 429
+            ? "quota_exceeded"
+            : "provider_error",
+    );
+  }
+  try {
+    return JSON.parse(await readBounded(response, 8_000_000));
+  } catch (error) {
+    if (error instanceof RequestError) throw error;
+    if (error instanceof SyntaxError)
+      throw new RequestError("Cloudflare response could not be read as JSON", response.status, "invalid_response");
+    throw new RequestError("Cloudflare response body failed or timed out", null, "network_error");
+  }
+}

package/src/tools/observe.ts

@@ -5,7 +5,7 @@ import { RequestError } from "../shared/http.js";
 export function registerObserveTool(server: McpServer): void {
   server.tool(
     "observe",
-    "Run the shared Pagesight API. Read-only operations: discover; bing.sites/queries/pages/traffic/crawl-stats/crawl-issues/url-info/link-counts/url-links; evidence.import (unverified UI findings); gsc.sites/sitemaps/inspect/report; ga.accounts/property/key-events/report/realtime; page (metadata and image ALT evidence); speed.psi/crux/history; doctor; snapshot; assess (saved evidence summary); compare. Pass the API operation object as request. Returns structured evidence, exact effective requests, provider responses and partial errors. See the CLI --help and README for report/config shapes.",
+    "Run the shared Pagesight API. Evidence operations: page.verify (anonymous browser SEO evidence and one validated internal anchor navigation; may generate site requests); cloudflare.audit (bounded zone analytics/settings/security evidence); technical.compare (saved exact-page technical changes); change.followup (saved experiment collection dates and per-report blockers); change.evaluate (deployment context and comparable snapshot evidence); discover; bing.sites/queries/pages/traffic/crawl-stats/crawl-issues/url-info/link-counts/url-links; evidence.import (unverified UI findings); gsc.sites/sitemaps/inspect/report; ga.accounts/property/key-events/report/realtime; page (metadata and image ALT evidence); speed.psi/crux/history; doctor; snapshot; crawl (bounded robots-aware site graph and indexing sample); investigate (fresh exact-page search, organic and technical investigation); opportunities (saved search candidates); assess (saved evidence summary); compare. Pass the API operation object as request. Returns structured evidence, exact effective requests, provider responses and partial errors. See the CLI --help and README for report/config shapes.",
     { request: operationSchema },
     async ({ request }) => {
       try {

package/src/web/fetch.ts

@@ -1,6 +1,6 @@
 import { readBounded, RequestError } from "../shared/http.js";
 
-export async function fetchText(url: string, maxBytes: number, origin?: string) {
+export async function fetchText(url: string, maxBytes: number, origin?: string, proxy?: string) {
   const redirects: Array<{ url: string; status: number; location: string }> = [];
   let current = url;
   const signal = AbortSignal.timeout(20_000);
@@ -15,6 +15,7 @@ export async function fetchText(url: string, maxBytes: number, origin?: string)
       throw new RequestError("Redirect left the permitted origin or protocol", null, "invalid_redirect");
     const response = await fetch(current, {
       signal,
+      ...(proxy ? { proxy } : {}),
       redirect: "manual",
       headers: { "User-Agent": "Pagesight/0.17", Accept: "text/html,application/xml,text/plain" },
     });

package/src/web/render-browser.ts

@@ -0,0 +1,190 @@
+import { chromium, type Browser, type Page } from "playwright";
+import { RequestError } from "../shared/http.js";
+import { extractSeo, type SeoDom } from "./render-dom.js";
+
+export interface RenderOptions {
+  url: string;
+  navigation?: { fromUrl: string; linkSelector: string };
+  settleMs: number;
+  timeoutMs: number;
+  viewport: { width: number; height: number };
+}
+export async function launchRenderer(proxy: string) {
+  try {
+    return await chromium.launch({
+      timeout: 15000,
+      proxy: { server: proxy, bypass: "<-loopback>" },
+      args: ["--disable-quic", "--force-webrtc-ip-handling-policy=disable_non_proxied_udp"],
+    });
+  } catch {
+    throw new RequestError(
+      "Chromium unavailable. Run bunx playwright install chromium in the Pagesight installation and check browser OS dependencies.",
+      null,
+      "browser_unavailable",
+    );
+  }
+}
+export async function parseServer(browser: Browser, html: string, url: string) {
+  const context = await browser.newContext({ javaScriptEnabled: false, offline: true, serviceWorkers: "block" });
+  try {
+    const page = await context.newPage();
+    return await page.evaluate(extractSeo, { html, url });
+  } finally {
+    await context.close();
+  }
+}
+async function isolated<T, I>(page: Page, fn: (input: I) => T, input: I): Promise<T> {
+  const session = await page.context().newCDPSession(page);
+  try {
+    const { frameTree } = await session.send("Page.getFrameTree");
+    const { executionContextId } = await session.send("Page.createIsolatedWorld", {
+      frameId: frameTree.frame.id,
+      worldName: "pagesight-evidence",
+    });
+    const result = await session.send("Runtime.callFunctionOn", {
+      functionDeclaration: fn.toString(),
+      executionContextId,
+      arguments: [{ value: input }],
+      returnByValue: true,
+    });
+    if (result.exceptionDetails) throw new Error("Isolated extraction failed");
+    return result.result.value as T;
+  } finally {
+    await session.detach();
+  }
+}
+function documentResponse(page: Page) {
+  const documents: Array<{ url: string; status: number; location: string | null; xRobotsTag: string | null }> = [];
+  const history = { documents, documentsTruncated: false };
+  page.on("response", (response) => {
+    if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
+      if (documents.length === 20) {
+        history.documentsTruncated = true;
+        return;
+      }
+      const headers = response.headers();
+      documents.push({
+        url: response.url(),
+        status: response.status(),
+        location: headers.location ?? null,
+        xRobotsTag: headers["x-robots-tag"] ?? null,
+      });
+    }
+  });
+  return history;
+}
+export async function captureRendered(browser: Browser, options: RenderOptions, navigate: boolean) {
+  const origin = new URL(options.url).origin;
+  const context = await browser.newContext({
+    viewport: options.viewport,
+    serviceWorkers: "block",
+    acceptDownloads: false,
+  });
+  let timedOut = false;
+  const deadline = setTimeout(() => {
+    timedOut = true;
+    void context.close().catch(() => {});
+  }, options.timeoutMs);
+  let blockedRequests = 0;
+  let requests = 0;
+  try {
+    await context.routeWebSocket("**/*", async (socket) => {
+      blockedRequests++;
+      await socket.close();
+    });
+    await context.route("**/*", async (route) => {
+      const request = route.request();
+      const url = new URL(request.url());
+      if (
+        ++requests > 500 ||
+        !["http:", "https:"].includes(url.protocol) ||
+        url.username ||
+        url.password ||
+        !["GET", "HEAD"].includes(request.method()) ||
+        (request.isNavigationRequest() && request.frame().parentFrame() === null && url.origin !== origin)
+      ) {
+        blockedRequests++;
+        await route.abort();
+      } else await route.continue();
+    });
+    const page = await context.newPage();
+    page.setDefaultTimeout(options.timeoutMs);
+    const history = documentResponse(page);
+    const consoleErrors: string[] = [];
+    page.on("pageerror", (error) => {
+      if (consoleErrors.length < 20) consoleErrors.push(error.name);
+    });
+    const initialUrl = navigate ? options.navigation!.fromUrl : options.url;
+    const initial = await page.goto(initialUrl, { waitUntil: "domcontentloaded" });
+    if (!initial || !initial.headers()["content-type"]?.includes("text/html"))
+      throw new RequestError("Browser document is not HTML.", initial?.status() ?? null, "not_html");
+    await page.waitForTimeout(options.settleMs);
+    let clickedLink: { selector: string; href: string } | null = null;
+    let source: SeoDom | null = null;
+    if (navigate) {
+      source = await isolated(page, extractSeo, { url: page.url() });
+      const link = page.locator(`css=${options.navigation!.linkSelector}`);
+      if ((await link.count()) !== 1)
+        throw new RequestError("Navigation selector must identify exactly one anchor.", null, "ambiguous_link");
+      const selected = await isolated(
+        page,
+        (selector: string) => {
+          const element = document.querySelector(selector)!;
+          return {
+            tag: element.tagName,
+            href: (element as HTMLAnchorElement).href,
+            target: element.getAttribute("target"),
+            download: element.hasAttribute("download"),
+          };
+        },
+        options.navigation!.linkSelector,
+      );
+      if (
+        selected.tag !== "A" ||
+        selected.href !== options.url ||
+        selected.download ||
+        (selected.target && selected.target !== "_self")
+      )
+        throw new RequestError(
+          "Select a same-tab anchor whose resolved href exactly matches the requested URL.",
+          null,
+          "invalid_link",
+        );
+      clickedLink = { selector: options.navigation!.linkSelector, href: selected.href };
+      await link.click({ noWaitAfter: true });
+      await page.waitForURL((url) => url.href === options.url, { waitUntil: "domcontentloaded" });
+      await page.waitForTimeout(options.settleMs);
+    }
+    const dom = await isolated(page, extractSeo, { url: page.url() });
+    return {
+      path: navigate ? "internal_navigation" : "direct",
+      initialUrl,
+      requestedUrl: options.url,
+      finalUrl: page.url(),
+      capturedAt: new Date().toISOString(),
+      dom,
+      domSha256: Bun.CryptoHasher.hash("sha256", JSON.stringify(dom), "hex"),
+      source,
+      clickedLink,
+      ...history,
+      blockedRequests,
+      requests,
+      pageErrorNames: consoleErrors,
+      targetDocumentResponse: history.documentsTruncated
+        ? null
+        : ([...history.documents].reverse().find((d) => d.url === page.url().split("#")[0]) ?? null),
+    };
+  } catch (error) {
+    if (error instanceof RequestError) throw error;
+    throw new RequestError(
+      timedOut
+        ? "Browser capture exceeded its time budget."
+        : "Browser navigation or capture failed; no rendered evidence was inferred.",
+      null,
+      timedOut ? "render_timeout" : "render_failed",
+    );
+  } finally {
+    clearTimeout(deadline);
+    await context.close().catch(() => {});
+  }
+}

package/src/web/render-dom.ts

@@ -0,0 +1,72 @@
+// Executed in Chromium for both detached server HTML and the rendered document.
+export function extractSeo(input: { html?: string; url: string }) {
+  const doc = input.html === undefined ? document : new DOMParser().parseFromString(input.html, "text/html");
+  const maxItems = 200;
+  const maxText = 4000;
+  let truncated = false;
+  const clip = (value: string | null) => {
+    if (value !== null && value.length > maxText) truncated = true;
+    return value === null ? null : value.slice(0, maxText);
+  };
+  const select = (selector: string) => {
+    const matches: Element[] = [];
+    const walker = doc.createTreeWalker(doc, NodeFilter.SHOW_ELEMENT);
+    let node: Node | null;
+    while ((node = walker.nextNode())) {
+      if (!(node as Element).matches(selector)) continue;
+      if (matches.length === maxItems) {
+        truncated = true;
+        break;
+      }
+      matches.push(node as Element);
+    }
+    return matches;
+  };
+  let baseUrl = input.url;
+  const baseHref = doc.querySelector("base[href]")?.getAttribute("href");
+  try {
+    if (baseHref) baseUrl = new URL(baseHref, input.url).href;
+  } catch {
+    /* Invalid base remains visible below. */
+  }
+  const resolve = (href: string | null) => {
+    try {
+      return href === null ? null : new URL(href, baseUrl).href;
+    } catch {
+      return null;
+    }
+  };
+  return {
+    url: input.url,
+    baseHref: clip(baseHref ?? null),
+    titles: select("title").map((e) => clip(e.textContent)),
+    descriptions: select('meta[name="description" i]').map((e) => clip(e.getAttribute("content"))),
+    canonicals: select('link[rel~="canonical" i]').map((e) => ({
+      raw: clip(e.getAttribute("href")),
+      resolved: clip(resolve(e.getAttribute("href"))),
+    })),
+    robots: select('meta[name="robots" i],meta[name="googlebot" i]').map((e) => ({
+      name: clip(e.getAttribute("name")),
+      content: clip(e.getAttribute("content")),
+    })),
+    h1: select("h1").map((e) => clip(e.textContent)),
+    jsonLd: select('script[type="application/ld+json" i]').map((e) => {
+      const raw = e.textContent ?? "";
+      let validJson = true;
+      try {
+        JSON.parse(raw);
+      } catch {
+        validJson = false;
+      }
+      return { raw: clip(raw), validJson };
+    }),
+    links: select("a[href]").map((e) => ({
+      raw: clip(e.getAttribute("href")),
+      resolved: clip(resolve(e.getAttribute("href"))),
+      text: clip(e.textContent),
+    })),
+    truncated,
+    limits: { maxItemsPerField: maxItems, maxCharactersPerValue: maxText },
+  };
+}
+export type SeoDom = ReturnType<typeof extractSeo>;

package/src/web/render-network.ts

@@ -0,0 +1,122 @@
+import { lookup } from "node:dns/promises";
+import { createServer, request, type OutgoingHttpHeaders } from "node:http";
+import { pipeline } from "node:stream";
+import { connect, type Socket } from "node:net";
+import ipaddr from "ipaddr.js";
+
+const hostname = (url: URL) => url.hostname.replace(/^\[|\]$/g, "");
+const port = (url: URL) => Number(url.port || (url.protocol === "https:" ? 443 : 80));
+export function publicAddress(address: string) {
+  if (address.includes("%") || !ipaddr.isValid(address)) return false;
+  return ipaddr.process(address).range() === "unicast";
+}
+export async function checkedRenderAddress(
+  url: URL,
+  target: URL,
+  resolve: (host: string) => Promise<Array<{ address: string }>> = (host) => lookup(host, { all: true }),
+) {
+  const host = hostname(url);
+  const addresses = ipaddr.isValid(host) ? [{ address: host }] : await resolve(host);
+  const explicitLocal = ipaddr.isValid(hostname(target)) && host === hostname(target) && port(url) === port(target);
+  if (!addresses.length || (!explicitLocal && addresses.some(({ address }) => !publicAddress(address))))
+    throw new Error("Network destination blocked");
+  return addresses[0]!.address;
+}
+export async function startRenderNetwork(target: string) {
+  const permitted = new URL(target);
+  const sockets = new Set<Socket>();
+  let blockedRequests = 0;
+  let attempts = 0;
+  let closed = false;
+  const destination = async (url: URL) => {
+    try {
+      if (++attempts > 1500) throw new Error("Connection budget exceeded");
+      const address = await checkedRenderAddress(url, permitted);
+      if (closed) throw new Error("Network closed");
+      return address;
+    } catch (error) {
+      blockedRequests++;
+      throw error;
+    }
+  };
+  const track = (socket: Socket) => {
+    sockets.add(socket);
+    socket.setTimeout(60000, () => socket.destroy());
+    socket.on("close", () => sockets.delete(socket));
+    socket.on("error", () => socket.destroy());
+    return socket;
+  };
+  // CONNECT needs raw sockets; pin the validated IP at the socket, never resolve it again.
+  const server = createServer(async (incoming, outgoing) => {
+    try {
+      const url = new URL(incoming.url!);
+      if (url.protocol !== "http:" || url.username || url.password || !["GET", "HEAD"].includes(incoming.method!))
+        throw new Error("Invalid proxy request");
+      const address = await destination(url);
+      const headers: OutgoingHttpHeaders = { ...incoming.headers, host: url.host };
+      delete headers["proxy-connection"];
+      delete headers["proxy-authorization"];
+      const upstream = request(
+        {
+          hostname: address,
+          port: port(url),
+          path: url.pathname + url.search,
+          method: incoming.method,
+          headers,
+          agent: false,
+        },
+        (response) => {
+          outgoing.writeHead(response.statusCode!, response.headers);
+          pipeline(response, outgoing, () => upstream.destroy());
+        },
+      );
+      upstream.on("socket", track);
+      upstream.on("error", () => {
+        outgoing.destroy();
+      });
+      outgoing.on("close", () => upstream.destroy());
+      pipeline(incoming, upstream, (error) => {
+        if (error) outgoing.destroy();
+      });
+    } catch {
+      outgoing.writeHead(403).end();
+    }
+  });
+  server.maxConnections = 256;
+  server.on("connection", track);
+  server.on("connect", async (incoming, client, head) => {
+    try {
+      const url = new URL(`https://${incoming.url}`);
+      if (url.username || url.password || url.pathname !== "/" || url.search || url.hash)
+        throw new Error("Invalid tunnel");
+      const address = await destination(url);
+      const upstream = track(connect({ host: address, port: port(url) }));
+      upstream.once("connect", () => {
+        client.write("HTTP/1.1 200 Connection Established\r\n\r\n");
+        if (head.length) upstream.write(head);
+        client.pipe(upstream);
+        upstream.pipe(client);
+      });
+      client.on("close", () => upstream.destroy());
+      upstream.on("close", () => client.destroy());
+    } catch {
+      client.end("HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n");
+    }
+  });
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", resolve);
+  });
+  const address = server.address() as { port: number };
+  return {
+    proxy: `http://127.0.0.1:${address.port}`,
+    get blockedRequests() {
+      return blockedRequests;
+    },
+    async close() {
+      closed = true;
+      for (const socket of sockets) socket.destroy();
+      await new Promise<void>((resolve) => server.close(() => resolve()));
+    },
+  };
+}