pagesight 0.18.0 → 0.20.0

package/src/web/render-browser.ts

@@ -0,0 +1,190 @@
+import { chromium, type Browser, type Page } from "playwright";
+import { RequestError } from "../shared/http.js";
+import { extractSeo, type SeoDom } from "./render-dom.js";
+
+export interface RenderOptions {
+  url: string;
+  navigation?: { fromUrl: string; linkSelector: string };
+  settleMs: number;
+  timeoutMs: number;
+  viewport: { width: number; height: number };
+}
+export async function launchRenderer(proxy: string) {
+  try {
+    return await chromium.launch({
+      timeout: 15000,
+      proxy: { server: proxy, bypass: "<-loopback>" },
+      args: ["--disable-quic", "--force-webrtc-ip-handling-policy=disable_non_proxied_udp"],
+    });
+  } catch {
+    throw new RequestError(
+      "Chromium unavailable. Run bunx playwright install chromium in the Pagesight installation and check browser OS dependencies.",
+      null,
+      "browser_unavailable",
+    );
+  }
+}
+export async function parseServer(browser: Browser, html: string, url: string) {
+  const context = await browser.newContext({ javaScriptEnabled: false, offline: true, serviceWorkers: "block" });
+  try {
+    const page = await context.newPage();
+    return await page.evaluate(extractSeo, { html, url });
+  } finally {
+    await context.close();
+  }
+}
+async function isolated<T, I>(page: Page, fn: (input: I) => T, input: I): Promise<T> {
+  const session = await page.context().newCDPSession(page);
+  try {
+    const { frameTree } = await session.send("Page.getFrameTree");
+    const { executionContextId } = await session.send("Page.createIsolatedWorld", {
+      frameId: frameTree.frame.id,
+      worldName: "pagesight-evidence",
+    });
+    const result = await session.send("Runtime.callFunctionOn", {
+      functionDeclaration: fn.toString(),
+      executionContextId,
+      arguments: [{ value: input }],
+      returnByValue: true,
+    });
+    if (result.exceptionDetails) throw new Error("Isolated extraction failed");
+    return result.result.value as T;
+  } finally {
+    await session.detach();
+  }
+}
+function documentResponse(page: Page) {
+  const documents: Array<{ url: string; status: number; location: string | null; xRobotsTag: string | null }> = [];
+  const history = { documents, documentsTruncated: false };
+  page.on("response", (response) => {
+    if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
+      if (documents.length === 20) {
+        history.documentsTruncated = true;
+        return;
+      }
+      const headers = response.headers();
+      documents.push({
+        url: response.url(),
+        status: response.status(),
+        location: headers.location ?? null,
+        xRobotsTag: headers["x-robots-tag"] ?? null,
+      });
+    }
+  });
+  return history;
+}
+export async function captureRendered(browser: Browser, options: RenderOptions, navigate: boolean) {
+  const origin = new URL(options.url).origin;
+  const context = await browser.newContext({
+    viewport: options.viewport,
+    serviceWorkers: "block",
+    acceptDownloads: false,
+  });
+  let timedOut = false;
+  const deadline = setTimeout(() => {
+    timedOut = true;
+    void context.close().catch(() => {});
+  }, options.timeoutMs);
+  let blockedRequests = 0;
+  let requests = 0;
+  try {
+    await context.routeWebSocket("**/*", async (socket) => {
+      blockedRequests++;
+      await socket.close();
+    });
+    await context.route("**/*", async (route) => {
+      const request = route.request();
+      const url = new URL(request.url());
+      if (
+        ++requests > 500 ||
+        !["http:", "https:"].includes(url.protocol) ||
+        url.username ||
+        url.password ||
+        !["GET", "HEAD"].includes(request.method()) ||
+        (request.isNavigationRequest() && request.frame().parentFrame() === null && url.origin !== origin)
+      ) {
+        blockedRequests++;
+        await route.abort();
+      } else await route.continue();
+    });
+    const page = await context.newPage();
+    page.setDefaultTimeout(options.timeoutMs);
+    const history = documentResponse(page);
+    const consoleErrors: string[] = [];
+    page.on("pageerror", (error) => {
+      if (consoleErrors.length < 20) consoleErrors.push(error.name);
+    });
+    const initialUrl = navigate ? options.navigation!.fromUrl : options.url;
+    const initial = await page.goto(initialUrl, { waitUntil: "domcontentloaded" });
+    if (!initial || !initial.headers()["content-type"]?.includes("text/html"))
+      throw new RequestError("Browser document is not HTML.", initial?.status() ?? null, "not_html");
+    await page.waitForTimeout(options.settleMs);
+    let clickedLink: { selector: string; href: string } | null = null;
+    let source: SeoDom | null = null;
+    if (navigate) {
+      source = await isolated(page, extractSeo, { url: page.url() });
+      const link = page.locator(`css=${options.navigation!.linkSelector}`);
+      if ((await link.count()) !== 1)
+        throw new RequestError("Navigation selector must identify exactly one anchor.", null, "ambiguous_link");
+      const selected = await isolated(
+        page,
+        (selector: string) => {
+          const element = document.querySelector(selector)!;
+          return {
+            tag: element.tagName,
+            href: (element as HTMLAnchorElement).href,
+            target: element.getAttribute("target"),
+            download: element.hasAttribute("download"),
+          };
+        },
+        options.navigation!.linkSelector,
+      );
+      if (
+        selected.tag !== "A" ||
+        selected.href !== options.url ||
+        selected.download ||
+        (selected.target && selected.target !== "_self")
+      )
+        throw new RequestError(
+          "Select a same-tab anchor whose resolved href exactly matches the requested URL.",
+          null,
+          "invalid_link",
+        );
+      clickedLink = { selector: options.navigation!.linkSelector, href: selected.href };
+      await link.click({ noWaitAfter: true });
+      await page.waitForURL((url) => url.href === options.url, { waitUntil: "domcontentloaded" });
+      await page.waitForTimeout(options.settleMs);
+    }
+    const dom = await isolated(page, extractSeo, { url: page.url() });
+    return {
+      path: navigate ? "internal_navigation" : "direct",
+      initialUrl,
+      requestedUrl: options.url,
+      finalUrl: page.url(),
+      capturedAt: new Date().toISOString(),
+      dom,
+      domSha256: Bun.CryptoHasher.hash("sha256", JSON.stringify(dom), "hex"),
+      source,
+      clickedLink,
+      ...history,
+      blockedRequests,
+      requests,
+      pageErrorNames: consoleErrors,
+      targetDocumentResponse: history.documentsTruncated
+        ? null
+        : ([...history.documents].reverse().find((d) => d.url === page.url().split("#")[0]) ?? null),
+    };
+  } catch (error) {
+    if (error instanceof RequestError) throw error;
+    throw new RequestError(
+      timedOut
+        ? "Browser capture exceeded its time budget."
+        : "Browser navigation or capture failed; no rendered evidence was inferred.",
+      null,
+      timedOut ? "render_timeout" : "render_failed",
+    );
+  } finally {
+    clearTimeout(deadline);
+    await context.close().catch(() => {});
+  }
+}

package/src/web/render-dom.ts

@@ -0,0 +1,72 @@
+// Executed in Chromium for both detached server HTML and the rendered document.
+export function extractSeo(input: { html?: string; url: string }) {
+  const doc = input.html === undefined ? document : new DOMParser().parseFromString(input.html, "text/html");
+  const maxItems = 200;
+  const maxText = 4000;
+  let truncated = false;
+  const clip = (value: string | null) => {
+    if (value !== null && value.length > maxText) truncated = true;
+    return value === null ? null : value.slice(0, maxText);
+  };
+  const select = (selector: string) => {
+    const matches: Element[] = [];
+    const walker = doc.createTreeWalker(doc, NodeFilter.SHOW_ELEMENT);
+    let node: Node | null;
+    while ((node = walker.nextNode())) {
+      if (!(node as Element).matches(selector)) continue;
+      if (matches.length === maxItems) {
+        truncated = true;
+        break;
+      }
+      matches.push(node as Element);
+    }
+    return matches;
+  };
+  let baseUrl = input.url;
+  const baseHref = doc.querySelector("base[href]")?.getAttribute("href");
+  try {
+    if (baseHref) baseUrl = new URL(baseHref, input.url).href;
+  } catch {
+    /* Invalid base remains visible below. */
+  }
+  const resolve = (href: string | null) => {
+    try {
+      return href === null ? null : new URL(href, baseUrl).href;
+    } catch {
+      return null;
+    }
+  };
+  return {
+    url: input.url,
+    baseHref: clip(baseHref ?? null),
+    titles: select("title").map((e) => clip(e.textContent)),
+    descriptions: select('meta[name="description" i]').map((e) => clip(e.getAttribute("content"))),
+    canonicals: select('link[rel~="canonical" i]').map((e) => ({
+      raw: clip(e.getAttribute("href")),
+      resolved: clip(resolve(e.getAttribute("href"))),
+    })),
+    robots: select('meta[name="robots" i],meta[name="googlebot" i]').map((e) => ({
+      name: clip(e.getAttribute("name")),
+      content: clip(e.getAttribute("content")),
+    })),
+    h1: select("h1").map((e) => clip(e.textContent)),
+    jsonLd: select('script[type="application/ld+json" i]').map((e) => {
+      const raw = e.textContent ?? "";
+      let validJson = true;
+      try {
+        JSON.parse(raw);
+      } catch {
+        validJson = false;
+      }
+      return { raw: clip(raw), validJson };
+    }),
+    links: select("a[href]").map((e) => ({
+      raw: clip(e.getAttribute("href")),
+      resolved: clip(resolve(e.getAttribute("href"))),
+      text: clip(e.textContent),
+    })),
+    truncated,
+    limits: { maxItemsPerField: maxItems, maxCharactersPerValue: maxText },
+  };
+}
+export type SeoDom = ReturnType<typeof extractSeo>;

package/src/web/render-network.ts

@@ -0,0 +1,122 @@
+import { lookup } from "node:dns/promises";
+import { createServer, request, type OutgoingHttpHeaders } from "node:http";
+import { pipeline } from "node:stream";
+import { connect, type Socket } from "node:net";
+import ipaddr from "ipaddr.js";
+
+const hostname = (url: URL) => url.hostname.replace(/^\[|\]$/g, "");
+const port = (url: URL) => Number(url.port || (url.protocol === "https:" ? 443 : 80));
+export function publicAddress(address: string) {
+  if (address.includes("%") || !ipaddr.isValid(address)) return false;
+  return ipaddr.process(address).range() === "unicast";
+}
+export async function checkedRenderAddress(
+  url: URL,
+  target: URL,
+  resolve: (host: string) => Promise<Array<{ address: string }>> = (host) => lookup(host, { all: true }),
+) {
+  const host = hostname(url);
+  const addresses = ipaddr.isValid(host) ? [{ address: host }] : await resolve(host);
+  const explicitLocal = ipaddr.isValid(hostname(target)) && host === hostname(target) && port(url) === port(target);
+  if (!addresses.length || (!explicitLocal && addresses.some(({ address }) => !publicAddress(address))))
+    throw new Error("Network destination blocked");
+  return addresses[0]!.address;
+}
+export async function startRenderNetwork(target: string) {
+  const permitted = new URL(target);
+  const sockets = new Set<Socket>();
+  let blockedRequests = 0;
+  let attempts = 0;
+  let closed = false;
+  const destination = async (url: URL) => {
+    try {
+      if (++attempts > 1500) throw new Error("Connection budget exceeded");
+      const address = await checkedRenderAddress(url, permitted);
+      if (closed) throw new Error("Network closed");
+      return address;
+    } catch (error) {
+      blockedRequests++;
+      throw error;
+    }
+  };
+  const track = (socket: Socket) => {
+    sockets.add(socket);
+    socket.setTimeout(60000, () => socket.destroy());
+    socket.on("close", () => sockets.delete(socket));
+    socket.on("error", () => socket.destroy());
+    return socket;
+  };
+  // CONNECT needs raw sockets; pin the validated IP at the socket, never resolve it again.
+  const server = createServer(async (incoming, outgoing) => {
+    try {
+      const url = new URL(incoming.url!);
+      if (url.protocol !== "http:" || url.username || url.password || !["GET", "HEAD"].includes(incoming.method!))
+        throw new Error("Invalid proxy request");
+      const address = await destination(url);
+      const headers: OutgoingHttpHeaders = { ...incoming.headers, host: url.host };
+      delete headers["proxy-connection"];
+      delete headers["proxy-authorization"];
+      const upstream = request(
+        {
+          hostname: address,
+          port: port(url),
+          path: url.pathname + url.search,
+          method: incoming.method,
+          headers,
+          agent: false,
+        },
+        (response) => {
+          outgoing.writeHead(response.statusCode!, response.headers);
+          pipeline(response, outgoing, () => upstream.destroy());
+        },
+      );
+      upstream.on("socket", track);
+      upstream.on("error", () => {
+        outgoing.destroy();
+      });
+      outgoing.on("close", () => upstream.destroy());
+      pipeline(incoming, upstream, (error) => {
+        if (error) outgoing.destroy();
+      });
+    } catch {
+      outgoing.writeHead(403).end();
+    }
+  });
+  server.maxConnections = 256;
+  server.on("connection", track);
+  server.on("connect", async (incoming, client, head) => {
+    try {
+      const url = new URL(`https://${incoming.url}`);
+      if (url.username || url.password || url.pathname !== "/" || url.search || url.hash)
+        throw new Error("Invalid tunnel");
+      const address = await destination(url);
+      const upstream = track(connect({ host: address, port: port(url) }));
+      upstream.once("connect", () => {
+        client.write("HTTP/1.1 200 Connection Established\r\n\r\n");
+        if (head.length) upstream.write(head);
+        client.pipe(upstream);
+        upstream.pipe(client);
+      });
+      client.on("close", () => upstream.destroy());
+      upstream.on("close", () => client.destroy());
+    } catch {
+      client.end("HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n");
+    }
+  });
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", resolve);
+  });
+  const address = server.address() as { port: number };
+  return {
+    proxy: `http://127.0.0.1:${address.port}`,
+    get blockedRequests() {
+      return blockedRequests;
+    },
+    async close() {
+      closed = true;
+      for (const socket of sockets) socket.destroy();
+      await new Promise<void>((resolve) => server.close(() => resolve()));
+    },
+  };
+}