pagesight 0.17.0 → 0.18.0

package/src/tools/speed/crux.ts

@@ -0,0 +1,176 @@
+import { type CruxHistoryResponse, type CruxResponse } from "../../providers/crux.js";
+const METRIC_LABELS: Record<string, string> = {
+  cumulative_layout_shift: "CLS",
+  first_contentful_paint: "FCP",
+  interaction_to_next_paint: "INP",
+  largest_contentful_paint: "LCP",
+  experimental_time_to_first_byte: "TTFB",
+  round_trip_time: "RTT",
+  navigation_types: "Navigation Types",
+  form_factors: "Form Factors",
+  largest_contentful_paint_image_element_render_delay: "LCP Image Render Delay",
+  largest_contentful_paint_image_resource_load_delay: "LCP Image Load Delay",
+  largest_contentful_paint_image_resource_load_duration: "LCP Image Load Duration",
+  largest_contentful_paint_image_time_to_first_byte: "LCP Image TTFB",
+  largest_contentful_paint_resource_type: "LCP Resource Type",
+};
+
+function formatDate(d: { year: number; month: number; day: number }): string {
+  return `${d.year}-${String(d.month).padStart(2, "0")}-${String(d.day).padStart(2, "0")}`;
+}
+
+export function formatCrux(target: string, result: CruxResponse): string {
+  const r = result.record;
+  const period = r.collectionPeriod;
+  const lines: string[] = [
+    `=== CrUX: ${target} ===`,
+    `Form factor: ${r.key.formFactor ?? "all"}`,
+    `Period: ${formatDate(period.firstDate)} to ${formatDate(period.lastDate)}`,
+    "",
+  ];
+
+  if (result.urlNormalizationDetails) {
+    const norm = result.urlNormalizationDetails;
+    if (norm.originalUrl !== norm.normalizedUrl) {
+      lines.push(`Normalized: ${norm.originalUrl} → ${norm.normalizedUrl}`, "");
+    }
+  }
+
+  lines.push("--- Metrics (p75) ---", "");
+
+  for (const [key, metric] of Object.entries(r.metrics)) {
+    const label = METRIC_LABELS[key] ?? key;
+
+    if (metric.percentiles) {
+      const val = metric.percentiles.p75;
+      const unit = key === "cumulative_layout_shift" ? "" : "ms";
+      lines.push(`${label}: ${val}${unit}`);
+
+      if (metric.histogram) {
+        const buckets = metric.histogram.map((b) => `${Math.round(b.density * 100)}%`).join(" / ");
+        lines.push(`  Distribution (good/needs improvement/poor): ${buckets}`);
+      }
+    } else if (metric.fractions) {
+      lines.push(`${label}:`);
+      for (const [fKey, fVal] of Object.entries(metric.fractions)) {
+        lines.push(`  ${fKey}: ${(fVal * 100).toFixed(1)}%`);
+      }
+    }
+  }
+
+  // Core Web Vitals assessment (Google ranking signal)
+  const cwvMetrics = r.metrics;
+  const lcp = cwvMetrics.largest_contentful_paint?.percentiles?.p75 as number | undefined;
+  const inp = cwvMetrics.interaction_to_next_paint?.percentiles?.p75 as number | undefined;
+  const cls = cwvMetrics.cumulative_layout_shift?.percentiles?.p75 as number | undefined;
+
+  if (lcp !== undefined || inp !== undefined || cls !== undefined) {
+    lines.push("", "--- Core Web Vitals Assessment ---", "");
+    const lcpPass = lcp !== undefined && lcp <= 2500;
+    const inpPass = inp !== undefined && inp <= 200;
+    const clsPass = cls !== undefined && cls <= 0.1;
+
+    if (lcp !== undefined)
+      lines.push(`LCP: ${lcp}ms ${lcpPass ? "GOOD" : lcp <= 4000 ? "NEEDS IMPROVEMENT" : "POOR"} (threshold: 2500ms)`);
+    if (inp !== undefined)
+      lines.push(`INP: ${inp}ms ${inpPass ? "GOOD" : inp <= 500 ? "NEEDS IMPROVEMENT" : "POOR"} (threshold: 200ms)`);
+    if (cls !== undefined)
+      lines.push(`CLS: ${cls} ${clsPass ? "GOOD" : cls <= 0.25 ? "NEEDS IMPROVEMENT" : "POOR"} (threshold: 0.1)`);
+
+    const allPresent = lcp !== undefined && inp !== undefined && cls !== undefined;
+    if (allPresent) {
+      const allPass = lcpPass && inpPass && clsPass;
+      lines.push(
+        "",
+        allPass
+          ? "Overall: PASS — all Core Web Vitals are good (positive ranking signal)"
+          : "Overall: FAIL — not all Core Web Vitals pass (may affect rankings)",
+      );
+    }
+  }
+
+  return lines.join("\n");
+}
+
+export function formatCruxHistory(target: string, result: CruxHistoryResponse): string {
+  const r = result.record;
+  const periods = r.collectionPeriods;
+  const lines: string[] = [
+    `=== CrUX History: ${target} ===`,
+    `Form factor: ${r.key.formFactor ?? "all"}`,
+    `Periods: ${periods.length} (${formatDate(periods[0].firstDate)} to ${formatDate(periods[periods.length - 1].lastDate)})`,
+    "",
+  ];
+
+  if (result.urlNormalizationDetails) {
+    const norm = result.urlNormalizationDetails;
+    if (norm.originalUrl !== norm.normalizedUrl) {
+      lines.push(`Normalized: ${norm.originalUrl} → ${norm.normalizedUrl}`, "");
+    }
+  }
+
+  lines.push("--- p75 Trend ---", "");
+
+  for (const [key, metric] of Object.entries(r.metrics)) {
+    const label = METRIC_LABELS[key] ?? key;
+
+    if (metric.percentilesTimeseries) {
+      const values = metric.percentilesTimeseries.p75s;
+      const first = values[0];
+      const last = values[values.length - 1];
+      const unit = key === "cumulative_layout_shift" ? "" : "ms";
+
+      if (first === null && last === null) {
+        lines.push(`${label}: insufficient data`);
+        continue;
+      }
+
+      lines.push(
+        `${label}: ${first ?? "N/A"}${first !== null ? unit : ""} → ${last ?? "N/A"}${last !== null ? unit : ""} (${values.length} points)`,
+      );
+
+      // Show trend direction
+      if (first !== null && last !== null) {
+        const f = Number(first);
+        const l = Number(last);
+        if (!Number.isNaN(f) && !Number.isNaN(l)) {
+          const change = ((l - f) / f) * 100;
+          const dir = change > 5 ? "worse" : change < -5 ? "improved" : "stable";
+          lines.push(`  Trend: ${change > 0 ? "+" : ""}${change.toFixed(1)}% (${dir})`);
+        }
+      }
+    } else if (metric.fractionTimeseries) {
+      lines.push(`${label}: (fraction timeseries, ${periods.length} points)`);
+      for (const [fKey, fData] of Object.entries(metric.fractionTimeseries)) {
+        const fracs = fData.fractions;
+        const first = fracs[0];
+        const last = fracs[fracs.length - 1];
+        if (first !== null && last !== null && !Number.isNaN(first) && !Number.isNaN(last)) {
+          lines.push(`  ${fKey}: ${(first * 100).toFixed(1)}% → ${(last * 100).toFixed(1)}%`);
+        }
+      }
+    }
+  }
+
+  // Show last 5 data points as table for core metrics
+  const coreMetrics = ["largest_contentful_paint", "interaction_to_next_paint", "cumulative_layout_shift"];
+  const available = coreMetrics.filter((m) => r.metrics[m]?.percentilesTimeseries);
+
+  if (available.length > 0 && periods.length >= 5) {
+    lines.push("", "--- Recent Data Points ---", "");
+    const lastN = 5;
+    const startIdx = periods.length - lastN;
+
+    lines.push(`${"Date".padEnd(12)} ${available.map((m) => (METRIC_LABELS[m] ?? m).padEnd(10)).join(" ")}`);
+    for (let i = startIdx; i < periods.length; i++) {
+      const date = formatDate(periods[i].lastDate);
+      const vals = available.map((m) => {
+        const v = r.metrics[m].percentilesTimeseries?.p75s[i];
+        return String(v ?? "N/A").padEnd(10);
+      });
+      lines.push(`${date.padEnd(12)} ${vals.join(" ")}`);
+    }
+  }
+
+  return lines.join("\n");
+}

package/src/tools/speed/pagespeed.ts

@@ -0,0 +1,273 @@
+import { type PsiAudit, type PsiAuditDetailItem, type PsiResult } from "../../providers/pagespeed.js";
+export const QUOTA_NOTE =
+  "\n\nNote: No GOOGLE_API_KEY configured — using shared quota (400 req/day). Set your own key to avoid rate limits.";
+
+function scoreLabel(score: number | null): string {
+  if (score === null) return "N/A";
+  const pct = Math.round(score * 100);
+  if (pct >= 90) return `${pct} (good)`;
+  if (pct >= 50) return `${pct} (needs improvement)`;
+  return `${pct} (poor)`;
+}
+
+export function scorePct(score: number | null): number | null {
+  return score === null ? null : Math.round(score * 100);
+}
+
+function cwvRating(category: string): string {
+  if (category === "FAST") return "good";
+  if (category === "AVERAGE") return "needs improvement";
+  if (category === "SLOW") return "poor";
+  return category;
+}
+
+function formatLoadingExperience(label: string, exp: PsiResult["loadingExperience"]): string[] {
+  if (!exp?.metrics || Object.keys(exp.metrics).length === 0) return [];
+
+  const lines: string[] = [`--- ${label} (CrUX Field Data) ---`, ""];
+  lines.push(`Overall: ${cwvRating(exp.overall_category)}`, "");
+
+  const metricNames: Record<string, string> = {
+    CUMULATIVE_LAYOUT_SHIFT_SCORE: "CLS",
+    EXPERIMENTAL_TIME_TO_FIRST_BYTE: "TTFB",
+    FIRST_CONTENTFUL_PAINT_MS: "FCP",
+    FIRST_INPUT_DELAY_MS: "FID",
+    INTERACTION_TO_NEXT_PAINT: "INP",
+    LARGEST_CONTENTFUL_PAINT_MS: "LCP",
+  };
+
+  for (const [key, metric] of Object.entries(exp.metrics)) {
+    const name = metricNames[key] ?? key;
+    const unit = key.includes("LAYOUT_SHIFT") ? "" : "ms";
+    const value = key.includes("LAYOUT_SHIFT") ? (metric.percentile / 100).toFixed(2) : `${metric.percentile}${unit}`;
+    lines.push(`${name}: ${value} (${cwvRating(metric.category)})`);
+  }
+
+  return lines;
+}
+
+function formatOpportunities(audits: Record<string, PsiAudit>): string[] {
+  const opportunities: PsiAudit[] = [];
+
+  for (const audit of Object.values(audits)) {
+    if (audit.score === null || audit.score >= 1) continue;
+    const mode = audit.scoreDisplayMode;
+    const hasItems = (audit.details?.items?.length ?? 0) > 0;
+    const hasNumeric = audit.numericValue && audit.numericValue > 0;
+    if (mode === "metricSavings" || ((mode === "numeric" || mode === "binary") && hasNumeric)) {
+      if (hasNumeric || hasItems) {
+        opportunities.push(audit);
+      }
+    }
+  }
+
+  if (opportunities.length === 0) return [];
+
+  opportunities.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
+
+  const lines: string[] = ["--- Opportunities ---", ""];
+  for (const audit of opportunities.slice(0, 10)) {
+    const severity = (audit.score ?? 0) < 0.5 ? "HIGH" : (audit.score ?? 0) < 0.9 ? "MEDIUM" : "LOW";
+    const unit = audit.numericUnit === "millisecond" ? "ms" : audit.numericUnit === "byte" ? " bytes" : "";
+    const savings = audit.displayValue ?? `${Math.round(audit.numericValue ?? 0)}${unit}`;
+    lines.push(`${severity}  ${audit.title}`);
+    lines.push(`  Potential savings: ${savings}`);
+
+    const items = audit.details?.items;
+    if (items && items.length > 0) {
+      for (const item of items.slice(0, 3)) {
+        lines.push(...formatDetailItem(item));
+      }
+      if (items.length > 3) {
+        lines.push(`  ... and ${items.length - 3} more resources`);
+      }
+    }
+    lines.push("");
+  }
+
+  return lines;
+}
+
+function formatDiagnostics(audits: Record<string, PsiAudit>): string[] {
+  const failing: PsiAudit[] = [];
+
+  for (const audit of Object.values(audits)) {
+    if (
+      audit.score !== null &&
+      audit.score < 0.5 &&
+      (audit.scoreDisplayMode === "numeric" || audit.scoreDisplayMode === "metricSavings") &&
+      audit.displayValue
+    ) {
+      failing.push(audit);
+    }
+  }
+
+  if (failing.length === 0) return [];
+
+  failing.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
+
+  const lines: string[] = ["--- Diagnostics ---", ""];
+  for (const audit of failing.slice(0, 10)) {
+    lines.push(`${audit.title}: ${audit.displayValue}`);
+
+    const linkMatch = audit.description?.match(/\[.*?\]\((https?:\/\/[^)]+)\)/);
+    if (linkMatch) lines.push(`  Learn more: ${linkMatch[1]}`);
+
+    const items = audit.details?.items;
+    if (items && items.length > 0) {
+      for (const item of items.slice(0, 3)) {
+        lines.push(...formatDetailItem(item));
+      }
+      if (items.length > 3) {
+        lines.push(`  ... and ${items.length - 3} more`);
+      }
+    }
+    lines.push("");
+  }
+
+  return lines;
+}
+
+function formatDetailItem(item: PsiAuditDetailItem): string[] {
+  const lines: string[] = [];
+
+  if (item.node) {
+    const n = item.node;
+    if (n.nodeLabel) lines.push(`  Element: "${n.nodeLabel}"`);
+    if (n.selector) lines.push(`  Selector: ${n.selector}`);
+    if (n.snippet) lines.push(`  HTML: ${n.snippet}`);
+    if (n.explanation) lines.push(`  Issue: ${n.explanation}`);
+  } else if (item.url) {
+    const parts = [`  ${item.url}`];
+    if (item.wastedMs) parts.push(`wastedMs=${Math.round(item.wastedMs)}`);
+    if (item.wastedBytes) parts.push(`wastedBytes=${Math.round(item.wastedBytes)}`);
+    if (item.totalBytes) parts.push(`totalBytes=${Math.round(item.totalBytes)}`);
+    lines.push(parts.join("  "));
+  }
+
+  return lines;
+}
+
+function formatFailingAudits(audits: Record<string, PsiAudit>, categoryRefs: string[]): string[] {
+  const failing: PsiAudit[] = [];
+
+  for (const ref of categoryRefs) {
+    const audit = audits[ref];
+    if (audit && audit.score !== null && audit.score < 1) {
+      failing.push(audit);
+    }
+  }
+
+  if (failing.length === 0) return [];
+
+  failing.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
+
+  const lines: string[] = [];
+  for (const audit of failing) {
+    const severity = audit.score === 0 ? "FAIL" : (audit.score ?? 0) < 0.5 ? "WARN" : "INFO";
+    lines.push(`[${severity}] ${audit.title}`);
+    if (audit.displayValue) lines.push(`  Value: ${audit.displayValue}`);
+
+    // Extract learn-more URL from description markdown
+    const linkMatch = audit.description?.match(/\[.*?\]\((https?:\/\/[^)]+)\)/);
+    if (linkMatch) lines.push(`  Learn more: ${linkMatch[1]}`);
+
+    const items = audit.details?.items;
+    if (items && items.length > 0) {
+      const maxItems = 5;
+      for (const item of items.slice(0, maxItems)) {
+        lines.push(...formatDetailItem(item));
+      }
+      if (items.length > maxItems) {
+        lines.push(`  ... and ${items.length - maxItems} more`);
+      }
+    }
+    lines.push("");
+  }
+
+  return lines;
+}
+
+export function formatPagespeed(url: string, result: PsiResult): string {
+  const lhr = result.lighthouseResult;
+  const lines: string[] = [
+    `=== PageSpeed: ${url} ===`,
+    `Strategy: ${lhr.configSettings.emulatedFormFactor}`,
+    `Lighthouse: ${lhr.lighthouseVersion}`,
+    `Analyzed: ${result.analysisUTCTimestamp}`,
+    "",
+  ];
+
+  // Runtime errors
+  if (lhr.runtimeError) {
+    lines.push(`ERROR: ${lhr.runtimeError.code} — ${lhr.runtimeError.message}`, "");
+  }
+
+  // Warnings
+  if (lhr.runWarnings && lhr.runWarnings.length > 0) {
+    for (const w of lhr.runWarnings) {
+      lines.push(`WARNING: ${w}`);
+    }
+    lines.push("");
+  }
+
+  // Category scores
+  lines.push("--- Scores ---", "");
+  for (const cat of Object.values(lhr.categories)) {
+    lines.push(`${cat.title}: ${scoreLabel(cat.score)}`);
+  }
+  lines.push("");
+
+  // Lab performance metrics
+  const cwvIds = [
+    "first-contentful-paint",
+    "largest-contentful-paint",
+    "total-blocking-time",
+    "cumulative-layout-shift",
+    "speed-index",
+    "interactive",
+  ];
+  const cwvLines: string[] = [];
+  for (const id of cwvIds) {
+    const audit = lhr.audits[id];
+    if (audit?.displayValue) {
+      cwvLines.push(`${audit.title}: ${audit.displayValue} ${scoreLabel(audit.score)}`);
+    }
+  }
+  if (cwvLines.length > 0) {
+    lines.push("--- Lab Performance Metrics ---", "", ...cwvLines, "");
+  }
+
+  // CrUX field data
+  const pageExp = formatLoadingExperience("Page", result.loadingExperience);
+  if (pageExp.length > 0) lines.push(...pageExp, "");
+
+  const originExp = formatLoadingExperience("Origin", result.originLoadingExperience);
+  if (originExp.length > 0) lines.push(...originExp, "");
+
+  // Opportunities
+  const opps = formatOpportunities(lhr.audits);
+  if (opps.length > 0) lines.push(...opps, "");
+
+  // Diagnostics
+  const diags = formatDiagnostics(lhr.audits);
+  if (diags.length > 0) lines.push(...diags, "");
+
+  // Failing audits per non-performance category (a11y, SEO, best practices)
+  for (const cat of Object.values(lhr.categories)) {
+    if (cat.id === "performance" || !cat.auditRefs) continue;
+    const score = cat.score !== null ? Math.round(cat.score * 100) : null;
+    if (score === null || score >= 100) continue;
+
+    const refs = cat.auditRefs.map((r) => r.id);
+    const details = formatFailingAudits(lhr.audits, refs);
+    if (details.length > 0) {
+      lines.push(`--- ${cat.title} Issues ---`, "", ...details);
+    }
+  }
+
+  // Timing
+  lines.push(`Analysis took ${(lhr.timing.total / 1000).toFixed(1)}s`);
+
+  return lines.join("\n");
+}

package/src/tools/speed/schema.ts

@@ -0,0 +1,39 @@
+import { z } from "zod";
+export const speedSchema = {
+  action: z
+    .enum(["pagespeed", "crux", "crux_history"])
+    .optional()
+    .describe("Which analysis to run. Auto-detected: 'pagespeed' when url/urls provided, 'crux' when origin provided."),
+  url: z.string().url().optional().describe("URL to analyze (PageSpeed or CrUX)."),
+  urls: z
+    .array(z.string().url())
+    .min(2)
+    .max(10)
+    .optional()
+    .describe("Multiple URLs (2-10) for batch PageSpeed. 2 = compare, 3+ = summary table."),
+  strategy: z.enum(["mobile", "desktop"]).optional().describe("Device strategy for PageSpeed. Default: 'mobile'."),
+  categories: z
+    .array(z.enum(["performance", "accessibility", "best-practices", "seo"]))
+    .optional()
+    .describe("Lighthouse categories. Default: all four."),
+  locale: z.string().optional().describe("Locale for PageSpeed results."),
+  origin: z.string().optional().describe("Origin for CrUX data (e.g., 'https://example.com'). Triggers CrUX mode."),
+  form_factor: z.enum(["DESKTOP", "PHONE", "TABLET"]).optional().describe("CrUX device filter."),
+  metrics: z
+    .array(
+      z.enum([
+        "cumulative_layout_shift",
+        "first_contentful_paint",
+        "interaction_to_next_paint",
+        "largest_contentful_paint",
+        "experimental_time_to_first_byte",
+        "round_trip_time",
+        "navigation_types",
+        "form_factors",
+      ]),
+    )
+    .optional()
+    .describe("CrUX metrics to query."),
+  periods: z.number().min(1).max(40).optional().describe("CrUX history periods (1-40). Default: 25."),
+};
+export type SpeedOptions = z.infer<z.ZodObject<typeof speedSchema>>;

package/src/tools/speed/tool.ts

@@ -0,0 +1,11 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { speedSchema } from "./schema.js";
+import { analyzeSpeed } from "./analyze.js";
+export function registerSpeedTool(server: McpServer): void {
+  server.tool(
+    "speed",
+    "Analyze site performance. Run PageSpeed Insights (lab metrics, Lighthouse scores, opportunities) for single or multiple URLs, or query Chrome UX Report for real-world field data and historical trends.",
+    speedSchema,
+    analyzeSpeed,
+  );
+}

package/src/web/fetch.ts

@@ -0,0 +1,31 @@
+import { readBounded, RequestError } from "../shared/http.js";
+
+export async function fetchText(url: string, maxBytes: number, origin?: string) {
+  const redirects: Array<{ url: string; status: number; location: string }> = [];
+  let current = url;
+  const signal = AbortSignal.timeout(20_000);
+  for (let hop = 0; hop <= 5; hop++) {
+    const parsed = new URL(current);
+    if (
+      !["http:", "https:"].includes(parsed.protocol) ||
+      parsed.username ||
+      parsed.password ||
+      (origin && parsed.origin !== origin)
+    )
+      throw new RequestError("Redirect left the permitted origin or protocol", null, "invalid_redirect");
+    const response = await fetch(current, {
+      signal,
+      redirect: "manual",
+      headers: { "User-Agent": "Pagesight/0.17", Accept: "text/html,application/xml,text/plain" },
+    });
+    const location = response.headers.get("location");
+    if ([301, 302, 303, 307, 308].includes(response.status) && location) {
+      redirects.push({ url: current, status: response.status, location });
+      await response.body?.cancel();
+      current = new URL(location, current).href;
+      continue;
+    }
+    return { response, body: await readBounded(response, maxBytes), finalUrl: current, redirects };
+  }
+  throw new RequestError("Redirect limit exceeded", null, "redirect_limit");
+}

package/src/web/images.ts

@@ -0,0 +1,62 @@
+/** Inspect fetched markup without executing scripts or requesting image URLs. */
+export async function imageEvidence(html: string) {
+  const images: Array<{
+    src: string | null;
+    altPresent: boolean;
+    altText: string | null;
+    inNoscript: boolean;
+    width: string | null;
+    height: string | null;
+    role: string | null;
+    ariaHidden: string | null;
+  }> = [];
+  let omittedImages = 0;
+  const noscript: string[] = [];
+  let currentNoscript = "";
+  const handler = (inNoscript: boolean) => ({
+    element(el: HTMLRewriterTypes.Element) {
+      if (images.length >= 200) {
+        omittedImages++;
+        return;
+      }
+      const altPresent = el.hasAttribute("alt");
+      images.push({
+        src: el.getAttribute("src"),
+        altPresent,
+        altText: altPresent ? (el.getAttribute("alt") ?? "") : null,
+        inNoscript,
+        width: el.getAttribute("width"),
+        height: el.getAttribute("height"),
+        role: el.getAttribute("role"),
+        ariaHidden: el.getAttribute("aria-hidden"),
+      });
+    },
+  });
+  await new HTMLRewriter()
+    .on("img", handler(false))
+    .on("noscript", {
+      element(el) {
+        currentNoscript = "";
+        el.onEndTag(() => {
+          noscript.push(currentNoscript);
+          currentNoscript = "";
+        });
+      },
+      text(chunk) {
+        currentNoscript += chunk.text;
+      },
+    })
+    .transform(new Response(html))
+    .text();
+  if (currentNoscript) noscript.push(currentNoscript);
+  for (const fragment of noscript)
+    await new HTMLRewriter().on("img", handler(true)).transform(new Response(fragment)).text();
+  return {
+    images,
+    omittedImages,
+    truncated: omittedImages > 0,
+    warnings: [
+      "Images are fetched HTML evidence, including noscript fallback markup; scripts and image URLs were not loaded. Empty ALT may be appropriate for decorative images; meaning requires context.",
+    ],
+  };
+}

package/src/web/page-observation.ts

@@ -0,0 +1,69 @@
+import { imageEvidence } from "./images.js";
+import { fetchText } from "./fetch.js";
+
+export async function observePage(url: string) {
+  const { response, body, finalUrl, redirects } = await fetchText(url, 2_000_000);
+  const warnings = ["Fetched HTML only; JavaScript execution and crawler access were not tested."];
+  let title = "";
+  const metadata: { canonical: string | null; description: string | null } = { canonical: null, description: null };
+  const robots: string[] = [];
+  const data: string[] = [];
+  const rewriter = new HTMLRewriter()
+    .on("title", {
+      text(chunk) {
+        title += chunk.text;
+      },
+    })
+    .on('link[rel="canonical"]', {
+      element(el) {
+        const href = el.getAttribute("href");
+        if (href) {
+          try {
+            metadata.canonical = new URL(href, finalUrl).href;
+          } catch {
+            warnings.push("Malformed canonical href; canonical is unknown.");
+          }
+        }
+      },
+    })
+    .on("meta", {
+      element(el) {
+        const name = el.getAttribute("name")?.toLowerCase();
+        const value = el.getAttribute("content");
+        if (name === "description") metadata.description = value;
+        if ((name === "robots" || name === "googlebot") && value) robots.push(`${name}: ${value}`);
+      },
+    })
+    .on('script[type="application/ld+json"]', {
+      text(chunk) {
+        if (!data.length) data.push("");
+        data[data.length - 1] += chunk.text;
+        if (chunk.lastInTextNode) data.push("");
+      },
+    });
+  await rewriter.transform(new Response(body)).text();
+  return {
+    url,
+    finalUrl,
+    status: response.status,
+    redirects,
+    contentType: response.headers.get("content-type"),
+    xRobotsTag: response.headers.get("x-robots-tag"),
+    title: title.trim() || null,
+    description: metadata.description,
+    descriptionLength: metadata.description === null ? null : Array.from(metadata.description).length,
+    canonical: metadata.canonical,
+    robots,
+    sha256: Bun.CryptoHasher.hash("sha256", body, "hex"),
+    bytes: Buffer.byteLength(body),
+    structuredData: data.filter(Boolean).map((text) => {
+      try {
+        return { value: JSON.parse(text), validJson: true };
+      } catch {
+        return { value: null, validJson: false };
+      }
+    }),
+    imageEvidence: await imageEvidence(body),
+    warnings,
+  };
+}